a
/
badguardhome
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge: new Statistics module 

Close #827

* commit '17a26155cdf0150ea9d8cf915311f16fafa6354a':
  - client: convert average processing time to milliseconds
  + client: handle clear statistics
  + client: update time interval notice
  + client: add notice to the time interval input
  + client: functional components for dashboard
  * client: fix missed translations
  * client: change the result message after adding a filter
  + client: handle the new statistics format
  + client: handle time interval for statistics
  + openapi: add /stats_config, /stats_info methods
  * openapi: update /stats method
  + statistics: store in separate file
  * stats: remove old code
  + config: "statistics_interval" setting
  + /control/stats_config, /control/stats_info
This commit is contained in:
Simon Zolin 2019-09-04 12:39:42 +03:00
parent 47e5fcb14c 17a26155cd
commit e346eddf94
51 changed files with 2177 additions and 1838 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
AGHTechDoc.md
View File

@ -37,6 +37,11 @@ Contents:
* Services Filter
* API: Get blocked services list
* API: Set blocked services list
* Statistics
* API: Get statistics data
* API: Clear statistics data
* API: Set statistics parameters
* API: Get statistics parameters
## First startup
@ -863,3 +868,111 @@ Request:
Response:
200 OK
## Statistics
Load (main thread):
. Load data from the last bucket from DB for the current hour
Runtime (DNS worker threads):
. Update current unit
Runtime (goroutine):
. Periodically check that current unit should be flushed to file (when the current hour changes)
. If so, flush it, allocate a new empty unit
Runtime (HTTP worker threads):
. To respond to "Get statistics" API request we:
. load all units from file
. load current unit
. process data from all loaded units:
. sum up data for "total counters" output values
. add value into "per time unit counters" output arrays
. aggregate data for "top_" output arrays; sort in descending order
Unload (main thread):
. Flush current unit to file
### API: Get statistics data
Request:
GET /control/stats
Response:
200 OK
{
time_units: hours | days
// total counters:
num_dns_queries: 123
num_blocked_filtering: 123
num_replaced_safebrowsing: 123
num_replaced_safesearch: 123
num_replaced_parental: 123
avg_processing_time: 123.123
// per time unit counters
dns_queries: [123, ...]
blocked_filtering: [123, ...]
replaced_parental: [123, ...]
replaced_safebrowsing: [123, ...]
top_queried_domains: [
{host: 123},
...
]
top_blocked_domains: [
{host: 123},
...
]
top_clients: [
{IP: 123},
...
]
}
### API: Clear statistics data
Request:
POST /control/stats_reset
Response:
200 OK
### API: Set statistics parameters
Request:
POST /control/stats_config
{
"interval": 1 | 7 | 30 | 90
}
Response:
200 OK
### API: Get statistics parameters
Request:
GET /control/stats_info
Response:
200 OK
{
"interval": 1 | 7 | 30 | 90
}

2
client/package.json vendored
View File

@ -6,7 +6,7 @@
"build-dev": "NODE_ENV=development ./node_modules/.bin/webpack --config webpack.dev.js",
"watch": "NODE_ENV=development ./node_modules/.bin/webpack --config webpack.dev.js --watch",
"build-prod": "NODE_ENV=production ./node_modules/.bin/webpack --config webpack.prod.js",
"lint": "eslint frontend/"
"lint": "eslint client/"
},
"dependencies": {
"@nivo/line": "^0.49.1",

26
client/src/__locales/en.json
View File

@ -4,7 +4,6 @@
"upstream_parallel": "Use parallel queries to speed up resolving by simultaneously querying all upstream servers",
"bootstrap_dns": "Bootstrap DNS servers",
"bootstrap_dns_desc": "Bootstrap DNS servers are used to resolve IP addresses of the DoH/DoT resolvers you specify as upstreams.",
"url_added_successfully": "URL added successfully",
"check_dhcp_servers": "Check for DHCP servers",
"save_config": "Save config",
"enabled_dhcp": "DHCP server enabled",
@ -67,18 +66,22 @@
"disabled_protection": "Disabled protection",
"refresh_statics": "Refresh statistics",
"dns_query": "DNS Queries",
"blocked_by": "Blocked by Filters",
"blocked_by": "<0>Blocked by Filters</0>",
"stats_malware_phishing": "Blocked malware/phishing",
"stats_adult": "Blocked adult websites",
"stats_query_domain": "Top queried domains",
"for_last_24_hours": "for the last 24 hours",
"for_last_days": "for the last {{count}} day",
"for_last_days_plural": "for the last {{count}} days",
"no_domains_found": "No domains found",
"requests_count": "Requests count",
"top_blocked_domains": "Top blocked domains",
"top_clients": "Top clients",
"no_clients_found": "No clients found",
"general_statistics": "General statistics",
"number_of_dns_query_24_hours": "A number of DNS quieries processed for the last 24 hours",
"number_of_dns_query_days": "A number of DNS queries processed for the last {{count}} day",
"number_of_dns_query_days_plural": "A number of DNS queries processed for the last {{count}} days",
"number_of_dns_query_24_hours": "A number of DNS queries processed for the last 24 hours",
"number_of_dns_query_blocked_24_hours": "A number of DNS requests blocked by adblock filters and hosts blocklists",
"number_of_dns_query_blocked_24_hours_by_sec": "A number of DNS requests blocked by the AdGuard browsing security module",
"number_of_dns_query_blocked_24_hours_adult": "A number of adult websites blocked",
@ -300,7 +303,6 @@
"client_deleted": "Client \"{{key}}\" successfully deleted",
"client_added": "Client \"{{key}}\" successfully added",
"client_updated": "Client \"{{key}}\" successfully updated",
"table_statistics": "Requests count (last 24 hours)",
"clients_not_found": "No clients found",
"client_confirm_delete": "Are you sure you want to delete client \"{{key}}\"?",
"filter_confirm_delete": "Are you sure you want to delete filter?",
@ -361,5 +363,19 @@
"encryption_certificates_source_path": "Set a certificates file path",
"encryption_certificates_source_content":"Paste the certificates contents",
"encryption_key_source_path": "Set a private key file",
"encryption_key_source_content": "Paste the private key contents"
"encryption_key_source_content": "Paste the private key contents",
"stats_params": "Statistics configuration",
"config_successfully_saved": "Configuration successfully saved",
"interval_24_hour": "24 hours",
"interval_days": "{{count}} day",
"interval_days_plural": "{{count}} days",
"domain": "Domain",
"answer": "Answer",
"filter_added_successfully": "The filter has been successfully added",
"statistics_logs": "Statistics and logs",
"statistics_retention": "Statistics retention",
"statistics_retention_desc": "If you decrease the interval value, some data will be lost",
"statistics_clear": " Clear statistics",
"statistics_clear_confirm": "Are you sure you want to clear statistics?",
"statistics_cleared": "Statistics successfully cleared"
}

61
client/src/actions/index.js
View File

@ -1,11 +1,10 @@
import { createAction } from 'redux-actions';
import round from 'lodash/round';
import { t } from 'i18next';
import { showLoading, hideLoading } from 'react-redux-loading-bar';
import axios from 'axios';
import versionCompare from '../helpers/versionCompare';
import { normalizeHistory, normalizeFilteringStatus, normalizeLogs, normalizeTextarea, sortClients } from '../helpers/helpers';
import { normalizeFilteringStatus, normalizeLogs, normalizeTextarea, sortClients } from '../helpers/helpers';
import { SETTINGS_NAMES, CHECK_TIMEOUT } from '../helpers/constants';
import { getTlsStatus } from './encryption';
import Api from '../api/Api';
@ -246,27 +245,6 @@ export const getClients = () => async (dispatch) => {
}
};
export const getTopStatsRequest = createAction('GET_TOP_STATS_REQUEST');
export const getTopStatsFailure = createAction('GET_TOP_STATS_FAILURE');
export const getTopStatsSuccess = createAction('GET_TOP_STATS_SUCCESS');
export const getTopStats = () => async (dispatch, getState) => {
dispatch(getTopStatsRequest());
const timer = setInterval(async () => {
const state = getState();
if (state.dashboard.isCoreRunning) {
clearInterval(timer);
try {
const stats = await apiClient.getGlobalStatsTop();
dispatch(getTopStatsSuccess(stats));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(getTopStatsFailure(error));
}
}
}, 100);
};
export const dnsStatusRequest = createAction('DNS_STATUS_REQUEST');
export const dnsStatusFailure = createAction('DNS_STATUS_FAILURE');
export const dnsStatusSuccess = createAction('DNS_STATUS_SUCCESS');
@ -314,27 +292,6 @@ export const disableDns = () => async (dispatch) => {
}
};
export const getStatsRequest = createAction('GET_STATS_REQUEST');
export const getStatsFailure = createAction('GET_STATS_FAILURE');
export const getStatsSuccess = createAction('GET_STATS_SUCCESS');
export const getStats = () => async (dispatch) => {
dispatch(getStatsRequest());
try {
const stats = await apiClient.getGlobalStats();
const processedStats = {
...stats,
avg_processing_time: round(stats.avg_processing_time, 2),
};
dispatch(getStatsSuccess(processedStats));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(getStatsFailure());
}
};
export const getLogsRequest = createAction('GET_LOGS_REQUEST');
export const getLogsFailure = createAction('GET_LOGS_FAILURE');
export const getLogsSuccess = createAction('GET_LOGS_SUCCESS');
@ -473,22 +430,6 @@ export const refreshFilters = () => async (dispatch) => {
export const handleRulesChange = createAction('HANDLE_RULES_CHANGE');
export const getStatsHistoryRequest = createAction('GET_STATS_HISTORY_REQUEST');
export const getStatsHistoryFailure = createAction('GET_STATS_HISTORY_FAILURE');
export const getStatsHistorySuccess = createAction('GET_STATS_HISTORY_SUCCESS');
export const getStatsHistory = () => async (dispatch) => {
dispatch(getStatsHistoryRequest());
try {
const statsHistory = await apiClient.getGlobalStatsHistory();
const normalizedHistory = normalizeHistory(statsHistory);
dispatch(getStatsHistorySuccess(normalizedHistory));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(getStatsHistoryFailure());
}
};
export const addFilterRequest = createAction('ADD_FILTER_REQUEST');
export const addFilterFailure = createAction('ADD_FILTER_FAILURE');
export const addFilterSuccess = createAction('ADD_FILTER_SUCCESS');

78
client/src/actions/stats.js Normal file
View File

@ -0,0 +1,78 @@
import { createAction } from 'redux-actions';
import Api from '../api/Api';
import { addErrorToast, addSuccessToast } from './index';
import { normalizeTopStats, secondsToMilliseconds } from '../helpers/helpers';
const apiClient = new Api();
export const getStatsConfigRequest = createAction('GET_LOGS_CONFIG_REQUEST');
export const getStatsConfigFailure = createAction('GET_LOGS_CONFIG_FAILURE');
export const getStatsConfigSuccess = createAction('GET_LOGS_CONFIG_SUCCESS');
export const getStatsConfig = () => async (dispatch) => {
dispatch(getStatsConfigRequest());
try {
const data = await apiClient.getStatsInfo();
dispatch(getStatsConfigSuccess(data));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(getStatsConfigFailure());
}
};
export const setStatsConfigRequest = createAction('SET_STATS_CONFIG_REQUEST');
export const setStatsConfigFailure = createAction('SET_STATS_CONFIG_FAILURE');
export const setStatsConfigSuccess = createAction('SET_STATS_CONFIG_SUCCESS');
export const setStatsConfig = config => async (dispatch) => {
dispatch(setStatsConfigRequest());
try {
await apiClient.setStatsConfig(config);
dispatch(addSuccessToast('config_successfully_saved'));
dispatch(setStatsConfigSuccess(config));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(setStatsConfigFailure());
}
};
export const getStatsRequest = createAction('GET_STATS_REQUEST');
export const getStatsFailure = createAction('GET_STATS_FAILURE');
export const getStatsSuccess = createAction('GET_STATS_SUCCESS');
export const getStats = () => async (dispatch) => {
dispatch(getStatsRequest());
try {
const stats = await apiClient.getStats();
const normalizedStats = {
...stats,
top_blocked_domains: normalizeTopStats(stats.top_blocked_domains),
top_clients: normalizeTopStats(stats.top_clients),
top_queried_domains: normalizeTopStats(stats.top_queried_domains),
avg_processing_time: secondsToMilliseconds(stats.avg_processing_time),
};
dispatch(getStatsSuccess(normalizedStats));
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(getStatsFailure());
}
};
export const resetStatsRequest = createAction('RESET_STATS_REQUEST');
export const resetStatsFailure = createAction('RESET_STATS_FAILURE');
export const resetStatsSuccess = createAction('RESET_STATS_SUCCESS');
export const resetStats = () => async (dispatch) => {
dispatch(getStatsRequest());
try {
await apiClient.resetStats();
dispatch(addSuccessToast('statistics_cleared'));
dispatch(resetStatsSuccess());
} catch (error) {
dispatch(addErrorToast({ error }));
dispatch(resetStatsFailure());
}
};

66
client/src/api/Api.js
View File

@ -1,6 +1,4 @@
import axios from 'axios';
import subHours from 'date-fns/sub_hours';
import dateFormat from 'date-fns/format';
export default class Api {
baseUrl = 'control';
@ -24,13 +22,9 @@ export default class Api {
}
// Global methods
GLOBAL_RESTART = { path: 'restart', method: 'POST' };
GLOBAL_START = { path: 'start', method: 'POST' };
GLOBAL_STATS = { path: 'stats', method: 'GET' };
GLOBAL_STATS_HISTORY = { path: 'stats_history', method: 'GET' };
GLOBAL_STATUS = { path: 'status', method: 'GET' };
GLOBAL_STOP = { path: 'stop', method: 'POST' };
GLOBAL_STATS_TOP = { path: 'stats_top', method: 'GET' };
GLOBAL_QUERY_LOG = { path: 'querylog', method: 'GET' };
GLOBAL_QUERY_LOG_ENABLE = { path: 'querylog_enable', method: 'POST' };
GLOBAL_QUERY_LOG_DISABLE = { path: 'querylog_disable', method: 'POST' };
@ -41,11 +35,6 @@ export default class Api {
GLOBAL_DISABLE_PROTECTION = { path: 'disable_protection', method: 'POST' };
GLOBAL_UPDATE = { path: 'update', method: 'POST' };
restartGlobalFiltering() {
const { path, method } = this.GLOBAL_RESTART;
return this.makeRequest(path, method);
}
startGlobalFiltering() {
const { path, method } = this.GLOBAL_START;
return this.makeRequest(path, method);
@ -56,36 +45,11 @@ export default class Api {
return this.makeRequest(path, method);
}
getGlobalStats() {
const { path, method } = this.GLOBAL_STATS;
return this.makeRequest(path, method);
}
getGlobalStatsHistory() {
const { path, method } = this.GLOBAL_STATS_HISTORY;
const format = 'YYYY-MM-DDTHH:mm:ssZ';
const dateNow = Date.now();
const config = {
params: {
start_time: dateFormat(subHours(dateNow, 24), format),
end_time: dateFormat(dateNow, format),
time_unit: 'hours',
},
};
return this.makeRequest(path, method, config);
}
getGlobalStatus() {
const { path, method } = this.GLOBAL_STATUS;
return this.makeRequest(path, method);
}
getGlobalStatsTop() {
const { path, method } = this.GLOBAL_STATS_TOP;
return this.makeRequest(path, method);
}
getQueryLog() {
const { path, method } = this.GLOBAL_QUERY_LOG;
return this.makeRequest(path, method);
@ -527,4 +491,34 @@ export default class Api {
};
return this.makeRequest(path, method, parameters);
}
// Settings for statistics
GET_STATS = { path: 'stats', method: 'GET' };
STATS_INFO = { path: 'stats_info', method: 'GET' };
STATS_CONFIG = { path: 'stats_config', method: 'POST' };
STATS_RESET = { path: 'stats_reset', method: 'POST' };
getStats() {
const { path, method } = this.GET_STATS;
return this.makeRequest(path, method);
}
getStatsInfo() {
const { path, method } = this.STATS_INFO;
return this.makeRequest(path, method);
}
setStatsConfig(data) {
const { path, method } = this.STATS_CONFIG;
const config = {
data,
headers: { 'Content-Type': 'application/json' },
};
return this.makeRequest(path, method, config);
}
resetStats() {
const { path, method } = this.STATS_RESET;
return this.makeRequest(path, method);
}
}

89
client/src/components/Dashboard/BlockedDomains.js
View File

@ -1,79 +1,76 @@
import React, { Component } from 'react';
import React from 'react';
import ReactTable from 'react-table';
import PropTypes from 'prop-types';
import map from 'lodash/map';
import { withNamespaces, Trans } from 'react-i18next';
import Card from '../ui/Card';
import Cell from '../ui/Cell';
import Popover from '../ui/Popover';
import DomainCell from './DomainCell';
import { getTrackerData } from '../../helpers/trackers/trackers';
import { getPercent } from '../../helpers/helpers';
import { STATUS_COLORS } from '../../helpers/constants';
class BlockedDomains extends Component {
columns = [{
Header: 'IP',
accessor: 'ip',
Cell: (row) => {
const CountCell = totalBlocked =>
function cell(row) {
const { value } = row;
const trackerData = getTrackerData(value);
const percent = getPercent(totalBlocked, value);
return (
<div className="logs__row">
<div className="logs__text" title={value}>
{value}
</div>
{trackerData && <Popover data={trackerData} />}
</div>
);
},
}, {
Header: <Trans>requests_count</Trans>,
accessor: 'domain',
maxWidth: 190,
Cell: ({ value }) => {
const {
return <Cell value={value} percent={percent} color={STATUS_COLORS.red} />;
};
const BlockedDomains = ({
t,
refreshButton,
topBlockedDomains,
subtitle,
blockedFiltering,
replacedSafebrowsing,
replacedParental,
} = this.props;
const blocked = blockedFiltering + replacedSafebrowsing + replacedParental;
const percent = getPercent(blocked, value);
}) => {
const totalBlocked = blockedFiltering + replacedSafebrowsing + replacedParental;
return (
<Cell value={value} percent={percent} color={STATUS_COLORS.red} />
);
},
}];
render() {
const { t } = this.props;
return (
<Card title={ t('top_blocked_domains') } subtitle={ t('for_last_24_hours') } bodyType="card-table" refresh={this.props.refreshButton}>
<Card
title={t('top_blocked_domains')}
subtitle={subtitle}
bodyType="card-table"
refresh={refreshButton}
>
<ReactTable
data={map(this.props.topBlockedDomains, (value, prop) => (
{ ip: prop, domain: value }
))}
columns={this.columns}
data={topBlockedDomains.map(({ name: domain, count }) => ({
domain,
count,
}))}
columns={[
{
Header: <Trans>domain</Trans>,
accessor: 'domain',
Cell: DomainCell,
},
{
Header: <Trans>requests_count</Trans>,
accessor: 'count',
maxWidth: 190,
Cell: CountCell(totalBlocked),
},
]}
showPagination={false}
noDataText={ t('no_domains_found') }
noDataText={t('no_domains_found')}
minRows={6}
className="-striped -highlight card-table-overflow stats__table"
/>
</Card>
);
}
}
};
BlockedDomains.propTypes = {
topBlockedDomains: PropTypes.object.isRequired,
topBlockedDomains: PropTypes.array.isRequired,
blockedFiltering: PropTypes.number.isRequired,
replacedSafebrowsing: PropTypes.number.isRequired,
replacedParental: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
t: PropTypes.func,
subtitle: PropTypes.string.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(BlockedDomains);

90
client/src/components/Dashboard/Clients.js
View File

@ -1,7 +1,6 @@
import React, { Component } from 'react';
import React from 'react';
import ReactTable from 'react-table';
import PropTypes from 'prop-types';
import map from 'lodash/map';
import { Trans, withNamespaces } from 'react-i18next';
import Card from '../ui/Card';
@ -10,26 +9,27 @@ import Cell from '../ui/Cell';
import { getPercent, getClientName } from '../../helpers/helpers';
import { STATUS_COLORS } from '../../helpers/constants';
class Clients extends Component {
getPercentColor = (percent) => {
const getClientsPercentColor = (percent) => {
if (percent > 50) {
return STATUS_COLORS.green;
} else if (percent > 10) {
return STATUS_COLORS.yellow;
}
return STATUS_COLORS.red;
}
};
columns = [{
Header: 'IP',
accessor: 'ip',
Cell: ({ value }) => {
const clientName = getClientName(this.props.clients, value)
|| getClientName(this.props.autoClients, value);
const ipCell = (clients, autoClients) =>
function cell(row) {
let client;
const { value } = row;
const clientName = getClientName(clients, value) || getClientName(autoClients, value);
if (clientName) {
client = <span>{clientName} <small>({value})</small></span>;
client = (
<span>
{clientName} <small>({value})</small>
</span>
);
} else {
client = value;
}
@ -41,47 +41,61 @@ class Clients extends Component {
</span>
</div>
);
};
const countCell = dnsQueries =>
function cell(row) {
const { value } = row;
const percent = getPercent(dnsQueries, value);
const percentColor = getClientsPercentColor(percent);
return <Cell value={value} percent={percent} color={percentColor} />;
};
const Clients = ({
t, refreshButton, topClients, subtitle, clients, autoClients, dnsQueries,
}) => (
<Card
title={t('top_clients')}
subtitle={subtitle}
bodyType="card-table"
refresh={refreshButton}
>
<ReactTable
data={topClients.map(({ name: ip, count }) => ({
ip,
count,
}))}
columns={[
{
Header: 'IP',
accessor: 'ip',
sortMethod: (a, b) =>
parseInt(a.replace(/\./g, ''), 10) - parseInt(b.replace(/\./g, ''), 10),
Cell: ipCell(clients, autoClients),
},
sortMethod: (a, b) => parseInt(a.replace(/\./g, ''), 10) - parseInt(b.replace(/\./g, ''), 10),
}, {
{
Header: <Trans>requests_count</Trans>,
accessor: 'count',
Cell: ({ value }) => {
const percent = getPercent(this.props.dnsQueries, value);
const percentColor = this.getPercentColor(percent);
return (
<Cell value={value} percent={percent} color={percentColor} />
);
Cell: countCell(dnsQueries),
},
}];
render() {
const { t } = this.props;
return (
<Card title={ t('top_clients') } subtitle={ t('for_last_24_hours') } bodyType="card-table" refresh={this.props.refreshButton}>
<ReactTable
data={map(this.props.topClients, (value, prop) => (
{ ip: prop, count: value }
))}
columns={this.columns}
]}
showPagination={false}
noDataText={ t('no_clients_found') }
noDataText={t('no_clients_found')}
minRows={6}
className="-striped -highlight card-table-overflow"
/>
</Card>
);
}
}
);
Clients.propTypes = {
topClients: PropTypes.object.isRequired,
topClients: PropTypes.array.isRequired,
dnsQueries: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
clients: PropTypes.array.isRequired,
autoClients: PropTypes.array.isRequired,
t: PropTypes.func,
subtitle: PropTypes.string.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(Clients);

86
client/src/components/Dashboard/Counters.js
View File

@ -1,88 +1,116 @@
import React from 'react';
import PropTypes from 'prop-types';
import { Trans, withNamespaces } from 'react-i18next';
import round from 'lodash/round';
import Card from '../ui/Card';
import Tooltip from '../ui/Tooltip';
const tooltipType = 'tooltip-custom--narrow';
const Counters = props => (
<Card title={ props.t('general_statistics') } subtitle={ props.t('for_last_24_hours') } bodyType="card-table" refresh={props.refreshButton}>
const Counters = (props) => {
const {
t,
interval,
refreshButton,
subtitle,
dnsQueries,
blockedFiltering,
replacedSafebrowsing,
replacedParental,
replacedSafesearch,
avgProcessingTime,
} = props;
const tooltipTitle =
interval === 1
? t('number_of_dns_query_24_hours')
: t('number_of_dns_query_days', { count: interval });
return (
<Card
title={t('general_statistics')}
subtitle={subtitle}
bodyType="card-table"
refresh={refreshButton}
>
<table className="table card-table">
<tbody>
<tr>
<td>
<Trans>dns_query</Trans>
<Tooltip text={ props.t('number_of_dns_query_24_hours') } type={tooltipType} />
<Tooltip text={tooltipTitle} type={tooltipType} />
</td>
<td className="text-right">
<span className="text-muted">
{props.dnsQueries}
</span>
<span className="text-muted">{dnsQueries}</span>
</td>
</tr>
<tr>
<td>
<a href="#filters">
<Trans>blocked_by</Trans>
</a>
<Tooltip text={ props.t('number_of_dns_query_blocked_24_hours') } type={tooltipType} />
<Trans components={[<a href="#filters" key="0">link</a>]}>
blocked_by
</Trans>
<Tooltip
text={t('number_of_dns_query_blocked_24_hours')}
type={tooltipType}
/>
</td>
<td className="text-right">
<span className="text-muted">
{props.blockedFiltering}
</span>
<span className="text-muted">{blockedFiltering}</span>
</td>
</tr>
<tr>
<td>
<Trans>stats_malware_phishing</Trans>
<Tooltip text={ props.t('number_of_dns_query_blocked_24_hours_by_sec') } type={tooltipType} />
<Tooltip
text={t('number_of_dns_query_blocked_24_hours_by_sec')}
type={tooltipType}
/>
</td>
<td className="text-right">
<span className="text-muted">
{props.replacedSafebrowsing}
</span>
<span className="text-muted">{replacedSafebrowsing}</span>
</td>
</tr>
<tr>
<td>
<Trans>stats_adult</Trans>
<Tooltip text={ props.t('number_of_dns_query_blocked_24_hours_adult') } type={tooltipType} />
<Tooltip
text={t('number_of_dns_query_blocked_24_hours_adult')}
type={tooltipType}
/>
</td>
<td className="text-right">
<span className="text-muted">
{props.replacedParental}
</span>
<span className="text-muted">{replacedParental}</span>
</td>
</tr>
<tr>
<td>
<Trans>enforced_save_search</Trans>
<Tooltip text={ props.t('number_of_dns_query_to_safe_search') } type={tooltipType} />
<Tooltip
text={t('number_of_dns_query_to_safe_search')}
type={tooltipType}
/>
</td>
<td className="text-right">
<span className="text-muted">
{props.replacedSafesearch}
</span>
<span className="text-muted">{replacedSafesearch}</span>
</td>
</tr>
<tr>
<td>
<Trans>average_processing_time</Trans>
<Tooltip text={ props.t('average_processing_time_hint') } type={tooltipType} />
<Tooltip text={t('average_processing_time_hint')} type={tooltipType} />
</td>
<td className="text-right">
<span className="text-muted">
{props.avgProcessingTime}
{avgProcessingTime ? `${round(avgProcessingTime)} ms` : 0}
</span>
</td>
</tr>
</tbody>
</table>
</Card>
);
);
};
Counters.propTypes = {
dnsQueries: PropTypes.number.isRequired,
@ -92,6 +120,8 @@ Counters.propTypes = {
replacedSafesearch: PropTypes.number.isRequired,
avgProcessingTime: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
subtitle: PropTypes.string.isRequired,
interval: PropTypes.number.isRequired,
t: PropTypes.func.isRequired,
};

5
client/src/components/Dashboard/Dashboard.css
View File

@ -20,3 +20,8 @@
border-top: 6px solid transparent;
border-bottom: 6px solid #585965;
}
.card-chart-bg {
left: -20px;
width: calc(100% + 20px);
}

24
client/src/components/Dashboard/DomainCell.js Normal file
View File

@ -0,0 +1,24 @@
import React from 'react';
import PropTypes from 'prop-types';
import { getTrackerData } from '../../helpers/trackers/trackers';
import Popover from '../ui/Popover';
const DomainCell = ({ value }) => {
const trackerData = getTrackerData(value);
return (
<div className="logs__row">
<div className="logs__text" title={value}>
{value}
</div>
{trackerData && <Popover data={trackerData} />}
</div>
);
};
DomainCell.propTypes = {
value: PropTypes.string.isRequired,
};
export default DomainCell;

85
client/src/components/Dashboard/QueriedDomains.js
View File

@ -1,81 +1,74 @@
import React, { Component } from 'react';
import React from 'react';
import ReactTable from 'react-table';
import PropTypes from 'prop-types';
import map from 'lodash/map';
import { withNamespaces, Trans } from 'react-i18next';
import Card from '../ui/Card';
import Cell from '../ui/Cell';
import Popover from '../ui/Popover';
import DomainCell from './DomainCell';
import { getTrackerData } from '../../helpers/trackers/trackers';
import { getPercent } from '../../helpers/helpers';
import { STATUS_COLORS } from '../../helpers/constants';
import { getPercent } from '../../helpers/helpers';
class QueriedDomains extends Component {
getPercentColor = (percent) => {
const getQueriedPercentColor = (percent) => {
if (percent > 10) {
return STATUS_COLORS.red;
} else if (percent > 5) {
return STATUS_COLORS.yellow;
}
return STATUS_COLORS.green;
}
};
columns = [{
Header: 'IP',
accessor: 'ip',
Cell: (row) => {
const countCell = dnsQueries =>
function cell(row) {
const { value } = row;
const trackerData = getTrackerData(value);
const percent = getPercent(dnsQueries, value);
const percentColor = getQueriedPercentColor(percent);
return (
<div className="logs__row">
<div className="logs__text" title={value}>
{value}
</div>
{trackerData && <Popover data={trackerData} />}
</div>
);
return <Cell value={value} percent={percent} color={percentColor} />;
};
const QueriedDomains = ({
t, refreshButton, topQueriedDomains, subtitle, dnsQueries,
}) => (
<Card
title={t('stats_query_domain')}
subtitle={subtitle}
bodyType="card-table"
refresh={refreshButton}
>
<ReactTable
data={topQueriedDomains.map(({ name: domain, count }) => ({
domain,
count,
}))}
columns={[
{
Header: <Trans>domain</Trans>,
accessor: 'domain',
Cell: DomainCell,
},
}, {
{
Header: <Trans>requests_count</Trans>,
accessor: 'count',
maxWidth: 190,
Cell: ({ value }) => {
const percent = getPercent(this.props.dnsQueries, value);
const percentColor = this.getPercentColor(percent);
return (
<Cell value={value} percent={percent} color={percentColor} />
);
Cell: countCell(dnsQueries),
},
}];
render() {
const { t } = this.props;
return (
<Card title={ t('stats_query_domain') } subtitle={ t('for_last_24_hours') } bodyType="card-table" refresh={this.props.refreshButton}>
<ReactTable
data={map(this.props.topQueriedDomains, (value, prop) => (
{ ip: prop, count: value }
))}
columns={this.columns}
]}
showPagination={false}
noDataText={ t('no_domains_found') }
noDataText={t('no_domains_found')}
minRows={6}
className="-striped -highlight card-table-overflow stats__table"
/>
</Card>
);
}
}
);
QueriedDomains.propTypes = {
topQueriedDomains: PropTypes.object.isRequired,
topQueriedDomains: PropTypes.array.isRequired,
dnsQueries: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
t: PropTypes.func,
subtitle: PropTypes.string.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(QueriedDomains);

143
client/src/components/Dashboard/Statistics.js
View File

@ -1,111 +1,78 @@
import React, { Component } from 'react';
import React from 'react';
import PropTypes from 'prop-types';
import { Trans, withNamespaces } from 'react-i18next';
import { withNamespaces, Trans } from 'react-i18next';
import Card from '../ui/Card';
import Line from '../ui/Line';
import StatsCard from './StatsCard';
import { getPercent, normalizeHistory } from '../../helpers/helpers';
import { getPercent } from '../../helpers/helpers';
import { STATUS_COLORS } from '../../helpers/constants';
const getNormalizedHistory = (data, interval, id) => [
{ data: normalizeHistory(data, interval), id },
];
class Statistics extends Component {
render() {
const {
const Statistics = ({
interval,
dnsQueries,
blockedFiltering,
replacedSafebrowsing,
replacedParental,
} = this.props;
const filteringData = [this.props.history[1]];
const queriesData = [this.props.history[2]];
const parentalData = [this.props.history[3]];
const safebrowsingData = [this.props.history[4]];
return (
numDnsQueries,
numBlockedFiltering,
numReplacedSafebrowsing,
numReplacedParental,
}) => (
<div className="row">
<div className="col-sm-6 col-lg-3">
<Card type="card--full" bodyType="card-wrap">
<div className="card-body-stats">
<div className="card-value card-value-stats text-blue">
{dnsQueries}
</div>
<div className="card-title-stats">
<Trans>dns_query</Trans>
</div>
</div>
<div className="card-chart-bg">
<Line data={queriesData} color={STATUS_COLORS.blue}/>
</div>
</Card>
<StatsCard
total={numDnsQueries}
lineData={getNormalizedHistory(dnsQueries, interval, 'dnsQuery')}
title={<Trans>dns_query</Trans>}
color="blue"
/>
</div>
<div className="col-sm-6 col-lg-3">
<Card type="card--full" bodyType="card-wrap">
<div className="card-body-stats">
<div className="card-value card-value-stats text-red">
{blockedFiltering}
</div>
<div className="card-value card-value-percent text-red">
{getPercent(dnsQueries, blockedFiltering)}
</div>
<div className="card-title-stats">
<a href="#filters">
<Trans>blocked_by</Trans>
</a>
</div>
</div>
<div className="card-chart-bg">
<Line data={filteringData} color={STATUS_COLORS.red}/>
</div>
</Card>
<StatsCard
total={numBlockedFiltering}
lineData={getNormalizedHistory(blockedFiltering, interval, 'blockedFiltering')}
percent={getPercent(numDnsQueries, numBlockedFiltering)}
title={<Trans components={[<a href="#filters" key="0">link</a>]}>blocked_by</Trans>}
color="red"
/>
</div>
<div className="col-sm-6 col-lg-3">
<Card type="card--full" bodyType="card-wrap">
<div className="card-body-stats">
<div className="card-value card-value-stats text-green">
{replacedSafebrowsing}
</div>
<div className="card-value card-value-percent text-green">
{getPercent(dnsQueries, replacedSafebrowsing)}
</div>
<div className="card-title-stats">
<Trans>stats_malware_phishing</Trans>
</div>
</div>
<div className="card-chart-bg">
<Line data={safebrowsingData} color={STATUS_COLORS.green}/>
</div>
</Card>
<StatsCard
total={numReplacedSafebrowsing}
lineData={getNormalizedHistory(
replacedSafebrowsing,
interval,
'replacedSafebrowsing',
)}
percent={getPercent(numDnsQueries, numReplacedSafebrowsing)}
title={<Trans>stats_malware_phishing</Trans>}
color="green"
/>
</div>
<div className="col-sm-6 col-lg-3">
<Card type="card--full" bodyType="card-wrap">
<div className="card-body-stats">
<div className="card-value card-value-stats text-yellow">
{replacedParental}
</div>
<div className="card-value card-value-percent text-yellow">
{getPercent(dnsQueries, replacedParental)}
</div>
<div className="card-title-stats">
<Trans>stats_adult</Trans>
<StatsCard
total={numReplacedParental}
lineData={getNormalizedHistory(replacedParental, interval, 'replacedParental')}
percent={getPercent(numDnsQueries, numReplacedParental)}
title={<Trans>stats_adult</Trans>}
color="yellow"
/>
</div>
</div>
<div className="card-chart-bg">
<Line data={parentalData} color={STATUS_COLORS.yellow}/>
</div>
</Card>
</div>
</div>
);
}
}
);
Statistics.propTypes = {
history: PropTypes.array.isRequired,
dnsQueries: PropTypes.number.isRequired,
blockedFiltering: PropTypes.number.isRequired,
replacedSafebrowsing: PropTypes.number.isRequired,
replacedParental: PropTypes.number.isRequired,
interval: PropTypes.number.isRequired,
dnsQueries: PropTypes.array.isRequired,
blockedFiltering: PropTypes.array.isRequired,
replacedSafebrowsing: PropTypes.array.isRequired,
replacedParental: PropTypes.array.isRequired,
numDnsQueries: PropTypes.number.isRequired,
numBlockedFiltering: PropTypes.number.isRequired,
numReplacedSafebrowsing: PropTypes.number.isRequired,
numReplacedParental: PropTypes.number.isRequired,
refreshButton: PropTypes.node.isRequired,
};

31
client/src/components/Dashboard/StatsCard.js Normal file
View File

@ -0,0 +1,31 @@
import React from 'react';
import PropTypes from 'prop-types';
import { STATUS_COLORS } from '../../helpers/constants';
import Card from '../ui/Card';
import Line from '../ui/Line';
const StatsCard = ({
total, lineData, percent, title, color,
}) => (
<Card type="card--full" bodyType="card-wrap">
<div className="card-body-stats">
<div className={`card-value card-value-stats text-${color}`}>{total}</div>
<div className="card-title-stats">{title}</div>
</div>
{percent >= 0 && (<div className={`card-value card-value-percent text-${color}`}>{percent}</div>)}
<div className="card-chart-bg">
<Line data={lineData} color={STATUS_COLORS[color]} />
</div>
</Card>
);
StatsCard.propTypes = {
total: PropTypes.number.isRequired,
lineData: PropTypes.array.isRequired,
title: PropTypes.object.isRequired,
color: PropTypes.string.isRequired,
percent: PropTypes.number,
};
export default StatsCard;

100
client/src/components/Dashboard/index.js
View File

@ -19,10 +19,9 @@ class Dashboard extends Component {
getAllStats = () => {
this.props.getStats();
this.props.getStatsHistory();
this.props.getTopStats();
this.props.getStatsConfig();
this.props.getClients();
}
};
getToggleFilteringButton = () => {
const { protectionEnabled, processingProtection } = this.props.dashboard;
@ -39,16 +38,20 @@ class Dashboard extends Component {
<Trans>{buttonText}</Trans>
</button>
);
}
};
render() {
const { dashboard, t } = this.props;
const { dashboard, stats, t } = this.props;
const dashboardProcessing =
dashboard.processing ||
dashboard.processingStats ||
dashboard.processingStatsHistory ||
dashboard.processingClients ||
dashboard.processingTopStats;
stats.processingStats ||
stats.processingGetConfig;
const subtitle =
stats.interval === 1
? t('for_last_24_hours')
: t('for_last_days', { count: stats.interval });
const refreshFullButton = (
<button
@ -59,6 +62,7 @@ class Dashboard extends Component {
<Trans>refresh_statics</Trans>
</button>
);
const refreshButton = (
<button
type="button"
@ -73,87 +77,85 @@ class Dashboard extends Component {
return (
<Fragment>
<PageTitle title={ t('dashboard') }>
<PageTitle title={t('dashboard')}>
<div className="page-title__actions">
{this.getToggleFilteringButton()}
{refreshFullButton}
</div>
</PageTitle>
{dashboardProcessing && <Loading />}
{!dashboardProcessing &&
{!dashboardProcessing && (
<div className="row row-cards">
{dashboard.statsHistory &&
<div className="col-lg-12">
<Statistics
history={dashboard.statsHistory}
interval={stats.interval}
dnsQueries={stats.dnsQueries}
blockedFiltering={stats.blockedFiltering}
replacedSafebrowsing={stats.replacedSafebrowsing}
replacedParental={stats.replacedParental}
numDnsQueries={stats.numDnsQueries}
numBlockedFiltering={stats.numBlockedFiltering}
numReplacedSafebrowsing={stats.numReplacedSafebrowsing}
numReplacedParental={stats.numReplacedParental}
refreshButton={refreshButton}
dnsQueries={dashboard.stats.dns_queries}
blockedFiltering={dashboard.stats.blocked_filtering}
replacedSafebrowsing={dashboard.stats.replaced_safebrowsing}
replacedParental={dashboard.stats.replaced_parental}
/>
</div>
}
<div className="col-lg-6">
{dashboard.stats &&
<Counters
subtitle={subtitle}
interval={stats.interval}
dnsQueries={stats.numDnsQueries}
blockedFiltering={stats.numBlockedFiltering}
replacedSafebrowsing={stats.numReplacedSafebrowsing}
replacedParental={stats.numReplacedParental}
replacedSafesearch={stats.numReplacedSafesearch}
avgProcessingTime={stats.avgProcessingTime}
refreshButton={refreshButton}
dnsQueries={dashboard.stats.dns_queries}
blockedFiltering={dashboard.stats.blocked_filtering}
replacedSafebrowsing={dashboard.stats.replaced_safebrowsing}
replacedParental={dashboard.stats.replaced_parental}
replacedSafesearch={dashboard.stats.replaced_safesearch}
avgProcessingTime={dashboard.stats.avg_processing_time}
/>
}
</div>
{dashboard.topStats &&
<Fragment>
<div className="col-lg-6">
<Clients
dnsQueries={dashboard.stats.dns_queries}
refreshButton={refreshButton}
topClients={dashboard.topStats.top_clients}
subtitle={subtitle}
dnsQueries={stats.numDnsQueries}
topClients={stats.topClients}
clients={dashboard.clients}
autoClients={dashboard.autoClients}
refreshButton={refreshButton}
/>
</div>
<div className="col-lg-6">
<QueriedDomains
dnsQueries={dashboard.stats.dns_queries}
subtitle={subtitle}
dnsQueries={stats.numDnsQueries}
topQueriedDomains={stats.topQueriedDomains}
refreshButton={refreshButton}
topQueriedDomains={dashboard.topStats.top_queried_domains}
/>
</div>
<div className="col-lg-6">
<BlockedDomains
subtitle={subtitle}
topBlockedDomains={stats.topBlockedDomains}
blockedFiltering={stats.numBlockedFiltering}
replacedSafebrowsing={stats.numReplacedSafebrowsing}
replacedParental={stats.numReplacedParental}
refreshButton={refreshButton}
topBlockedDomains={dashboard.topStats.top_blocked_domains}
blockedFiltering={dashboard.stats.blocked_filtering}
replacedSafebrowsing={dashboard.stats.replaced_safebrowsing}
replacedParental={dashboard.stats.replaced_parental}
/>
</div>
</Fragment>
}
</div>
}
)}
</Fragment>
);
}
}
Dashboard.propTypes = {
getStats: PropTypes.func,
getStatsHistory: PropTypes.func,
getTopStats: PropTypes.func,
dashboard: PropTypes.object,
isCoreRunning: PropTypes.bool,
getFiltering: PropTypes.func,
toggleProtection: PropTypes.func,
getClients: PropTypes.func,
processingProtection: PropTypes.bool,
t: PropTypes.func,
dashboard: PropTypes.object.isRequired,
stats: PropTypes.object.isRequired,
getStats: PropTypes.func.isRequired,
getStatsConfig: PropTypes.func.isRequired,
toggleProtection: PropTypes.func.isRequired,
getClients: PropTypes.func.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(Dashboard);

2
client/src/components/Filters/Modal.js
View File

@ -80,7 +80,7 @@ class Modal extends Component {
}
return (
<div className="description">
<Trans>url_added_successfully</Trans>
<Trans>filter_added_successfully</Trans>
</div>
);
};

31
client/src/components/Settings/Clients/AutoClients.js
View File

@ -3,32 +3,13 @@ import PropTypes from 'prop-types';
import { withNamespaces } from 'react-i18next';
import ReactTable from 'react-table';
import { CLIENT_ID } from '../../../helpers/constants';
import Card from '../../ui/Card';
class AutoClients extends Component {
getClient = (name, clients) => {
const client = clients.find(item => name === item.name);
if (client) {
const identifier = client.mac ? CLIENT_ID.MAC : CLIENT_ID.IP;
return {
identifier,
use_global_settings: true,
...client,
};
}
return {
identifier: 'ip',
use_global_settings: true,
};
};
getStats = (ip, stats) => {
if (stats && stats.top_clients) {
return stats.top_clients[ip];
if (stats) {
const statsForCurrentIP = stats.find(item => item.name === ip);
return statsForCurrentIP && statsForCurrentIP.count;
}
return '';
@ -59,11 +40,11 @@ class AutoClients extends Component {
Cell: this.cellWrap,
},
{
Header: this.props.t('table_statistics'),
Header: this.props.t('requests_count'),
accessor: 'statistics',
Cell: (row) => {
const clientIP = row.original.ip;
const clientStats = clientIP && this.getStats(clientIP, this.props.topStats);
const clientStats = clientIP && this.getStats(clientIP, this.props.topClients);
if (clientStats) {
return (
@ -112,7 +93,7 @@ class AutoClients extends Component {
AutoClients.propTypes = {
t: PropTypes.func.isRequired,
autoClients: PropTypes.array.isRequired,
topStats: PropTypes.object.isRequired,
topClients: PropTypes.array.isRequired,
};
export default withNamespaces()(AutoClients);

11
client/src/components/Settings/Clients/ClientsTable.js
View File

@ -63,8 +63,9 @@ class ClientsTable extends Component {
};
getStats = (ip, stats) => {
if (stats && stats.top_clients) {
return stats.top_clients[ip];
if (stats) {
const statsForCurrentIP = stats.find(item => item.name === ip);
return statsForCurrentIP && statsForCurrentIP.count;
}
return '';
@ -149,11 +150,11 @@ class ClientsTable extends Component {
},
},
{
Header: this.props.t('table_statistics'),
Header: this.props.t('requests_count'),
accessor: 'statistics',
Cell: (row) => {
const clientIP = row.original.ip;
const clientStats = clientIP && this.getStats(clientIP, this.props.topStats);
const clientStats = clientIP && this.getStats(clientIP, this.props.topClients);
if (clientStats) {
return (
@ -276,7 +277,7 @@ class ClientsTable extends Component {
ClientsTable.propTypes = {
t: PropTypes.func.isRequired,
clients: PropTypes.array.isRequired,
topStats: PropTypes.object.isRequired,
topClients: PropTypes.array.isRequired,
toggleClientModal: PropTypes.func.isRequired,
deleteClient: PropTypes.func.isRequired,
addClient: PropTypes.func.isRequired,

15
client/src/components/Settings/Clients/index.js
View File

@ -10,13 +10,14 @@ import Loading from '../../ui/Loading';
class Clients extends Component {
componentDidMount() {
this.props.getClients();
this.props.getTopStats();
this.props.getStats();
}
render() {
const {
t,
dashboard,
stats,
clients,
addClient,
updateClient,
@ -27,12 +28,12 @@ class Clients extends Component {
return (
<Fragment>
<PageTitle title={t('client_settings')} />
{(dashboard.processingTopStats || dashboard.processingClients) && <Loading />}
{!dashboard.processingTopStats && !dashboard.processingClients && (
{(stats.processingStats || dashboard.processingClients) && <Loading />}
{!stats.processingStats && !dashboard.processingClients && (
<Fragment>
<ClientsTable
clients={dashboard.clients}
topStats={dashboard.topStats}
topClients={stats.topClients}
isModalOpen={clients.isModalOpen}
modalClientName={clients.modalClientName}
modalType={clients.modalType}
@ -46,7 +47,7 @@ class Clients extends Component {
/>
<AutoClients
autoClients={dashboard.autoClients}
topStats={dashboard.topStats}
topClients={stats.topClients}
/>
</Fragment>
)}
@ -58,14 +59,14 @@ class Clients extends Component {
Clients.propTypes = {
t: PropTypes.func.isRequired,
dashboard: PropTypes.object.isRequired,
stats: PropTypes.object.isRequired,
clients: PropTypes.object.isRequired,
toggleClientModal: PropTypes.func.isRequired,
deleteClient: PropTypes.func.isRequired,
addClient: PropTypes.func.isRequired,
updateClient: PropTypes.func.isRequired,
getClients: PropTypes.func.isRequired,
getTopStats: PropTypes.func.isRequired,
topStats: PropTypes.object,
getStats: PropTypes.func.isRequired,
};
export default withNamespaces()(Clients);

2
client/src/components/Settings/Dns/Rewrites/Modal.js
View File

@ -24,7 +24,7 @@ const Modal = (props) => {
<div className="modal-content">
<div className="modal-header">
<h4 className="modal-title">
<Trans>Add DNS rewrite</Trans>
<Trans>rewrite_add</Trans>
</h4>
<button type="button" className="close" onClick={() => toggleRewritesModal()}>
<span className="sr-only">Close</span>

4
client/src/components/Settings/Dns/Rewrites/Table.js
View File

@ -14,12 +14,12 @@ class Table extends Component {
columns = [
{
Header: 'Domain',
Header: this.props.t('domain'),
accessor: 'domain',
Cell: this.cellWrap,
},
{
Header: 'Answer',
Header: this.props.t('answer'),
accessor: 'answer',
Cell: this.cellWrap,
},

2
client/src/components/Settings/Services/index.js
View File

@ -44,7 +44,7 @@ class Services extends Component {
return (
<Card
title={t('blocked_services')}
subtitle={t('Allows to quickly block popular sites.')}
subtitle={t('blocked_services_desc')}
bodyType="card-body box-body--settings"
>
<div className="form">

5
client/src/components/Settings/Settings.css
View File

@ -104,3 +104,8 @@
min-width: 23px;
padding: 5px;
}
.custom-control-label,
.custom-control-label:before {
transition: 0.3s ease-in-out background-color, 0.3s ease-in-out color;
}

74
client/src/components/Settings/StatsConfig/Form.js Normal file
View File

@ -0,0 +1,74 @@
import React from 'react';
import PropTypes from 'prop-types';
import { Field, reduxForm } from 'redux-form';
import { Trans, withNamespaces } from 'react-i18next';
import flow from 'lodash/flow';
import { renderRadioField, toNumber } from '../../../helpers/form';
import { STATS_INTERVALS_DAYS } from '../../../helpers/constants';
const getIntervalFields = (processing, t, handleChange, toNumber) =>
STATS_INTERVALS_DAYS.map((interval) => {
const title = interval === 1
? t('interval_24_hour')
: t('interval_days', { count: interval });
return (
<Field
key={interval}
name="interval"
type="radio"
component={renderRadioField}
value={interval}
placeholder={title}
onChange={handleChange}
normalize={toNumber}
disabled={processing}
/>
);
});
const Form = (props) => {
const {
handleSubmit, handleChange, processing, t,
} = props;
return (
<form onSubmit={handleSubmit}>
<div className="row">
<div className="col-12">
<label className="form__label form__label--with-desc" htmlFor="server_name">
<Trans>statistics_retention</Trans>
</label>
<div className="form__desc form__desc--top">
<Trans>statistics_retention_desc</Trans>
</div>
</div>
<div className="col-12">
<div className="form__group mt-2">
<div className="custom-controls-stacked">
{getIntervalFields(processing, t, handleChange, toNumber)}
</div>
</div>
</div>
</div>
</form>
);
};
Form.propTypes = {
handleSubmit: PropTypes.func.isRequired,
handleChange: PropTypes.func,
change: PropTypes.func.isRequired,
submitting: PropTypes.bool.isRequired,
invalid: PropTypes.bool.isRequired,
processing: PropTypes.bool.isRequired,
t: PropTypes.func.isRequired,
};
export default flow([
withNamespaces(),
reduxForm({
form: 'logConfigForm',
}),
])(Form);

63
client/src/components/Settings/StatsConfig/index.js Normal file
View File

@ -0,0 +1,63 @@
import React, { Component } from 'react';
import PropTypes from 'prop-types';
import { withNamespaces, Trans } from 'react-i18next';
import debounce from 'lodash/debounce';
import { DEBOUNCE_TIMEOUT } from '../../../helpers/constants';
import Form from './Form';
import Card from '../../ui/Card';
class StatsConfig extends Component {
handleFormChange = debounce((values) => {
this.props.setStatsConfig(values);
}, DEBOUNCE_TIMEOUT);
handleReset = () => {
const { t, resetStats } = this.props;
// eslint-disable-next-line no-alert
if (window.confirm(t('statistics_clear_confirm'))) {
resetStats();
}
};
render() {
const {
t, interval, processing, processingReset,
} = this.props;
return (
<Card title={t('statistics_logs')} bodyType="card-body box-body--settings">
<div className="form">
<Form
initialValues={{
interval,
}}
onSubmit={this.handleFormChange}
onChange={this.handleFormChange}
processing={processing}
/>
<button
type="button"
className="btn btn-outline-secondary btn-sm mt-3"
onClick={this.handleReset}
disabled={processingReset}
>
<Trans>statistics_clear</Trans>
</button>
</div>
</Card>
);
}
}
StatsConfig.propTypes = {
interval: PropTypes.number.isRequired,
processing: PropTypes.bool.isRequired,
processingReset: PropTypes.bool.isRequired,
setStatsConfig: PropTypes.func.isRequired,
resetStats: PropTypes.func.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(StatsConfig);

31
client/src/components/Settings/index.js
View File

@ -3,6 +3,7 @@ import PropTypes from 'prop-types';
import { withNamespaces, Trans } from 'react-i18next';
import Services from './Services';
import StatsConfig from './StatsConfig';
import Checkbox from '../ui/Checkbox';
import Loading from '../ui/Loading';
import PageTitle from '../ui/PageTitle';
@ -37,6 +38,7 @@ class Settings extends Component {
componentDidMount() {
this.props.initSettings(this.settings);
this.props.getBlockedServices();
this.props.getStatsConfig();
}
renderSettings = (settings) => {
@ -62,7 +64,13 @@ class Settings extends Component {
render() {
const {
settings, services, setBlockedServices, t,
settings,
services,
setBlockedServices,
setStatsConfig,
resetStats,
stats,
t,
} = this.props;
return (
<Fragment>
@ -78,6 +86,15 @@ class Settings extends Component {
</div>
</Card>
</div>
<div className="col-md-12">
<StatsConfig
interval={stats.interval}
processing={stats.processingSetConfig}
processingReset={stats.processingReset}
setStatsConfig={setStatsConfig}
resetStats={resetStats}
/>
</div>
<div className="col-md-12">
<Services
services={services}
@ -93,11 +110,13 @@ class Settings extends Component {
}
Settings.propTypes = {
initSettings: PropTypes.func,
settings: PropTypes.object,
settingsList: PropTypes.object,
toggleSetting: PropTypes.func,
t: PropTypes.func,
initSettings: PropTypes.func.isRequired,
settings: PropTypes.object.isRequired,
toggleSetting: PropTypes.func.isRequired,
getStatsConfig: PropTypes.func.isRequired,
setStatsConfig: PropTypes.func.isRequired,
resetStats: PropTypes.func.isRequired,
t: PropTypes.func.isRequired,
};
export default withNamespaces()(Settings);

4
client/src/components/ui/Cell.js
View File

@ -5,9 +5,7 @@ const Cell = props => (
<div className="stats__row">
<div className="stats__row-value mb-1">
<strong>{props.value}</strong>
<small className="ml-3 text-muted">
{props.percent}%
</small>
<small className="ml-3 text-muted">{props.percent}%</small>
</div>
<div className="progress progress-xs">
<div

20
client/src/components/ui/Line.js
View File

@ -4,33 +4,27 @@ import { ResponsiveLine } from '@nivo/line';
import './Line.css';
const Line = props => (
props.data &&
const Line = ({ data, color }) => (
data &&
<ResponsiveLine
data={props.data}
data={data}
margin={{
top: 15,
right: 0,
bottom: 1,
left: 0,
left: 20,
}}
minY="auto"
stacked={false}
curve='linear'
axisBottom={{
tickSize: 0,
tickPadding: 10,
}}
axisLeft={{
tickSize: 0,
tickPadding: 10,
}}
axisBottom={null}
axisLeft={null}
enableGridX={false}
enableGridY={false}
enableDots={false}
enableArea={true}
animate={false}
colorBy={() => (props.color)}
colorBy={() => (color)}
tooltip={slice => (
<div>
{slice.data.map(d => (

8
client/src/containers/Clients.js
View File

@ -1,20 +1,22 @@
import { connect } from 'react-redux';
import { getClients, getTopStats } from '../actions';
import { getClients } from '../actions';
import { getStats } from '../actions/stats';
import { addClient, updateClient, deleteClient, toggleClientModal } from '../actions/clients';
import Clients from '../components/Settings/Clients';
const mapStateToProps = (state) => {
const { dashboard, clients } = state;
const { dashboard, clients, stats } = state;
const props = {
dashboard,
clients,
stats,
};
return props;
};
const mapDispatchToProps = {
getClients,
getTopStats,
getStats,
addClient,
updateClient,
deleteClient,

17
client/src/containers/Dashboard.js
View File

@ -1,14 +1,23 @@
import { connect } from 'react-redux';
import * as actionCreators from '../actions';
import { toggleProtection, getClients } from '../actions';
import { getStats, getStatsConfig, setStatsConfig } from '../actions/stats';
import Dashboard from '../components/Dashboard';
const mapStateToProps = (state) => {
const { dashboard } = state;
const props = { dashboard };
const { dashboard, stats } = state;
const props = { dashboard, stats };
return props;
};
const mapDispatchToProps = {
toggleProtection,
getClients,
getStats,
getStatsConfig,
setStatsConfig,
};
export default connect(
mapStateToProps,
actionCreators,
mapDispatchToProps,
)(Dashboard);

7
client/src/containers/Settings.js
View File

@ -1,13 +1,15 @@
import { connect } from 'react-redux';
import { initSettings, toggleSetting } from '../actions';
import { getBlockedServices, setBlockedServices } from '../actions/services';
import { getStatsConfig, setStatsConfig, resetStats } from '../actions/stats';
import Settings from '../components/Settings';
const mapStateToProps = (state) => {
const { settings, services } = state;
const { settings, services, stats } = state;
const props = {
settings,
services,
stats,
};
return props;
};
@ -17,6 +19,9 @@ const mapDispatchToProps = {
toggleSetting,
getBlockedServices,
setBlockedServices,
getStatsConfig,
setStatsConfig,
resetStats,
};
export default connect(

2
client/src/helpers/constants.js
View File

@ -260,3 +260,5 @@ export const FILTERED_STATUS = {
FILTERED_BLOCKED_SERVICE: 'FilteredBlockedService',
REWRITE: 'Rewrite',
};
export const STATS_INTERVALS_DAYS = [1, 7, 30, 90];

50
client/src/helpers/helpers.js
View File

@ -2,11 +2,12 @@ import dateParse from 'date-fns/parse';
import dateFormat from 'date-fns/format';
import subHours from 'date-fns/sub_hours';
import addHours from 'date-fns/add_hours';
import addDays from 'date-fns/add_days';
import subDays from 'date-fns/sub_days';
import round from 'lodash/round';
import axios from 'axios';
import {
STATS_NAMES,
STANDARD_DNS_PORT,
STANDARD_WEB_PORT,
STANDARD_HTTPS_PORT,
@ -49,29 +50,28 @@ export const normalizeLogs = logs => logs.map((log) => {
};
});
export const normalizeHistory = history => Object.keys(history).map((key) => {
let id = STATS_NAMES[key];
if (!id) {
id = key.replace(/_/g, ' ').replace(/^\w/, c => c.toUpperCase());
export const normalizeHistory = (history, interval) => {
if (interval === 1 || interval === 7) {
const hoursAgo = subHours(Date.now(), 24 * interval);
return history.map((item, index) => ({
x: dateFormat(addHours(hoursAgo, index), 'D MMM HH:00'),
y: round(item, 2),
}));
}
const dayAgo = subHours(Date.now(), 24);
const daysAgo = subDays(Date.now(), interval - 1);
return history.map((item, index) => ({
x: dateFormat(addDays(daysAgo, index), 'D MMM YYYY'),
y: round(item, 2),
}));
};
const data = history[key].map((item, index) => {
const formatHour = dateFormat(addHours(dayAgo, index), 'ddd HH:00');
const roundValue = round(item, 2);
return {
x: formatHour,
y: roundValue,
};
});
return {
id,
data,
};
});
export const normalizeTopStats = stats => (
stats.map(item => ({
name: Object.keys(item)[0],
count: Object.values(item)[0],
}))
);
export const normalizeFilteringStatus = (filteringStatus) => {
const { enabled, filters, user_rules: userRules } = filteringStatus;
@ -233,3 +233,11 @@ export const sortClients = (clients) => {
export const toggleAllServices = (services, change, isSelected) => {
services.forEach(service => change(`blocked_services.${service.id}`, isSelected));
};
export const secondsToMilliseconds = (seconds) => {
if (seconds) {
return seconds * 1000;
}
return seconds;
};

32
client/src/reducers/index.js
View File

@ -11,6 +11,7 @@ import clients from './clients';
import access from './access';
import rewrites from './rewrites';
import services from './services';
import stats from './stats';
const settings = handleActions({
[actions.initSettingsRequest]: state => ({ ...state, processing: true }),
@ -93,27 +94,6 @@ const dashboard = handleActions({
return newState;
},
[actions.getStatsRequest]: state => ({ ...state, processingStats: true }),
[actions.getStatsFailure]: state => ({ ...state, processingStats: false }),
[actions.getStatsSuccess]: (state, { payload }) => {
const newState = { ...state, stats: payload, processingStats: false };
return newState;
},
[actions.getTopStatsRequest]: state => ({ ...state, processingTopStats: true }),
[actions.getTopStatsFailure]: state => ({ ...state, processingTopStats: false }),
[actions.getTopStatsSuccess]: (state, { payload }) => {
const newState = { ...state, topStats: payload, processingTopStats: false };
return newState;
},
[actions.getStatsHistoryRequest]: state => ({ ...state, processingStatsHistory: true }),
[actions.getStatsHistoryFailure]: state => ({ ...state, processingStatsHistory: false }),
[actions.getStatsHistorySuccess]: (state, { payload }) => {
const newState = { ...state, statsHistory: payload, processingStatsHistory: false };
return newState;
},
[actions.toggleLogStatusRequest]: state => ({ ...state, logStatusProcessing: true }),
[actions.toggleLogStatusFailure]: state => ({ ...state, logStatusProcessing: false }),
[actions.toggleLogStatusSuccess]: (state) => {
@ -199,8 +179,6 @@ const dashboard = handleActions({
}, {
processing: true,
isCoreRunning: false,
processingTopStats: true,
processingStats: true,
logStatusProcessing: false,
processingVersion: true,
processingFiltering: true,
@ -217,7 +195,6 @@ const dashboard = handleActions({
dnsVersion: '',
clients: [],
autoClients: [],
topStats: [],
});
const queryLogs = handleActions({
@ -230,7 +207,11 @@ const queryLogs = handleActions({
[actions.downloadQueryLogRequest]: state => ({ ...state, logsDownloading: true }),
[actions.downloadQueryLogFailure]: state => ({ ...state, logsDownloading: false }),
[actions.downloadQueryLogSuccess]: state => ({ ...state, logsDownloading: false }),
}, { getLogsProcessing: false, logsDownloading: false });
}, {
getLogsProcessing: false,
logsDownloading: false,
logs: [],
});
const filtering = handleActions({
[actions.setRulesRequest]: state => ({ ...state, processingRules: true }),
@ -426,6 +407,7 @@ export default combineReducers({
access,
rewrites,
services,
stats,
loadingBar: loadingBarReducer,
form: formReducer,
});

97
client/src/reducers/stats.js Normal file
View File

@ -0,0 +1,97 @@
import { handleActions } from 'redux-actions';
import * as actions from '../actions/stats';
const defaultStats = {
dnsQueries: [],
blockedFiltering: [],
replacedParental: [],
replacedSafebrowsing: [],
topBlockedDomains: [],
topClients: [],
topQueriedDomains: [],
numBlockedFiltering: 0,
numDnsQueries: 0,
numReplacedParental: 0,
numReplacedSafebrowsing: 0,
numReplacedSafesearch: 0,
avgProcessingTime: 0,
};
const stats = handleActions(
{
[actions.getStatsConfigRequest]: state => ({ ...state, processingGetConfig: true }),
[actions.getStatsConfigFailure]: state => ({ ...state, processingGetConfig: false }),
[actions.getStatsConfigSuccess]: (state, { payload }) => ({
...state,
interval: payload.interval,
processingGetConfig: false,
}),
[actions.setStatsConfigRequest]: state => ({ ...state, processingSetConfig: true }),
[actions.setStatsConfigFailure]: state => ({ ...state, processingSetConfig: false }),
[actions.setStatsConfigSuccess]: (state, { payload }) => ({
...state,
interval: payload.interval,
processingSetConfig: false,
}),
[actions.getStatsRequest]: state => ({ ...state, processingStats: true }),
[actions.getStatsFailure]: state => ({ ...state, processingStats: false }),
[actions.getStatsSuccess]: (state, { payload }) => {
const {
dns_queries: dnsQueries,
blocked_filtering: blockedFiltering,
replaced_parental: replacedParental,
replaced_safebrowsing: replacedSafebrowsing,
top_blocked_domains: topBlockedDomains,
top_clients: topClients,
top_queried_domains: topQueriedDomains,
num_blocked_filtering: numBlockedFiltering,
num_dns_queries: numDnsQueries,
num_replaced_parental: numReplacedParental,
num_replaced_safebrowsing: numReplacedSafebrowsing,
num_replaced_safesearch: numReplacedSafesearch,
avg_processing_time: avgProcessingTime,
} = payload;
const newState = {
...state,
processingStats: false,
dnsQueries,
blockedFiltering,
replacedParental,
replacedSafebrowsing,
topBlockedDomains,
topClients,
topQueriedDomains,
numBlockedFiltering,
numDnsQueries,
numReplacedParental,
numReplacedSafebrowsing,
numReplacedSafesearch,
avgProcessingTime,
};
return newState;
},
[actions.resetStatsRequest]: state => ({ ...state, processingReset: true }),
[actions.resetStatsFailure]: state => ({ ...state, processingReset: false }),
[actions.resetStatsSuccess]: state => ({
...state,
...defaultStats,
processingReset: false,
}),
},
{
processingGetConfig: false,
processingSetConfig: false,
processingStats: true,
processingReset: false,
interval: 1,
...defaultStats,
},
);
export default stats;

97
dnsforward/dnsforward.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/log"
@ -40,7 +41,7 @@ type Server struct {
dnsProxy *proxy.Proxy // DNS proxy instance
dnsFilter *dnsfilter.Dnsfilter // DNS filter instance
queryLog *queryLog // Query log instance
stats *stats // General server statistics
stats stats.Stats
AllowedClients map[string]bool // IP addresses of whitelist clients
DisallowedClients map[string]bool // IP addresses of clients that should be blocked
@ -55,22 +56,14 @@ type Server struct {
// NewServer creates a new instance of the dnsforward.Server
// baseDir is the base directory for query logs
// Note: this function must be called only once
func NewServer(baseDir string) *Server {
func NewServer(baseDir string, stats stats.Stats) *Server {
s := &Server{
queryLog: newQueryLog(baseDir),
stats: newStats(),
}
log.Tracef("Loading stats from querylog")
err := s.queryLog.fillStatsFromQueryLog(s.stats)
if err != nil {
log.Error("failed to load stats from querylog: %s", err)
}
s.stats = stats
log.Printf("Start DNS server periodic jobs")
go s.queryLog.periodicQueryLogRotate()
go s.queryLog.runningTop.periodicHourlyTopRotate()
go s.stats.statsRotator()
return s
}
@ -357,38 +350,6 @@ func (s *Server) GetQueryLog() []map[string]interface{} {
return s.queryLog.getQueryLog()
}
// GetStatsTop returns the current stop stats
func (s *Server) GetStatsTop() *StatsTop {
s.RLock()
defer s.RUnlock()
return s.queryLog.runningTop.getStatsTop()
}
// PurgeStats purges current server stats
func (s *Server) PurgeStats() {
s.Lock()
defer s.Unlock()
s.stats.purgeStats()
}
// GetAggregatedStats returns aggregated stats data for the 24 hours
func (s *Server) GetAggregatedStats() map[string]interface{} {
s.RLock()
defer s.RUnlock()
return s.stats.getAggregatedStats()
}
// GetStatsHistory gets stats history aggregated by the specified time unit
// timeUnit is either time.Second, time.Minute, time.Hour, or 24*time.Hour
// start is start of the time range
// end is end of the time range
// returns nil if time unit is not supported
func (s *Server) GetStatsHistory(timeUnit time.Duration, startTime time.Time, endTime time.Time) (map[string]interface{}, error) {
s.RLock()
defer s.RUnlock()
return s.stats.getStatsHistory(timeUnit, startTime, endTime)
}
// Return TRUE if this client should be blocked
func (s *Server) isBlockedIP(ip string) bool {
if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 {
@ -507,21 +468,61 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
shouldLog = false
}
if s.conf.QueryLogEnabled && shouldLog {
elapsed := time.Since(start)
if s.conf.QueryLogEnabled && shouldLog {
upstreamAddr := ""
if d.Upstream != nil {
upstreamAddr = d.Upstream.Address()
}
entry := s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
if entry != nil {
s.stats.incrementCounters(entry)
}
_ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
}
s.updateStats(d, elapsed, *res)
return nil
}
func (s *Server) updateStats(d *proxy.DNSContext, elapsed time.Duration, res dnsfilter.Result) {
if s.stats == nil {
return
}
e := stats.Entry{}
e.Domain = strings.ToLower(d.Req.Question[0].Name)
e.Domain = e.Domain[:len(e.Domain)-1] // remove last "."
switch addr := d.Addr.(type) {
case *net.UDPAddr:
e.Client = addr.IP
case *net.TCPAddr:
e.Client = addr.IP
}
e.Time = uint(elapsed / 1000)
switch res.Reason {
case dnsfilter.NotFilteredNotFound:
fallthrough
case dnsfilter.NotFilteredWhiteList:
fallthrough
case dnsfilter.NotFilteredError:
e.Result = stats.RNotFiltered
case dnsfilter.FilteredSafeBrowsing:
e.Result = stats.RSafeBrowsing
case dnsfilter.FilteredParental:
e.Result = stats.RParental
case dnsfilter.FilteredSafeSearch:
e.Result = stats.RSafeSearch
case dnsfilter.FilteredBlackList:
fallthrough
case dnsfilter.FilteredInvalid:
fallthrough
case dnsfilter.FilteredBlockedService:
e.Result = stats.RFiltered
}
s.stats.Update(e)
}
// filterDNSRequest applies the dnsFilter and sets d.Res if the request was filtered
func (s *Server) filterDNSRequest(d *proxy.DNSContext) (*dnsfilter.Result, error) {
var res dnsfilter.Result

61
dnsforward/dnsforward_test.go
View File

@ -48,10 +48,6 @@ func TestServer(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 0, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
// message over TCP
req = createGoogleATestMessage()
@ -66,11 +62,6 @@ func TestServer(t *testing.T) {
// check query log and stats again
log = s.GetQueryLog()
assert.Equal(t, 2, len(log), "Log size")
stats = s.GetStatsTop()
// Length did not change as we queried the same domain
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 0, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -100,10 +91,6 @@ func TestServerWithProtectionDisabled(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 0, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -195,11 +182,6 @@ func TestSafeSearch(t *testing.T) {
exchangeAndAssertResponse(t, &client, addr, host, "213.180.193.56")
}
// Check aggregated stats
assert.Equal(t, s.GetAggregatedStats()["replaced_safesearch"], float64(len(yandexDomains)))
assert.Equal(t, s.GetAggregatedStats()["blocked_filtering"], float64(len(yandexDomains)))
assert.Equal(t, s.GetAggregatedStats()["dns_queries"], float64(len(yandexDomains)))
// Let's lookup for google safesearch ip
ips, err := net.LookupIP("forcesafesearch.google.com")
if err != nil {
@ -220,27 +202,6 @@ func TestSafeSearch(t *testing.T) {
exchangeAndAssertResponse(t, &client, addr, host, ip.String())
}
// Check aggregated stats
assert.Equal(t, s.GetAggregatedStats()["replaced_safesearch"], float64(len(yandexDomains)+len(googleDomains)))
assert.Equal(t, s.GetAggregatedStats()["blocked_filtering"], float64(len(yandexDomains)+len(googleDomains)))
assert.Equal(t, s.GetAggregatedStats()["dns_queries"], float64(len(yandexDomains)+len(googleDomains)))
// Do one more exchange
exchangeAndAssertResponse(t, &client, addr, "google-public-dns-a.google.com.", "8.8.8.8")
// Check aggregated stats
assert.Equal(t, s.GetAggregatedStats()["replaced_safesearch"], float64(len(yandexDomains)+len(googleDomains)))
assert.Equal(t, s.GetAggregatedStats()["blocked_filtering"], float64(len(yandexDomains)+len(googleDomains)))
assert.Equal(t, s.GetAggregatedStats()["dns_queries"], float64(len(yandexDomains)+len(googleDomains)+1))
// Count of blocked domains (there is `yandex.com` duplicate in yandexDomains array)
blockedCount := len(yandexDomains) - 1 + len(googleDomains)
assert.Equal(t, len(s.GetStatsTop().Blocked), blockedCount)
// Count of domains (blocked domains + `google-public-dns-a.google.com`)
domainsCount := blockedCount + 1
assert.Equal(t, len(s.GetStatsTop().Domains), domainsCount)
err = s.Stop()
if err != nil {
t.Fatalf("Can not stopd server cause: %s", err)
@ -272,10 +233,6 @@ func TestInvalidRequest(t *testing.T) {
// invalid requests aren't written to the query log
log := s.GetQueryLog()
assert.Equal(t, 0, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 0, len(stats.Domains), "Top domains length")
assert.Equal(t, 0, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 0, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -313,10 +270,6 @@ func TestBlockedRequest(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 1, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -362,10 +315,6 @@ func TestNullBlockedRequest(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 1, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -410,10 +359,6 @@ func TestBlockedByHosts(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 1, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -469,10 +414,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
stats := s.GetStatsTop()
assert.Equal(t, 1, len(stats.Domains), "Top domains length")
assert.Equal(t, 1, len(stats.Blocked), "Top blocked length")
assert.Equal(t, 1, len(stats.Clients), "Top clients length")
err = s.Stop()
if err != nil {
@ -481,7 +422,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
}
func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t))
s := NewServer(createDataDir(t), nil)
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}

10
dnsforward/querylog.go
View File

@ -26,7 +26,6 @@ const (
// queryLog is a structure that writes and reads the DNS query log
type queryLog struct {
logFile string // path to the log file
runningTop *dayTop // current top charts
logBufferLock sync.RWMutex
logBuffer []*logEntry
@ -41,9 +40,7 @@ type queryLog struct {
func newQueryLog(baseDir string) *queryLog {
l := &queryLog{
logFile: filepath.Join(baseDir, queryLogFileName),
runningTop: &dayTop{},
}
l.runningTop.init()
return l
}
@ -112,13 +109,6 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
}
l.queryLogLock.Unlock()
// add it to running top
err = l.runningTop.addEntry(&entry, question, now)
if err != nil {
log.Printf("Failed to add entry to running top: %s", err)
// don't do failure, just log
}
// if buffer needs to be flushed to disk, do it now
if needFlush {
// write to file

96
dnsforward/querylog_file.go
View File

@ -178,99 +178,3 @@ func (l *queryLog) periodicQueryLogRotate() {
}
}
}
func (l *queryLog) genericLoader(onEntry func(entry *logEntry) error, needMore func() bool, timeWindow time.Duration) error {
now := time.Now()
// read from querylog files, try newest file first
var files []string
if enableGzip {
files = []string{
l.logFile + ".gz",
l.logFile + ".gz.1",
}
} else {
files = []string{
l.logFile,
l.logFile + ".1",
}
}
// read from all files
for _, file := range files {
if !needMore() {
break
}
if _, err := os.Stat(file); os.IsNotExist(err) {
// do nothing, file doesn't exist
continue
}
f, err := os.Open(file)
if err != nil {
log.Error("Failed to open file \"%s\": %s", file, err)
// try next file
continue
}
defer f.Close()
var d *json.Decoder
if enableGzip {
zr, err := gzip.NewReader(f)
if err != nil {
log.Error("Failed to create gzip reader: %s", err)
continue
}
defer zr.Close()
d = json.NewDecoder(zr)
} else {
d = json.NewDecoder(f)
}
i := 0
over := 0
max := 10000 * time.Second
var sum time.Duration
// entries on file are in oldest->newest order
// we want maxLen newest
for d.More() {
if !needMore() {
break
}
var entry logEntry
err := d.Decode(&entry)
if err != nil {
log.Error("Failed to decode: %s", err)
// next entry can be fine, try more
continue
}
if now.Sub(entry.Time) > timeWindow {
// log.Tracef("skipping entry") // debug logging
continue
}
if entry.Elapsed > max {
over++
} else {
sum += entry.Elapsed
}
i++
err = onEntry(&entry)
if err != nil {
return err
}
}
elapsed := time.Since(now)
var perunit time.Duration
var avg time.Duration
if i > 0 {
perunit = elapsed / time.Duration(i)
avg = sum / time.Duration(i)
}
log.Debug("file \"%s\": read %d entries in %v, %v/entry, %v over %v, %v avg", file, i, elapsed, perunit, over, max, avg)
}
return nil
}

315
dnsforward/querylog_top.go
View File

@ -1,315 +0,0 @@
package dnsforward
import (
"fmt"
"os"
"path"
"runtime"
"strings"
"sync"
"time"
"github.com/AdguardTeam/golibs/log"
"github.com/bluele/gcache"
"github.com/miekg/dns"
)
type hourTop struct {
domains gcache.Cache
blocked gcache.Cache
clients gcache.Cache
mutex sync.RWMutex
}
func (h *hourTop) init() {
h.domains = gcache.New(queryLogTopSize).LRU().Build()
h.blocked = gcache.New(queryLogTopSize).LRU().Build()
h.clients = gcache.New(queryLogTopSize).LRU().Build()
}
type dayTop struct {
hours []*hourTop
hoursLock sync.RWMutex // writelock this lock ONLY WHEN rotating or intializing hours!
loaded bool
loadedLock sync.Mutex
}
func (d *dayTop) init() {
d.hoursWriteLock()
for i := 0; i < 24; i++ {
hour := hourTop{}
hour.init()
d.hours = append(d.hours, &hour)
}
d.hoursWriteUnlock()
}
func (d *dayTop) rotateHourlyTop() {
log.Printf("Rotating hourly top")
hour := &hourTop{}
hour.init()
d.hoursWriteLock()
d.hours = append([]*hourTop{hour}, d.hours...)
d.hours = d.hours[:24]
d.hoursWriteUnlock()
}
func (d *dayTop) periodicHourlyTopRotate() {
t := time.Hour
for range time.Tick(t) {
d.rotateHourlyTop()
}
}
func (h *hourTop) incrementValue(key string, cache gcache.Cache) error {
h.Lock()
defer h.Unlock()
ivalue, err := cache.Get(key)
if err == gcache.KeyNotFoundError {
// we just set it and we're done
err = cache.Set(key, 1)
if err != nil {
log.Printf("Failed to set hourly top value: %s", err)
return err
}
return nil
}
if err != nil {
log.Printf("gcache encountered an error during get: %s", err)
return err
}
cachedValue, ok := ivalue.(int)
if !ok {
err = fmt.Errorf("SHOULD NOT HAPPEN: gcache has non-int as value: %v", ivalue)
log.Println(err)
return err
}
err = cache.Set(key, cachedValue+1)
if err != nil {
log.Printf("Failed to set hourly top value: %s", err)
return err
}
return nil
}
func (h *hourTop) incrementDomains(key string) error {
return h.incrementValue(key, h.domains)
}
func (h *hourTop) incrementBlocked(key string) error {
return h.incrementValue(key, h.blocked)
}
func (h *hourTop) incrementClients(key string) error {
return h.incrementValue(key, h.clients)
}
// if does not exist -- return 0
func (h *hourTop) lockedGetValue(key string, cache gcache.Cache) (int, error) {
ivalue, err := cache.Get(key)
if err == gcache.KeyNotFoundError {
return 0, nil
}
if err != nil {
log.Printf("gcache encountered an error during get: %s", err)
return 0, err
}
value, ok := ivalue.(int)
if !ok {
err := fmt.Errorf("SHOULD NOT HAPPEN: gcache has non-int as value: %v", ivalue)
log.Println(err)
return 0, err
}
return value, nil
}
func (h *hourTop) lockedGetDomains(key string) (int, error) {
return h.lockedGetValue(key, h.domains)
}
func (h *hourTop) lockedGetBlocked(key string) (int, error) {
return h.lockedGetValue(key, h.blocked)
}
func (h *hourTop) lockedGetClients(key string) (int, error) {
return h.lockedGetValue(key, h.clients)
}
func (d *dayTop) addEntry(entry *logEntry, q *dns.Msg, now time.Time) error {
// figure out which hour bucket it belongs to
hour := int(now.Sub(entry.Time).Hours())
if hour >= 24 {
log.Printf("t %v is >24 hours ago, ignoring", entry.Time)
return nil
}
// if a DNS query doesn't have questions, do nothing
if len(q.Question) == 0 {
return nil
}
hostname := strings.ToLower(strings.TrimSuffix(q.Question[0].Name, "."))
// if question hostname is empty, do nothing
if hostname == "" {
return nil
}
// get value, if not set, crate one
d.hoursReadLock()
defer d.hoursReadUnlock()
err := d.hours[hour].incrementDomains(hostname)
if err != nil {
log.Printf("Failed to increment value: %s", err)
return err
}
if entry.Result.IsFiltered {
err := d.hours[hour].incrementBlocked(hostname)
if err != nil {
log.Printf("Failed to increment value: %s", err)
return err
}
}
if len(entry.IP) > 0 {
err := d.hours[hour].incrementClients(entry.IP)
if err != nil {
log.Printf("Failed to increment value: %s", err)
return err
}
}
return nil
}
func (l *queryLog) fillStatsFromQueryLog(s *stats) error {
now := time.Now()
l.runningTop.loadedWriteLock()
defer l.runningTop.loadedWriteUnlock()
if l.runningTop.loaded {
return nil
}
onEntry := func(entry *logEntry) error {
if len(entry.Question) == 0 {
log.Printf("entry question is absent, skipping")
return nil
}
if entry.Time.After(now) {
log.Printf("t %v vs %v is in the future, ignoring", entry.Time, now)
return nil
}
q := new(dns.Msg)
if err := q.Unpack(entry.Question); err != nil {
log.Printf("failed to unpack dns message question: %s", err)
return err
}
if len(q.Question) != 1 {
log.Printf("malformed dns message, has no questions, skipping")
return nil
}
err := l.runningTop.addEntry(entry, q, now)
if err != nil {
log.Printf("Failed to add entry to running top: %s", err)
return err
}
l.queryLogLock.Lock()
l.queryLogCache = append(l.queryLogCache, entry)
if len(l.queryLogCache) > queryLogSize {
toremove := len(l.queryLogCache) - queryLogSize
l.queryLogCache = l.queryLogCache[toremove:]
}
l.queryLogLock.Unlock()
s.incrementCounters(entry)
return nil
}
needMore := func() bool { return true }
err := l.genericLoader(onEntry, needMore, queryLogTimeLimit)
if err != nil {
log.Printf("Failed to load entries from querylog: %s", err)
return err
}
l.runningTop.loaded = true
return nil
}
// StatsTop represents top stat charts
type StatsTop struct {
Domains map[string]int // Domains - top requested domains
Blocked map[string]int // Blocked - top blocked domains
Clients map[string]int // Clients - top DNS clients
}
// getStatsTop returns the current top stats
func (d *dayTop) getStatsTop() *StatsTop {
s := &StatsTop{
Domains: map[string]int{},
Blocked: map[string]int{},
Clients: map[string]int{},
}
do := func(keys []interface{}, getter func(key string) (int, error), result map[string]int) {
for _, ikey := range keys {
key, ok := ikey.(string)
if !ok {
continue
}
value, err := getter(key)
if err != nil {
log.Printf("Failed to get top domains value for %v: %s", key, err)
return
}
result[key] += value
}
}
d.hoursReadLock()
for hour := 0; hour < 24; hour++ {
d.hours[hour].RLock()
do(d.hours[hour].domains.Keys(false), d.hours[hour].lockedGetDomains, s.Domains)
do(d.hours[hour].blocked.Keys(false), d.hours[hour].lockedGetBlocked, s.Blocked)
do(d.hours[hour].clients.Keys(false), d.hours[hour].lockedGetClients, s.Clients)
d.hours[hour].RUnlock()
}
d.hoursReadUnlock()
return s
}
func (d *dayTop) hoursWriteLock() { tracelock(); d.hoursLock.Lock() }
func (d *dayTop) hoursWriteUnlock() { tracelock(); d.hoursLock.Unlock() }
func (d *dayTop) hoursReadLock() { tracelock(); d.hoursLock.RLock() }
func (d *dayTop) hoursReadUnlock() { tracelock(); d.hoursLock.RUnlock() }
func (d *dayTop) loadedWriteLock() { tracelock(); d.loadedLock.Lock() }
func (d *dayTop) loadedWriteUnlock() { tracelock(); d.loadedLock.Unlock() }
func (h *hourTop) Lock() { tracelock(); h.mutex.Lock() }
func (h *hourTop) RLock() { tracelock(); h.mutex.RLock() }
func (h *hourTop) RUnlock() { tracelock(); h.mutex.RUnlock() }
func (h *hourTop) Unlock() { tracelock(); h.mutex.Unlock() }
func tracelock() {
if false { // not commented out to make code checked during compilation
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := path.Base(runtime.FuncForPC(pc[1]).Name())
lockf := path.Base(runtime.FuncForPC(pc[0]).Name())
fmt.Fprintf(os.Stderr, "%s(): %s\n", f, lockf)
}
}

355
dnsforward/stats.go
View File

@ -1,355 +0,0 @@
package dnsforward
import (
"fmt"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
)
// how far back to keep the stats
const statsHistoryElements = 60 + 1 // +1 for calculating delta
// entries for single time period (for example all per-second entries)
type statsEntries map[string][statsHistoryElements]float64
// each periodic stat is a map of arrays
type periodicStats struct {
entries statsEntries
period time.Duration // how long one entry lasts
lastRotate time.Time // last time this data was rotated
sync.RWMutex
}
// stats is the DNS server historical statistics
type stats struct {
perSecond periodicStats
perMinute periodicStats
perHour periodicStats
perDay periodicStats
requests *counter // total number of requests
filtered *counter // total number of filtered requests
filteredLists *counter // total number of requests blocked by filter lists
filteredSafebrowsing *counter // total number of requests blocked by safebrowsing
filteredParental *counter // total number of requests blocked by the parental control
whitelisted *counter // total number of requests whitelisted by filter lists
safesearch *counter // total number of requests for which safe search rules were applied
errorsTotal *counter // total number of errors
elapsedTime *histogram // requests duration histogram
}
// initializes an empty stats structure
func newStats() *stats {
s := &stats{
requests: newDNSCounter("requests_total"),
filtered: newDNSCounter("filtered_total"),
filteredLists: newDNSCounter("filtered_lists_total"),
filteredSafebrowsing: newDNSCounter("filtered_safebrowsing_total"),
filteredParental: newDNSCounter("filtered_parental_total"),
whitelisted: newDNSCounter("whitelisted_total"),
safesearch: newDNSCounter("safesearch_total"),
errorsTotal: newDNSCounter("errors_total"),
elapsedTime: newDNSHistogram("request_duration"),
}
// Initializes empty per-sec/minute/hour/day stats
s.purgeStats()
return s
}
func initPeriodicStats(periodic *periodicStats, period time.Duration) {
periodic.Lock()
periodic.entries = statsEntries{}
periodic.lastRotate = time.Now()
periodic.period = period
periodic.Unlock()
}
func (s *stats) purgeStats() {
initPeriodicStats(&s.perSecond, time.Second)
initPeriodicStats(&s.perMinute, time.Minute)
initPeriodicStats(&s.perHour, time.Hour)
initPeriodicStats(&s.perDay, time.Hour*24)
}
func (p *periodicStats) Inc(name string, when time.Time) {
// calculate how many periods ago this happened
elapsed := int64(time.Since(when) / p.period)
// log.Tracef("%s: %v as %v -> [%v]", name, time.Since(when), p.period, elapsed)
if elapsed >= statsHistoryElements {
return // outside of our timeframe
}
p.Lock()
currentValues := p.entries[name]
currentValues[elapsed]++
p.entries[name] = currentValues
p.Unlock()
}
func (p *periodicStats) Observe(name string, when time.Time, value float64) {
// calculate how many periods ago this happened
elapsed := int64(time.Since(when) / p.period)
// log.Tracef("%s: %v as %v -> [%v]", name, time.Since(when), p.period, elapsed)
if elapsed >= statsHistoryElements {
return // outside of our timeframe
}
p.Lock()
{
countname := name + "_count"
currentValues := p.entries[countname]
v := currentValues[elapsed]
// log.Tracef("Will change p.entries[%s][%d] from %v to %v", countname, elapsed, value, value+1)
v++
currentValues[elapsed] = v
p.entries[countname] = currentValues
}
{
totalname := name + "_sum"
currentValues := p.entries[totalname]
currentValues[elapsed] += value
p.entries[totalname] = currentValues
}
p.Unlock()
}
func (p *periodicStats) statsRotate(now time.Time) {
p.Lock()
rotations := int64(now.Sub(p.lastRotate) / p.period)
if rotations > statsHistoryElements {
rotations = statsHistoryElements
}
// calculate how many times we should rotate
for r := int64(0); r < rotations; r++ {
for key, values := range p.entries {
newValues := [statsHistoryElements]float64{}
for i := 1; i < len(values); i++ {
newValues[i] = values[i-1]
}
p.entries[key] = newValues
}
}
if rotations > 0 {
p.lastRotate = now
}
p.Unlock()
}
func (s *stats) statsRotator() {
for range time.Tick(time.Second) {
now := time.Now()
s.perSecond.statsRotate(now)
s.perMinute.statsRotate(now)
s.perHour.statsRotate(now)
s.perDay.statsRotate(now)
}
}
// counter that wraps around prometheus Counter but also adds to periodic stats
type counter struct {
name string // used as key in periodic stats
value int64
sync.Mutex
}
func newDNSCounter(name string) *counter {
// log.Tracef("called")
return &counter{
name: name,
}
}
func (s *stats) incWithTime(c *counter, when time.Time) {
s.perSecond.Inc(c.name, when)
s.perMinute.Inc(c.name, when)
s.perHour.Inc(c.name, when)
s.perDay.Inc(c.name, when)
c.Lock()
c.value++
c.Unlock()
}
type histogram struct {
name string // used as key in periodic stats
count int64
total float64
sync.Mutex
}
func newDNSHistogram(name string) *histogram {
return &histogram{
name: name,
}
}
func (s *stats) observeWithTime(h *histogram, value float64, when time.Time) {
s.perSecond.Observe(h.name, when, value)
s.perMinute.Observe(h.name, when, value)
s.perHour.Observe(h.name, when, value)
s.perDay.Observe(h.name, when, value)
h.Lock()
h.count++
h.total += value
h.Unlock()
}
// -----
// stats
// -----
func (s *stats) incrementCounters(entry *logEntry) {
s.incWithTime(s.requests, entry.Time)
if entry.Result.IsFiltered {
s.incWithTime(s.filtered, entry.Time)
}
switch entry.Result.Reason {
case dnsfilter.NotFilteredWhiteList:
s.incWithTime(s.whitelisted, entry.Time)
case dnsfilter.NotFilteredError:
s.incWithTime(s.errorsTotal, entry.Time)
case dnsfilter.FilteredBlackList:
s.incWithTime(s.filteredLists, entry.Time)
case dnsfilter.FilteredSafeBrowsing:
s.incWithTime(s.filteredSafebrowsing, entry.Time)
case dnsfilter.FilteredParental:
s.incWithTime(s.filteredParental, entry.Time)
case dnsfilter.FilteredInvalid:
// do nothing
case dnsfilter.FilteredSafeSearch:
s.incWithTime(s.safesearch, entry.Time)
}
s.observeWithTime(s.elapsedTime, entry.Elapsed.Seconds(), entry.Time)
}
// getAggregatedStats returns aggregated stats data for the 24 hours
func (s *stats) getAggregatedStats() map[string]interface{} {
const numHours = 24
historical := s.generateMapFromStats(&s.perHour, 0, numHours)
// sum them up
summed := map[string]interface{}{}
for key, values := range historical {
summedValue := 0.0
floats, ok := values.([]float64)
if !ok {
continue
}
for _, v := range floats {
summedValue += v
}
summed[key] = summedValue
}
// don't forget to divide by number of elements in returned slice
if val, ok := summed["avg_processing_time"]; ok {
if flval, flok := val.(float64); flok {
flval /= numHours
summed["avg_processing_time"] = flval
}
}
summed["stats_period"] = "24 hours"
return summed
}
func (s *stats) generateMapFromStats(stats *periodicStats, start int, end int) map[string]interface{} {
stats.RLock()
defer stats.RUnlock()
// clamp
start = clamp(start, 0, statsHistoryElements)
end = clamp(end, 0, statsHistoryElements)
avgProcessingTime := make([]float64, 0)
count := getReversedSlice(stats.entries[s.elapsedTime.name+"_count"], start, end)
sum := getReversedSlice(stats.entries[s.elapsedTime.name+"_sum"], start, end)
for i := 0; i < len(count); i++ {
var avg float64
if count[i] != 0 {
avg = sum[i] / count[i]
avg *= 1000
}
avgProcessingTime = append(avgProcessingTime, avg)
}
result := map[string]interface{}{
"dns_queries": getReversedSlice(stats.entries[s.requests.name], start, end),
"blocked_filtering": getReversedSlice(stats.entries[s.filtered.name], start, end),
"replaced_safebrowsing": getReversedSlice(stats.entries[s.filteredSafebrowsing.name], start, end),
"replaced_safesearch": getReversedSlice(stats.entries[s.safesearch.name], start, end),
"replaced_parental": getReversedSlice(stats.entries[s.filteredParental.name], start, end),
"avg_processing_time": avgProcessingTime,
}
return result
}
// getStatsHistory gets stats history aggregated by the specified time unit
// timeUnit is either time.Second, time.Minute, time.Hour, or 24*time.Hour
// start is start of the time range
// end is end of the time range
// returns nil if time unit is not supported
func (s *stats) getStatsHistory(timeUnit time.Duration, startTime time.Time, endTime time.Time) (map[string]interface{}, error) {
var stats *periodicStats
switch timeUnit {
case time.Second:
stats = &s.perSecond
case time.Minute:
stats = &s.perMinute
case time.Hour:
stats = &s.perHour
case 24 * time.Hour:
stats = &s.perDay
}
if stats == nil {
return nil, fmt.Errorf("unsupported time unit: %v", timeUnit)
}
now := time.Now()
// check if start and time times are within supported time range
timeRange := timeUnit * statsHistoryElements
if startTime.Add(timeRange).Before(now) {
return nil, fmt.Errorf("start_time parameter is outside of supported range: %s", startTime.String())
}
if endTime.Add(timeRange).Before(now) {
return nil, fmt.Errorf("end_time parameter is outside of supported range: %s", startTime.String())
}
// calculate start and end of our array
// basically it's how many hours/minutes/etc have passed since now
start := int(now.Sub(endTime) / timeUnit)
end := int(now.Sub(startTime) / timeUnit)
// swap them around if they're inverted
if start > end {
start, end = end, start
}
return s.generateMapFromStats(stats, start, end), nil
}
func clamp(value, low, high int) int {
if value < low {
return low
}
if value > high {
return high
}
return value
}
// --------------------------
// helper functions for stats
// --------------------------
func getReversedSlice(input [statsHistoryElements]float64, start int, end int) []float64 {
output := make([]float64, 0)
for i := start; i <= end; i++ {
output = append([]float64{input[i]}, output...)
}
return output
}

4
go.mod
View File

@ -7,7 +7,9 @@ require (
github.com/AdguardTeam/golibs v0.2.1
github.com/AdguardTeam/urlfilter v0.5.0
github.com/NYTimes/gziphandler v1.1.1
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833
github.com/etcd-io/bbolt v1.3.3
github.com/go-test/deep v1.0.1
github.com/gobuffalo/packr v1.19.0
github.com/joomcode/errorx v0.8.0
@ -17,8 +19,8 @@ require (
github.com/miekg/dns v1.1.8
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
github.com/stretchr/testify v1.4.0
go.etcd.io/bbolt v1.3.3 // indirect
golang.org/x/net v0.0.0-20190620200207-3b0461eec859
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477
gopkg.in/yaml.v2 v2.2.2
)

4
go.sum
View File

@ -28,6 +28,7 @@ github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833/go.mod h1:8c4/i2Vlov
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
@ -80,6 +81,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@ -112,8 +114,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+lg4zao9W4HIDzlFbMYgSgtvNVHh00MEHvbGpQ=
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477/go.mod h1:QDV1vrFSrowdoOba0UM8VJPUZONT7dnfdLsM+GG53Z8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

10
home/config.go
View File

@ -12,6 +12,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dhcpd"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/golibs/file"
"github.com/AdguardTeam/golibs/log"
yaml "gopkg.in/yaml.v2"
@ -68,6 +69,7 @@ type configuration struct {
controlLock sync.Mutex
transport *http.Transport
client *http.Client
stats stats.Stats
// cached version.json to avoid hammering github.io for each page reload
versionCheckJSON []byte
@ -107,6 +109,9 @@ type dnsConfig struct {
BindHost string `yaml:"bind_host"`
Port int `yaml:"port"`
// time interval for statistics (in days)
StatsInterval uint `yaml:"statistics_interval"`
dnsforward.FilteringConfig `yaml:",inline"`
UpstreamDNS []string `yaml:"upstream_dns"`
@ -163,6 +168,7 @@ var config = configuration{
DNS: dnsConfig{
BindHost: "0.0.0.0",
Port: 53,
StatsInterval: 1,
FilteringConfig: dnsforward.FilteringConfig{
ProtectionEnabled: true, // whether or not use any of dnsfilter features
FilteringEnabled: true, // whether or not use filter lists
@ -264,6 +270,10 @@ func parseConfig() error {
return err
}
if !checkStatsInterval(config.DNS.StatsInterval) {
config.DNS.StatsInterval = 1
}
for _, cy := range config.Clients {
cli := Client{
Name: cy.Name,

150
home/control.go
View File

@ -1,12 +1,10 @@
package home
import (
"bytes"
"encoding/json"
"fmt"
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
@ -177,149 +175,6 @@ func handleQueryLog(w http.ResponseWriter, r *http.Request) {
}
}
func handleStatsTop(w http.ResponseWriter, r *http.Request) {
s := config.dnsServer.GetStatsTop()
// use manual json marshalling because we want maps to be sorted by value
statsJSON := bytes.Buffer{}
statsJSON.WriteString("{\n")
gen := func(json *bytes.Buffer, name string, top map[string]int, addComma bool) {
json.WriteString(" ")
json.WriteString(fmt.Sprintf("%q", name))
json.WriteString(": {\n")
sorted := sortByValue(top)
// no more than 50 entries
if len(sorted) > 50 {
sorted = sorted[:50]
}
for i, key := range sorted {
json.WriteString(" ")
json.WriteString(fmt.Sprintf("%q", key))
json.WriteString(": ")
json.WriteString(strconv.Itoa(top[key]))
if i+1 != len(sorted) {
json.WriteByte(',')
}
json.WriteByte('\n')
}
json.WriteString(" }")
if addComma {
json.WriteByte(',')
}
json.WriteByte('\n')
}
gen(&statsJSON, "top_queried_domains", s.Domains, true)
gen(&statsJSON, "top_blocked_domains", s.Blocked, true)
gen(&statsJSON, "top_clients", s.Clients, true)
statsJSON.WriteString(" \"stats_period\": \"24 hours\"\n")
statsJSON.WriteString("}\n")
w.Header().Set("Content-Type", "application/json")
_, err := w.Write(statsJSON.Bytes())
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't write body: %s", err)
}
}
// handleStatsReset resets the stats caches
func handleStatsReset(w http.ResponseWriter, r *http.Request) {
config.dnsServer.PurgeStats()
_, err := fmt.Fprintf(w, "OK\n")
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't write body: %s", err)
}
}
// handleStats returns aggregated stats data for the 24 hours
func handleStats(w http.ResponseWriter, r *http.Request) {
summed := config.dnsServer.GetAggregatedStats()
statsJSON, err := json.Marshal(summed)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(statsJSON)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
}
// HandleStatsHistory returns historical stats data for the 24 hours
func handleStatsHistory(w http.ResponseWriter, r *http.Request) {
// handle time unit and prepare our time window size
timeUnitString := r.URL.Query().Get("time_unit")
var timeUnit time.Duration
switch timeUnitString {
case "seconds":
timeUnit = time.Second
case "minutes":
timeUnit = time.Minute
case "hours":
timeUnit = time.Hour
case "days":
timeUnit = time.Hour * 24
default:
http.Error(w, "Must specify valid time_unit parameter", http.StatusBadRequest)
return
}
// parse start and end time
startTime, err := time.Parse(time.RFC3339, r.URL.Query().Get("start_time"))
if err != nil {
httpError(w, http.StatusBadRequest, "Must specify valid start_time parameter: %s", err)
return
}
endTime, err := time.Parse(time.RFC3339, r.URL.Query().Get("end_time"))
if err != nil {
httpError(w, http.StatusBadRequest, "Must specify valid end_time parameter: %s", err)
return
}
data, err := config.dnsServer.GetStatsHistory(timeUnit, startTime, endTime)
if err != nil {
httpError(w, http.StatusBadRequest, "Cannot get stats history: %s", err)
return
}
statsJSON, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(statsJSON)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
}
// sortByValue is a helper function for querylog API
func sortByValue(m map[string]int) []string {
type kv struct {
k string
v int
}
var ss []kv
for k, v := range m {
ss = append(ss, kv{k, v})
}
sort.Slice(ss, func(l, r int) bool {
return ss[l].v > ss[r].v
})
sorted := []string{}
for _, v := range ss {
sorted = append(sorted, v.k)
}
return sorted
}
// -----------------------
// upstreams configuration
// -----------------------
@ -722,10 +577,6 @@ func registerControlHandlers() {
httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS)
httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage)
httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage)
httpRegister(http.MethodGet, "/control/stats_top", handleStatsTop)
httpRegister(http.MethodGet, "/control/stats", handleStats)
httpRegister(http.MethodGet, "/control/stats_history", handleStatsHistory)
httpRegister(http.MethodPost, "/control/stats_reset", handleStatsReset)
http.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON)))
httpRegister(http.MethodPost, "/control/update", handleUpdate)
httpRegister(http.MethodPost, "/control/filtering/enable", handleFilteringEnable)
@ -760,6 +611,7 @@ func registerControlHandlers() {
RegisterClientsHandlers()
registerRewritesHandlers()
RegisterBlockedServicesHandlers()
RegisterStatsHandlers()
http.HandleFunc("/dns-query", postInstall(handleDOH))
}

92
home/control_stats.go Normal file
View File

@ -0,0 +1,92 @@
package home
import (
"encoding/json"
"net/http"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/golibs/log"
)
type statsConfig struct {
Interval uint `json:"interval"`
}
// Get stats configuration
func handleStatsInfo(w http.ResponseWriter, r *http.Request) {
resp := statsConfig{}
resp.Interval = config.DNS.StatsInterval
jsonVal, err := json.Marshal(resp)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "http write: %s", err)
}
}
// Set stats configuration
func handleStatsConfig(w http.ResponseWriter, r *http.Request) {
reqData := statsConfig{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
httpError(w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkStatsInterval(reqData.Interval) {
httpError(w, http.StatusBadRequest, "Unsupported interval")
return
}
config.DNS.StatsInterval = reqData.Interval
config.stats.Configure(int(config.DNS.StatsInterval))
returnOK(w)
}
// handleStats returns aggregated stats data
func handleStats(w http.ResponseWriter, r *http.Request) {
units := stats.Hours
if config.DNS.StatsInterval > 7 {
units = stats.Days
}
counter := log.StartTimer()
d := config.stats.GetData(units)
counter.LogElapsed("Stats: prepared data")
if d == nil {
httpError(w, http.StatusInternalServerError, "Couldn't get statistics data")
return
}
data, err := json.Marshal(d)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Write(data)
}
// handleStatsReset resets the stats
func handleStatsReset(w http.ResponseWriter, r *http.Request) {
config.stats.Clear()
returnOK(w)
}
// RegisterStatsHandlers - register handlers
func RegisterStatsHandlers() {
httpRegister(http.MethodGet, "/control/stats", handleStats)
httpRegister(http.MethodPost, "/control/stats_reset", handleStatsReset)
httpRegister(http.MethodPost, "/control/stats_config", handleStatsConfig)
httpRegister(http.MethodGet, "/control/stats_info", handleStatsInfo)
}
func checkStatsInterval(i uint) bool {
return i == 1 || i == 7 || i == 30 || i == 90
}

14
home/dns.go
View File

@ -8,6 +8,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/log"
@ -33,7 +34,11 @@ func initDNSServer(baseDir string) {
log.Fatalf("Cannot create DNS data dir at %s: %s", baseDir, err)
}
config.dnsServer = dnsforward.NewServer(baseDir)
config.stats = stats.New("./data/stats.db", int(config.DNS.StatsInterval), nil)
if config.stats == nil {
log.Fatal("config.stats == nil")
}
config.dnsServer = dnsforward.NewServer(baseDir, config.stats)
initRDNS()
}
@ -152,11 +157,6 @@ func startDNSServer() error {
return errorx.Decorate(err, "Couldn't start forwarding DNS server")
}
top := config.dnsServer.GetStatsTop()
for k := range top.Clients {
beginAsyncRDNS(k)
}
return nil
}
@ -183,5 +183,7 @@ func stopDNSServer() error {
return errorx.Decorate(err, "Couldn't stop forwarding DNS server")
}
config.stats.Close()
return nil
}

188
openapi/openapi.yaml
View File

@ -214,18 +214,6 @@ paths:
# General statistics methods
# --------------------------------------------------
/stats_top:
get:
tags:
- stats
operationId: statusTop
summary: 'Get DNS server top client, domain and blocked statistics'
responses:
200:
description: OK
schema:
$ref: "#/definitions/StatsTop"
/stats:
get:
tags:
@ -234,46 +222,10 @@ paths:
summary: 'Get DNS server statistics'
responses:
200:
description: 'Returns general statistics for the last 24 hours'
description: 'Returns statistics data'
schema:
$ref: "#/definitions/Stats"
/stats_history:
get:
tags:
- stats
operationId: stats_history
summary: 'Get historical DNS server statistics for the last 24 hours'
parameters:
-
name: start_time
in: query
type: string
description: 'Start time in ISO8601 (example: `2018-05-04T17:55:33+00:00`)'
required: true
-
name: end_time
in: query
type: string
description: 'End time in ISO8601 (example: `2018-05-04T17:55:33+00:00`)'
required: true
-
name: time_unit
in: query
type: string
description: 'Time unit (`minutes` or `hours`)'
required: true
enum:
- minutes
- hours
responses:
501:
description: 'Requested time window is outside of supported range. It will be supported later, but not now.'
200:
description: 'Returns historical stats for the specified time interval.'
schema:
$ref: '#/definitions/StatsHistory'
/stats_reset:
post:
tags:
@ -284,6 +236,34 @@ paths:
200:
description: OK
/stats_info:
get:
tags:
- stats
operationId: statsInfo
summary: 'Get statistics parameters'
responses:
200:
schema:
$ref: "#/definitions/StatsConfig"
/stats_config:
post:
tags:
- stats
operationId: statsConfig
summary: "Set statistics parameters"
consumes:
- application/json
parameters:
- in: "body"
name: "body"
schema:
$ref: "#/definitions/StatsConfig"
responses:
200:
description: OK
# --------------------------------------------------
# TLS server methods
# --------------------------------------------------
@ -1103,28 +1083,29 @@ definitions:
type: "boolean"
Stats:
type: "object"
description: "General server stats for the last 24 hours"
required:
- "dns_queries"
- "blocked_filtering"
- "replaced_safebrowsing"
- "replaced_parental"
- "replaced_safesearch"
- "avg_processing_time"
description: "Server statistics data"
properties:
dns_queries:
time_units:
type: "string"
description: "Time units (hours | days)"
example: "hours"
num_dns_queries:
type: "integer"
description: "Total number of DNS queries"
example: 123
blocked_filtering:
num_blocked_filtering:
type: "integer"
description: "Number of requests blocked by filtering rules"
example: 50
replaced_safebrowsing:
num_replaced_safebrowsing:
type: "integer"
description: "Number of requests blocked by the safebrowsing module"
description: "Number of requests blocked by safebrowsing module"
example: 5
replaced_parental:
num_replaced_safesearch:
type: "integer"
description: "Number of requests blocked by safesearch module"
example: 5
num_replaced_parental:
type: "integer"
description: "Number of blocked adult websites"
example: 15
@ -1133,110 +1114,43 @@ definitions:
format: "float"
description: "Average time in milliseconds on processing a DNS"
example: 0.34
StatsTop:
type: "object"
description: "Server stats top charts"
required:
- "top_queried_domains"
- "top_clients"
- "top_blocked_domains"
properties:
top_queried_domains:
type: "array"
items:
type: "object"
example:
example.org: 12312
example.com: 321
example.net: 5555
top_clients:
type: "array"
items:
type: "object"
example:
127.0.0.1: 12312
192.168.0.1: 13211
192.168.0.3: 13211
top_blocked_domains:
type: "array"
items:
type: "object"
example:
example.org: 12312
example.com: 321
example.net: 5555
StatsHistory:
type: "object"
description: "Historical stats of the DNS server. Example below is for 5 minutes. Values are from oldest to newest."
required:
- "dns_queries"
- "blocked_filtering"
- "replaced_safebrowsing"
- "replaced_parental"
- "replaced_safesearch"
- "avg_processing_time"
properties:
dns_queries:
type: "array"
items:
type: "integer"
example:
- 1201
- 1501
- 1251
- 1231
- 120
blocked_filtering:
type: "array"
items:
type: "integer"
example:
- 421
- 124
- 5
- 12
- 43
replaced_safebrowsing:
type: "array"
items:
type: "integer"
example:
- 1
- 0
- 5
- 0
- 0
replaced_parental:
type: "array"
items:
type: "integer"
example:
- 120
- 10
- 5
- 12
- 1
replaced_safesearch:
type: "array"
items:
StatsConfig:
type: "object"
description: "Statistics configuration"
properties:
interval:
type: "integer"
example:
- 1
- 0
- 0
- 0
- 5
avg_processing_time:
type: "array"
items:
type: "number"
format: "float"
example:
- 1.25
- 5.12
- 4.12
- 123.12
- 0.12
description: "Time period to keep data (1 | 7 | 30 | 90)"
DhcpConfig:
type: "object"
description: "Built-in DHCP server configuration"

68
stats/stats.go Normal file
View File

@ -0,0 +1,68 @@
// Module for managing statistics for DNS filtering server
package stats
import (
"net"
)
type unitIDCallback func() int
// New - create object
// filename: DB file name
// limit: time limit (in days)
// unitID: user function to get the current unit ID. If nil, the current time hour is used.
func New(filename string, limit int, unitID unitIDCallback) Stats {
return createObject(filename, limit, unitID)
}
// Stats - main interface
type Stats interface {
// Close object.
// This function is not thread safe
// (can't be called in parallel with any other function of this interface).
Close()
// Set new configuration at runtime.
// limit: time limit (in days)
Configure(limit int)
// Reset counters and clear database
Clear()
// Update counters
Update(e Entry)
// Get data
GetData(timeUnit TimeUnit) map[string]interface{}
}
// TimeUnit - time unit
type TimeUnit int
// Supported time units
const (
Hours TimeUnit = iota
Days
)
// Result of DNS request processing
type Result int
// Supported result values
const (
RNotFiltered Result = iota + 1
RFiltered
RSafeBrowsing
RSafeSearch
RParental
rLast
)
// Entry - data to add
type Entry struct {
Domain string
Client net.IP
Result Result
Time uint // processing time (msec)
}

115
stats/stats_test.go Normal file
View File

@ -0,0 +1,115 @@
package stats
import (
"fmt"
"net"
"os"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func UIntArrayEquals(a []uint, b []uint) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func TestStats(t *testing.T) {
s := New("./stats.db", 1, nil)
e := Entry{}
e.Domain = "domain"
e.Client = net.ParseIP("127.0.0.1")
e.Result = RFiltered
e.Time = 123456
s.Update(e)
e.Domain = "domain"
e.Client = net.ParseIP("127.0.0.1")
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)
d := s.GetData(Hours)
a := []uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
assert.True(t, UIntArrayEquals(d["dns_queries"].([]uint), a))
a = []uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
assert.True(t, UIntArrayEquals(d["blocked_filtering"].([]uint), a))
a = []uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
assert.True(t, UIntArrayEquals(d["replaced_safebrowsing"].([]uint), a))
a = []uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
assert.True(t, UIntArrayEquals(d["replaced_parental"].([]uint), a))
m := d["top_queried_domains"].([]map[string]uint)
assert.True(t, m[0]["domain"] == 1)
m = d["top_blocked_domains"].([]map[string]uint)
assert.True(t, m[0]["domain"] == 1)
m = d["top_clients"].([]map[string]uint)
assert.True(t, m[0]["127.0.0.1"] == 2)
assert.True(t, d["num_dns_queries"].(uint) == 2)
assert.True(t, d["num_blocked_filtering"].(uint) == 1)
assert.True(t, d["num_replaced_safebrowsing"].(uint) == 0)
assert.True(t, d["num_replaced_safesearch"].(uint) == 0)
assert.True(t, d["num_replaced_parental"].(uint) == 0)
assert.True(t, d["avg_processing_time"].(float64) == 0.123456)
s.Clear()
s.Close()
os.Remove("./stats.db")
}
func TestLargeNumbers(t *testing.T) {
var hour int32
hour = 1
newID := func() int {
// use "atomic" to make Go race detector happy
return int(atomic.LoadInt32(&hour))
}
// log.SetLevel(log.DEBUG)
fn := "./stats.db"
os.Remove(fn)
s := New(fn, 1, newID)
e := Entry{}
n := 1000 // number of distinct clients and domains every hour
for h := 0; h != 12; h++ {
if h != 0 {
atomic.AddInt32(&hour, 1)
time.Sleep(1500 * time.Millisecond)
}
for i := 0; i != n; i++ {
e.Domain = fmt.Sprintf("domain%d", i)
e.Client = net.ParseIP("127.0.0.1")
e.Client[2] = byte((i & 0xff00) >> 8)
e.Client[3] = byte(i & 0xff)
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)
}
}
d := s.GetData(Hours)
assert.True(t, d["num_dns_queries"].(uint) == uint(int(hour)*n))
s.Close()
os.Remove(fn)
}

672
stats/stats_unit.go Normal file
View File

@ -0,0 +1,672 @@
package stats
import (
"bytes"
"encoding/binary"
"encoding/gob"
"fmt"
"os"
"sort"
"sync"
"time"
"github.com/AdguardTeam/golibs/log"
bolt "github.com/etcd-io/bbolt"
)
const (
maxDomains = 100 // max number of top domains to store in file or return via Get()
maxClients = 100 // max number of top clients to store in file or return via Get()
)
// statsCtx - global context
type statsCtx struct {
limit int // maximum time we need to keep data for (in hours)
filename string // database file name
unitID unitIDCallback // user function which returns the current unit ID
db *bolt.DB
unit *unit // the current unit
unitLock sync.Mutex // protect 'unit'
}
// data for 1 time unit
type unit struct {
id int // unit ID. Default: absolute hour since Jan 1, 1970
nTotal int // total requests
nResult []int // number of requests per one result
timeSum int // sum of processing time of all requests (usec)
// top:
domains map[string]int // number of requests per domain
blockedDomains map[string]int // number of blocked requests per domain
clients map[string]int // number of requests per client
}
// name-count pair
type countPair struct {
Name string
Count uint
}
// structure for storing data in file
type unitDB struct {
NTotal uint
NResult []uint
Domains []countPair
BlockedDomains []countPair
Clients []countPair
TimeAvg uint // usec
}
func createObject(filename string, limitDays int, unitID unitIDCallback) *statsCtx {
s := statsCtx{}
s.limit = limitDays * 24
s.filename = filename
s.unitID = newUnitID
if unitID != nil {
s.unitID = unitID
}
if !s.dbOpen() {
return nil
}
id := s.unitID()
tx := s.beginTxn(true)
var udb *unitDB
if tx != nil {
log.Tracef("Deleting old units...")
firstID := id - s.limit - 1
unitDel := 0
forEachBkt := func(name []byte, b *bolt.Bucket) error {
id := btoi(name)
if id < firstID {
err := tx.DeleteBucket(name)
if err != nil {
log.Debug("tx.DeleteBucket: %s", err)
}
log.Debug("Stats: deleted unit %d", id)
unitDel++
return nil
}
return fmt.Errorf("")
}
_ = tx.ForEach(forEachBkt)
udb = s.loadUnitFromDB(tx, id)
if unitDel != 0 {
s.commitTxn(tx)
} else {
_ = tx.Rollback()
}
}
u := unit{}
s.initUnit(&u, id)
if udb != nil {
deserialize(&u, udb)
}
s.unit = &u
go s.periodicFlush()
log.Debug("Stats: initialized")
return &s
}
func (s *statsCtx) dbOpen() bool {
var err error
log.Tracef("db.Open...")
s.db, err = bolt.Open(s.filename, 0644, nil)
if err != nil {
log.Error("Stats: open DB: %s: %s", s.filename, err)
return false
}
log.Tracef("db.Open")
return true
}
// Atomically swap the currently active unit with a new value
// Return old value
func (s *statsCtx) swapUnit(new *unit) *unit {
s.unitLock.Lock()
u := s.unit
s.unit = new
s.unitLock.Unlock()
return u
}
// Get unit ID for the current hour
func newUnitID() int {
return int(time.Now().Unix() / (60 * 60))
}
// Initialize a unit
func (s *statsCtx) initUnit(u *unit, id int) {
u.id = id
u.nResult = make([]int, rLast)
u.domains = make(map[string]int)
u.blockedDomains = make(map[string]int)
u.clients = make(map[string]int)
}
// Open a DB transaction
func (s *statsCtx) beginTxn(wr bool) *bolt.Tx {
db := s.db
if db == nil {
return nil
}
log.Tracef("db.Begin...")
tx, err := db.Begin(wr)
if err != nil {
log.Error("db.Begin: %s", err)
return nil
}
log.Tracef("db.Begin")
return tx
}
func (s *statsCtx) commitTxn(tx *bolt.Tx) {
err := tx.Commit()
if err != nil {
log.Debug("tx.Commit: %s", err)
return
}
log.Tracef("tx.Commit")
}
// Get unit name
func unitName(id int) []byte {
return itob(id)
}
// Convert integer to 8-byte array (big endian)
func itob(v int) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(v))
return b
}
// Convert 8-byte array (big endian) to integer
func btoi(b []byte) int {
return int(binary.BigEndian.Uint64(b))
}
// Flush the current unit to DB and delete an old unit when a new hour is started
func (s *statsCtx) periodicFlush() {
for {
s.unitLock.Lock()
ptr := s.unit
s.unitLock.Unlock()
if ptr == nil {
break
}
id := s.unitID()
if ptr.id == id {
time.Sleep(time.Second)
continue
}
nu := unit{}
s.initUnit(&nu, id)
u := s.swapUnit(&nu)
udb := serialize(u)
tx := s.beginTxn(true)
if tx == nil {
continue
}
ok1 := s.flushUnitToDB(tx, u.id, udb)
ok2 := s.deleteUnit(tx, id-s.limit)
if ok1 || ok2 {
s.commitTxn(tx)
} else {
_ = tx.Rollback()
}
}
log.Tracef("periodicFlush() exited")
}
// Delete unit's data from file
func (s *statsCtx) deleteUnit(tx *bolt.Tx, id int) bool {
err := tx.DeleteBucket(unitName(id))
if err != nil {
log.Tracef("bolt DeleteBucket: %s", err)
return false
}
log.Debug("Stats: deleted unit %d", id)
return true
}
func convertMapToArray(m map[string]int, max int) []countPair {
a := []countPair{}
for k, v := range m {
pair := countPair{}
pair.Name = k
pair.Count = uint(v)
a = append(a, pair)
}
less := func(i, j int) bool {
if a[i].Count >= a[j].Count {
return true
}
return false
}
sort.Slice(a, less)
if max > len(a) {
max = len(a)
}
return a[:max]
}
func convertArrayToMap(a []countPair) map[string]int {
m := map[string]int{}
for _, it := range a {
m[it.Name] = int(it.Count)
}
return m
}
func serialize(u *unit) *unitDB {
udb := unitDB{}
udb.NTotal = uint(u.nTotal)
for _, it := range u.nResult {
udb.NResult = append(udb.NResult, uint(it))
}
if u.nTotal != 0 {
udb.TimeAvg = uint(u.timeSum / u.nTotal)
}
udb.Domains = convertMapToArray(u.domains, maxDomains)
udb.BlockedDomains = convertMapToArray(u.blockedDomains, maxDomains)
udb.Clients = convertMapToArray(u.clients, maxClients)
return &udb
}
func deserialize(u *unit, udb *unitDB) {
u.nTotal = int(udb.NTotal)
for _, it := range udb.NResult {
u.nResult = append(u.nResult, int(it))
}
u.domains = convertArrayToMap(udb.Domains)
u.blockedDomains = convertArrayToMap(udb.BlockedDomains)
u.clients = convertArrayToMap(udb.Clients)
u.timeSum = int(udb.TimeAvg) * u.nTotal
}
func (s *statsCtx) flushUnitToDB(tx *bolt.Tx, id int, udb *unitDB) bool {
log.Tracef("Flushing unit %d", id)
bkt, err := tx.CreateBucketIfNotExists(unitName(id))
if err != nil {
log.Error("tx.CreateBucketIfNotExists: %s", err)
return false
}
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err = enc.Encode(udb)
if err != nil {
log.Error("gob.Encode: %s", err)
return false
}
err = bkt.Put([]byte{0}, buf.Bytes())
if err != nil {
log.Error("bkt.Put: %s", err)
return false
}
return true
}
func (s *statsCtx) loadUnitFromDB(tx *bolt.Tx, id int) *unitDB {
bkt := tx.Bucket(unitName(id))
if bkt == nil {
return nil
}
log.Tracef("Loading unit %d", id)
var buf bytes.Buffer
buf.Write(bkt.Get([]byte{0}))
dec := gob.NewDecoder(&buf)
udb := unitDB{}
err := dec.Decode(&udb)
if err != nil {
log.Error("gob Decode: %s", err)
return nil
}
return &udb
}
func convertTopArray(a []countPair) []map[string]uint {
m := []map[string]uint{}
for _, it := range a {
ent := map[string]uint{}
ent[it.Name] = it.Count
m = append(m, ent)
}
return m
}
func (s *statsCtx) Configure(limit int) {
if limit < 0 {
return
}
s.limit = limit * 24
log.Debug("Stats: set limit: %d", limit)
}
func (s *statsCtx) Close() {
u := s.swapUnit(nil)
udb := serialize(u)
tx := s.beginTxn(true)
if tx != nil {
if s.flushUnitToDB(tx, u.id, udb) {
s.commitTxn(tx)
} else {
_ = tx.Rollback()
}
}
if s.db != nil {
log.Tracef("db.Close...")
_ = s.db.Close()
log.Tracef("db.Close")
}
log.Debug("Stats: closed")
}
func (s *statsCtx) Clear() {
tx := s.beginTxn(true)
if tx != nil {
db := s.db
s.db = nil
_ = tx.Rollback()
// the active transactions can continue using database,
// but no new transactions will be opened
_ = db.Close()
log.Tracef("db.Close")
// all active transactions are now closed
}
u := unit{}
s.initUnit(&u, s.unitID())
_ = s.swapUnit(&u)
err := os.Remove(s.filename)
if err != nil {
log.Error("os.Remove: %s", err)
}
_ = s.dbOpen()
log.Debug("Stats: cleared")
}
func (s *statsCtx) Update(e Entry) {
if e.Result == 0 ||
len(e.Domain) == 0 ||
!(len(e.Client) == 4 || len(e.Client) == 16) {
return
}
client := e.Client.String()
s.unitLock.Lock()
u := s.unit
u.nResult[e.Result]++
if e.Result == RNotFiltered {
u.domains[e.Domain]++
} else {
u.blockedDomains[e.Domain]++
}
u.clients[client]++
u.timeSum += int(e.Time)
u.nTotal++
s.unitLock.Unlock()
}
/* Algorithm:
. Prepare array of N units, where N is the value of "limit" configuration setting
. Load data for the most recent units from file
If a unit with required ID doesn't exist, just add an empty unit
. Get data for the current unit
. Process data from the units and prepare an output map object:
* per time unit counters:
* DNS-queries/time-unit
* blocked/time-unit
* safebrowsing-blocked/time-unit
* parental-blocked/time-unit
If time-unit is an hour, just add values from each unit to an array.
If time-unit is a day, aggregate per-hour data into days.
* top counters:
* queries/domain
* queries/blocked-domain
* queries/client
To get these values we first sum up data for all units into a single map.
Then we get the pairs with the highest numbers (the values are sorted in descending order)
* total counters:
* DNS-queries
* blocked
* safebrowsing-blocked
* safesearch-blocked
* parental-blocked
These values are just the sum of data for all units.
*/
// nolint (gocyclo)
func (s *statsCtx) GetData(timeUnit TimeUnit) map[string]interface{} {
d := map[string]interface{}{}
tx := s.beginTxn(false)
if tx == nil {
return nil
}
units := []*unitDB{} //per-hour units
lastID := s.unitID()
firstID := lastID - s.limit + 1
for i := firstID; i != lastID; i++ {
u := s.loadUnitFromDB(tx, i)
if u == nil {
u = &unitDB{}
u.NResult = make([]uint, rLast)
}
units = append(units, u)
}
_ = tx.Rollback()
s.unitLock.Lock()
cu := serialize(s.unit)
cuID := s.unit.id
s.unitLock.Unlock()
if cuID != lastID {
units = units[1:]
}
units = append(units, cu)
if len(units) != s.limit {
log.Fatalf("len(units) != s.limit: %d %d", len(units), s.limit)
}
// per time unit counters:
// 720 hours may span 31 days, so we skip data for the first day in this case
firstDayID := (firstID + 24 - 1) / 24 * 24 // align_ceil(24)
a := []uint{}
if timeUnit == Hours {
for _, u := range units {
a = append(a, u.NTotal)
}
} else {
var sum uint
id := firstDayID
nextDayID := firstDayID + 24
for i := firstDayID - firstID; i != len(units); i++ {
sum += units[i].NTotal
if id == nextDayID {
a = append(a, sum)
sum = 0
nextDayID += 24
}
id++
}
if id < nextDayID {
a = append(a, sum)
}
if len(a) != s.limit/24 {
log.Fatalf("len(a) != s.limit: %d %d", len(a), s.limit)
}
}
d["dns_queries"] = a
a = []uint{}
if timeUnit == Hours {
for _, u := range units {
a = append(a, u.NResult[RFiltered])
}
} else {
var sum uint
id := firstDayID
nextDayID := firstDayID + 24
for i := firstDayID - firstID; i != len(units); i++ {
sum += units[i].NResult[RFiltered]
if id == nextDayID {
a = append(a, sum)
sum = 0
nextDayID += 24
}
id++
}
if id < nextDayID {
a = append(a, sum)
}
}
d["blocked_filtering"] = a
a = []uint{}
if timeUnit == Hours {
for _, u := range units {
a = append(a, u.NResult[RSafeBrowsing])
}
} else {
var sum uint
id := firstDayID
nextDayID := firstDayID + 24
for i := firstDayID - firstID; i != len(units); i++ {
sum += units[i].NResult[RSafeBrowsing]
if id == nextDayID {
a = append(a, sum)
sum = 0
nextDayID += 24
}
id++
}
if id < nextDayID {
a = append(a, sum)
}
}
d["replaced_safebrowsing"] = a
a = []uint{}
if timeUnit == Hours {
for _, u := range units {
a = append(a, u.NResult[RParental])
}
} else {
var sum uint
id := firstDayID
nextDayID := firstDayID + 24
for i := firstDayID - firstID; i != len(units); i++ {
sum += units[i].NResult[RParental]
if id == nextDayID {
a = append(a, sum)
sum = 0
nextDayID += 24
}
id++
}
if id < nextDayID {
a = append(a, sum)
}
}
d["replaced_parental"] = a
// top counters:
m := map[string]int{}
for _, u := range units {
for _, it := range u.Domains {
m[it.Name] += int(it.Count)
}
}
a2 := convertMapToArray(m, maxDomains)
d["top_queried_domains"] = convertTopArray(a2)
m = map[string]int{}
for _, u := range units {
for _, it := range u.BlockedDomains {
m[it.Name] += int(it.Count)
}
}
a2 = convertMapToArray(m, maxDomains)
d["top_blocked_domains"] = convertTopArray(a2)
m = map[string]int{}
for _, u := range units {
for _, it := range u.Clients {
m[it.Name] += int(it.Count)
}
}
a2 = convertMapToArray(m, maxClients)
d["top_clients"] = convertTopArray(a2)
// total counters:
sum := unitDB{}
timeN := 0
sum.NResult = make([]uint, rLast)
for _, u := range units {
sum.NTotal += u.NTotal
sum.TimeAvg += u.TimeAvg
if u.TimeAvg != 0 {
timeN++
}
sum.NResult[RFiltered] += u.NResult[RFiltered]
sum.NResult[RSafeBrowsing] += u.NResult[RSafeBrowsing]
sum.NResult[RSafeSearch] += u.NResult[RSafeSearch]
sum.NResult[RParental] += u.NResult[RParental]
}
d["num_dns_queries"] = sum.NTotal
d["num_blocked_filtering"] = sum.NResult[RFiltered]
d["num_replaced_safebrowsing"] = sum.NResult[RSafeBrowsing]
d["num_replaced_safesearch"] = sum.NResult[RSafeSearch]
d["num_replaced_parental"] = sum.NResult[RParental]
avgTime := float64(0)
if timeN != 0 {
avgTime = float64(sum.TimeAvg/uint(timeN)) / 1000000
}
d["avg_processing_time"] = avgTime
d["time_units"] = "hours"
if timeUnit == Days {
d["time_units"] = "days"
}
return d
}