+ client: server side pagination for the query logs

2019-09-20 15:05:10 +03:00 · 2019-09-20 15:05:10 +03:00 · 9ac6049405
parent 81828c87c1
commit 9ac6049405
12 changed files with 323 additions and 99 deletions
--- a/client/src/__locales/en.json
+++ b/client/src/__locales/en.json
@ -163,7 +163,6 @@
    "show_filtered_type": "Show filtered",
    "no_logs_found": "No logs found",
    "refresh_btn": "Refresh",
-    "last_dns_queries": "Last 5000 DNS queries",
    "previous_btn": "Previous",
    "next_btn": "Next",
    "loading_table_status": "Loading...",
@ -182,6 +181,7 @@
    "query_log_enable": "Enable log",
    "query_log_configuration": "Logs configuration",
    "query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
+    "query_log_strict_search": "Use double quotes for strict search",
    "source_label": "Source",
    "found_in_known_domain_db": "Found in the known domains database.",
    "category_label": "Category",
--- a/client/src/actions/queryLogs.js
+++ b/client/src/actions/queryLogs.js
@ -4,15 +4,18 @@ import apiClient from '../api/Api';
 import { addErrorToast, addSuccessToast } from './index';
 import { normalizeLogs } from '../helpers/helpers';

+export const setLogsPagination = createAction('LOGS_PAGINATION');
+
 export const getLogsRequest = createAction('GET_LOGS_REQUEST');
 export const getLogsFailure = createAction('GET_LOGS_FAILURE');
 export const getLogsSuccess = createAction('GET_LOGS_SUCCESS');

-export const getLogs = () => async (dispatch) => {
+export const getLogs = config => async (dispatch) => {
    dispatch(getLogsRequest());
    try {
-        const logs = normalizeLogs(await apiClient.getQueryLog());
-        dispatch(getLogsSuccess(logs));
+        const { filter, lastRowTime: older_than } = config;
+        const logs = normalizeLogs(await apiClient.getQueryLog({ filter, older_than }));
+        dispatch(getLogsSuccess({ logs, ...config }));
    } catch (error) {
        dispatch(addErrorToast({ error }));
        dispatch(getLogsFailure(error));
--- a/client/src/api/Api.js
+++ b/client/src/api/Api.js
@ -482,14 +482,18 @@ class Api {
    }

    // Query log
-    GET_QUERY_LOG = { path: 'querylog', method: 'GET' };
+    GET_QUERY_LOG = { path: 'querylog', method: 'POST' };
    QUERY_LOG_CONFIG = { path: 'querylog_config', method: 'POST' };
    QUERY_LOG_INFO = { path: 'querylog_info', method: 'GET' };
    QUERY_LOG_CLEAR = { path: 'querylog_clear', method: 'POST' };

-    getQueryLog() {
+    getQueryLog(data) {
        const { path, method } = this.GET_QUERY_LOG;
-        return this.makeRequest(path, method);
+        const config = {
+            data,
+            headers: { 'Content-Type': 'application/json' },
+        };
+        return this.makeRequest(path, method, config);
    }

    getQueryLogInfo() {
--- a/client/src/components/Logs/Logs.css
+++ b/client/src/components/Logs/Logs.css
@ -107,6 +107,11 @@
    border: 1px solid rgba(0, 40, 100, 0.12);
 }

+.logs__table .rt-thead.-filters select {
+    background: #fff url("data:image/svg+xml;base64,PHN2ZyB4bWxucz0naHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmcnIHZpZXdCb3g9JzAgMCAxMCA1Jz48cGF0aCBmaWxsPScjOTk5JyBkPSdNMCAwTDEwIDBMNSA1TDAgMCcvPjwvc3ZnPg==") no-repeat right 0.75rem center;
+    background-size: 8px 10px;
+}
+
 .logs__table .rt-thead.-filters input:focus,
 .logs__table .rt-thead.-filters select:focus {
    border-color: #1991eb;
@ -130,6 +135,21 @@
    overflow: hidden;
 }

+.logs__input-wrap {
+    position: relative;
+}
+
+.logs__notice {
+    position: absolute;
+    z-index: 1;
+    top: 8px;
+    right: 10px;
+    margin-top: 3px;
+    font-size: 12px;
+    text-align: left;
+    color: #a5a5a5;
+}
+
 .logs__whois {
    display: inline;
 }
--- a/client/src/components/Logs/index.js
+++ b/client/src/components/Logs/index.js
@ -5,9 +5,14 @@ import escapeRegExp from 'lodash/escapeRegExp';
 import endsWith from 'lodash/endsWith';
 import { Trans, withNamespaces } from 'react-i18next';
 import { HashLink as Link } from 'react-router-hash-link';
+import debounce from 'lodash/debounce';

-import { formatTime, formatDateTime } from '../../helpers/helpers';
-import { SERVICES, FILTERED_STATUS } from '../../helpers/constants';
+import {
+    formatTime,
+    formatDateTime,
+    isValidQuestionType,
+} from '../../helpers/helpers';
+import { SERVICES, FILTERED_STATUS, DEBOUNCE_TIMEOUT } from '../../helpers/constants';
 import { getTrackerData } from '../../helpers/trackers/trackers';
 import { formatClientCell } from '../../helpers/formatClientCell';

@ -16,8 +21,12 @@ import Card from '../ui/Card';
 import Loading from '../ui/Loading';
 import PopoverFiltered from '../ui/PopoverFilter';
 import Popover from '../ui/Popover';
+import Tooltip from '../ui/Tooltip';
 import './Logs.css';

+const TABLE_FIRST_PAGE = 0;
+const TABLE_DEFAULT_PAGE_SIZE = 50;
+const INITIAL_REQUEST_DATA = ['', {}, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE];
 const FILTERED_REASON = 'Filtered';
 const RESPONSE_FILTER = {
    ALL: 'all',
@ -26,26 +35,36 @@ const RESPONSE_FILTER = {

 class Logs extends Component {
    componentDidMount() {
-        this.getLogs();
+        this.getLogs(...INITIAL_REQUEST_DATA);
        this.props.getFilteringStatus();
        this.props.getClients();
        this.props.getLogsConfig();
    }

-    componentDidUpdate(prevProps) {
-        // get logs when queryLog becomes enabled
-        if (this.props.queryLogs.enabled && !prevProps.queryLogs.enabled) {
-            this.props.getLogs();
-        }
-    }
-
-    getLogs = () => {
-        // get logs on initialization if queryLogIsEnabled
+    getLogs = (lastRowTime, filter, page, pageSize) => {
        if (this.props.queryLogs.enabled) {
-            this.props.getLogs();
+            this.props.getLogs({
+                lastRowTime, filter, page, pageSize,
+            });
        }
    };

+    refreshLogs = (lastRowTime, filter, page, pageSize, refreshLogs = true) => {
+        this.props.getLogs({
+            lastRowTime, filter, page, pageSize, refreshLogs,
+        });
+    };
+
+    handleLogsFiltering = debounce((lastRowTime, filter, page, pageSize, filtered) => {
+        this.props.getLogs({
+            lastRowTime,
+            filter,
+            page,
+            pageSize,
+            filtered,
+        });
+    }, DEBOUNCE_TIMEOUT);
+
    renderTooltip = (isFiltered, rule, filter, service) =>
        isFiltered && <PopoverFiltered rule={rule} filter={filter} service={service} />;

@ -215,8 +234,64 @@ class Logs extends Component {
        );
    };

-    renderLogs(logs) {
-        const { t } = this.props;
+    getFilterInput = ({ filter, onChange }) => (
+        <Fragment>
+            <div className="logs__input-wrap">
+                <input
+                    type="text"
+                    className="form-control"
+                    onChange={event => onChange(event.target.value)}
+                    value={filter ? filter.value : ''}
+                />
+                <span className="logs__notice">
+                    <Tooltip text={this.props.t('query_log_strict_search')} type='tooltip-custom--logs' />
+                </span>
+            </div>
+        </Fragment>
+    );
+
+    getFilters = (filtered) => {
+        const filteredObj = filtered.reduce((acc, cur) => ({ ...acc, [cur.id]: cur.value }), {});
+        const {
+            domain, client, type, response,
+        } = filteredObj;
+
+        return {
+            domain: domain || '',
+            client: client || '',
+            question_type: isValidQuestionType(type) ? type.toUpperCase() : '',
+            response_status: response === RESPONSE_FILTER.FILTERED ? response : '',
+        };
+    };
+
+    fetchData = (state) => {
+        const {
+            filtered, pageSize, page, pages,
+        } = state;
+        const { allLogs } = this.props.queryLogs;
+        const isLastPage = pages && (page + 1 >= pages);
+        const isFiltersPresent = filtered.length > 0;
+        const filter = this.getFilters(filtered);
+
+        if (isFiltersPresent) {
+            this.handleLogsFiltering('', filter, page, pageSize, true);
+        } else if (isLastPage) {
+            const lastRow = allLogs[allLogs.length - 1];
+            const lastRowTime = (lastRow && lastRow.time) || '';
+            this.getLogs(lastRowTime, filter, page, pageSize);
+        } else {
+            this.props.setLogsPagination({ page, pageSize });
+        }
+    };
+
+    renderLogs() {
+        const { queryLogs, dashboard, t } = this.props;
+        const { processingClients } = dashboard;
+        const {
+            processingGetLogs, processingGetConfig, logs, pages,
+        } = queryLogs;
+        const isLoading = processingGetLogs || processingClients || processingGetConfig;
+
        const columns = [
            {
                Header: t('time_table_header'),
@ -230,6 +305,7 @@ class Logs extends Component {
                accessor: 'domain',
                minWidth: 180,
                Cell: this.getDomainCell,
+                Filter: this.getFilterInput,
            },
            {
                Header: t('type_table_header'),
@ -251,7 +327,7 @@ class Logs extends Component {
                },
                Filter: ({ filter, onChange }) => (
                    <select
-                        className="form-control"
+                        className="form-control custom-select"
                        onChange={event => onChange(event.target.value)}
                        value={filter ? filter.value : RESPONSE_FILTER.ALL}
                    >
@ -270,19 +346,26 @@ class Logs extends Component {
                maxWidth: 240,
                minWidth: 240,
                Cell: this.getClientCell,
+                Filter: this.getFilterInput,
            },
        ];

-        if (logs) {
        return (
            <ReactTable
-                    className="logs__table"
+                manual
                filterable
-                    data={logs}
+                minRows={5}
+                pages={pages}
                columns={columns}
+                sortable={false}
+                data={logs || []}
+                loading={isLoading}
+                showPageJump={false}
+                onFetchData={this.fetchData}
+                onPageChange={page => this.setState({ page })}
+                className="logs__table"
                showPagination={true}
-                    defaultPageSize={50}
-                    minRows={7}
+                defaultPageSize={TABLE_DEFAULT_PAGE_SIZE}
                previousText={t('previous_btn')}
                nextText={t('next_btn')}
                loadingText={t('loading_table_status')}
@ -331,21 +414,15 @@ class Logs extends Component {
        );
    }

-        return null;
-    }
-
    render() {
-        const { queryLogs, dashboard, t } = this.props;
-        const { enabled, processingGetLogs, processingGetConfig } = queryLogs;
-        const { processingClients } = dashboard;
-        const isDataReady =
-            !processingGetLogs && !processingGetConfig && !dashboard.processingClients;
+        const { queryLogs, t } = this.props;
+        const { enabled, processingGetConfig } = queryLogs;

        const refreshButton = enabled ? (
            <button
+                type="button"
                className="btn btn-icon btn-outline-primary btn-sm ml-3"
-                type="submit"
-                onClick={this.getLogs}
+                onClick={() => this.refreshLogs(...INITIAL_REQUEST_DATA)}
            >
                <svg className="icons">
                    <use xlinkHref="#refresh" />
@ -357,22 +434,24 @@ class Logs extends Component {

        return (
            <Fragment>
-                <PageTitle title={t('query_log')} subtitle={t('last_dns_queries')}>
-                    {refreshButton}
-                </PageTitle>
-                <Card>
-                    {enabled && (processingGetLogs || processingClients || processingGetConfig) && (
-                        <Loading />
-                    )}
-                    {enabled && isDataReady && this.renderLogs(queryLogs.logs)}
+                <PageTitle title={t('query_log')}>{refreshButton}</PageTitle>
+                {enabled && processingGetConfig && <Loading />}
+                {enabled && !processingGetConfig && <Card>{this.renderLogs()}</Card>}
                {!enabled && !processingGetConfig && (
+                    <Card>
                        <div className="lead text-center py-6">
-                            <Trans components={[<Link to="/settings#logs-config" key="0">link</Link>]}>
+                            <Trans
+                                components={[
+                                    <Link to="/settings#logs-config" key="0">
+                                        link
+                                    </Link>,
+                                ]}
+                            >
                                query_log_disabled
                            </Trans>
                        </div>
-                    )}
                    </Card>
+                )}
            </Fragment>
        );
    }
@ -388,6 +467,7 @@ Logs.propTypes = {
    addSuccessToast: PropTypes.func.isRequired,
    getClients: PropTypes.func.isRequired,
    getLogsConfig: PropTypes.func.isRequired,
+    setLogsPagination: PropTypes.func.isRequired,
    t: PropTypes.func.isRequired,
 };

--- a/client/src/components/ui/Popover.css
+++ b/client/src/components/ui/Popover.css
@ -38,6 +38,7 @@
    position: absolute;
    bottom: calc(100% + 3px);
    left: 50%;
+    z-index: 1;
    min-width: 275px;
    padding: 10px 15px;
    font-size: 0.8rem;
--- a/client/src/components/ui/Tooltip.css
+++ b/client/src/components/ui/Tooltip.css
@ -52,3 +52,23 @@
 .tooltip-custom--narrow:before {
    width: 220px;
 }
+
+.tooltip-custom--logs {
+    border-radius: 50%;
+    background-image: url("./svg/help-circle-gray.svg");
+    background-color: #fff;
+}
+
+.tooltip-custom--logs:before {
+    bottom: initial;
+    top: calc(100% + 10px);
+    right: -10px;
+    left: initial;
+    transform: none;
+}
+
+.tooltip-custom--logs:after {
+    top: 8px;
+    border-top: none;
+    border-bottom: 6px solid #585965;
+}
--- a/client/src/components/ui/svg/help-circle-gray.svg
+++ b/client/src/components/ui/svg/help-circle-gray.svg
@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#9aa0ac" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-help-circle"><circle cx="12" cy="12" r="10"></circle><path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3"></path><line x1="12" y1="17" x2="12" y2="17"></line></svg>
--- a/client/src/containers/Logs.js
+++ b/client/src/containers/Logs.js
@ -1,7 +1,7 @@
 import { connect } from 'react-redux';
 import { addSuccessToast, getClients } from '../actions';
 import { getFilteringStatus, setRules } from '../actions/filtering';
-import { getLogs, getLogsConfig } from '../actions/queryLogs';
+import { getLogs, getLogsConfig, setLogsPagination } from '../actions/queryLogs';
 import Logs from '../components/Logs';

 const mapStateToProps = (state) => {
@ -17,6 +17,7 @@ const mapDispatchToProps = {
    addSuccessToast,
    getClients,
    getLogsConfig,
+    setLogsPagination,
 };

 export default connect(
--- a/client/src/helpers/constants.js
+++ b/client/src/helpers/constants.js
@ -274,3 +274,47 @@ export const WHOIS_ICONS = {
    netname: 'network',
    descr: '',
 };
+
+export const DNS_RECORD_TYPES = [
+    'A',
+    'AAAA',
+    'AFSDB',
+    'APL',
+    'CAA',
+    'CDNSKEY',
+    'CDS',
+    'CERT',
+    'CNAME',
+    'CSYNC',
+    'DHCID',
+    'DLV',
+    'DNAME',
+    'DNSKEY',
+    'DS',
+    'HIP',
+    'IPSECKEY',
+    'KEY',
+    'KX',
+    'LOC',
+    'MX',
+    'NAPTR',
+    'NS',
+    'NSEC',
+    'NSEC3',
+    'NSEC3PARAM',
+    'OPENPGPKEY',
+    'PTR',
+    'RRSIG',
+    'RP',
+    'SIG',
+    'SMIMEA',
+    'SOA',
+    'SRV',
+    'SSHFP',
+    'TA',
+    'TKEY',
+    'TLSA',
+    'TSIG',
+    'TXT',
+    'URI',
+];
--- a/client/src/helpers/helpers.js
+++ b/client/src/helpers/helpers.js
@ -14,6 +14,7 @@ import {
    STANDARD_WEB_PORT,
    STANDARD_HTTPS_PORT,
    CHECK_TIMEOUT,
+    DNS_RECORD_TYPES,
 } from './constants';

 export const formatTime = (time) => {
@ -318,3 +319,5 @@ export const normalizeWhois = (whois) => {

    return whois;
 };
+
+export const isValidQuestionType = type => type && DNS_RECORD_TYPES.includes(type.toUpperCase());
--- a/client/src/reducers/queryLogs.js
+++ b/client/src/reducers/queryLogs.js
@ -4,11 +4,54 @@ import * as actions from '../actions/queryLogs';

 const queryLogs = handleActions(
    {
+        [actions.setLogsPagination]: (state, { payload }) => {
+            const { page, pageSize } = payload;
+            const { allLogs } = state;
+            const rowsStart = pageSize * page;
+            const rowsEnd = (pageSize * page) + pageSize;
+            const logsSlice = allLogs.slice(rowsStart, rowsEnd);
+            const pages = Math.ceil(allLogs.length / pageSize);
+
+            return {
+                ...state,
+                pages,
+                logs: logsSlice,
+            };
+        },
+
        [actions.getLogsRequest]: state => ({ ...state, processingGetLogs: true }),
        [actions.getLogsFailure]: state => ({ ...state, processingGetLogs: false }),
        [actions.getLogsSuccess]: (state, { payload }) => {
-            const newState = { ...state, logs: payload, processingGetLogs: false };
-            return newState;
+            const {
+                logs, lastRowTime, page, pageSize, filtered, refreshLogs,
+            } = payload;
+            let logsWithOffset = state.allLogs.length > 0 ? state.allLogs : logs;
+            let allLogs = logs;
+
+            if (lastRowTime) {
+                logsWithOffset = [...state.allLogs, ...logs];
+                allLogs = [...state.allLogs, ...logs];
+            }
+
+            if (filtered || refreshLogs) {
+                logsWithOffset = logs;
+                allLogs = logs;
+            }
+
+            const pages = Math.ceil(logsWithOffset.length / pageSize);
+            const total = logsWithOffset.length;
+            const rowsStart = pageSize * page;
+            const rowsEnd = (pageSize * page) + pageSize;
+            const logsSlice = logsWithOffset.slice(rowsStart, rowsEnd);
+
+            return {
+                ...state,
+                pages,
+                total,
+                allLogs,
+                logs: logsSlice,
+                processingGetLogs: false,
+            };
        },

        [actions.clearLogsRequest]: state => ({ ...state, processingClear: true }),
@ -42,6 +85,10 @@ const queryLogs = handleActions(
        processingSetConfig: false,
        logs: [],
        interval: 1,
+        allLogs: [],
+        pages: 0,
+        offset: 0,
+        total: 0,
        enabled: true,
    },
 );
				`@ -0,0 +1 @@`
				`<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#9aa0ac" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-help-circle"><circle cx="12" cy="12" r="10"></circle><path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3"></path><line x1="12" y1="17" x2="12" y2="17"></line></svg>`