+ client: server side pagination for the query logs

client/src/__locales/en.json

@@ -163,7 +163,6 @@
     "show_filtered_type": "Show filtered",
     "no_logs_found": "No logs found",
     "refresh_btn": "Refresh",
-    "last_dns_queries": "Last 5000 DNS queries",
     "previous_btn": "Previous",
     "next_btn": "Next",
     "loading_table_status": "Loading...",
@@ -182,6 +181,7 @@
     "query_log_enable": "Enable log",
     "query_log_configuration": "Logs configuration",
     "query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
+    "query_log_strict_search": "Use double quotes for strict search",
     "source_label": "Source",
     "found_in_known_domain_db": "Found in the known domains database.",
     "category_label": "Category",

client/src/actions/queryLogs.js

@@ -4,15 +4,18 @@ import apiClient from '../api/Api';
 import { addErrorToast, addSuccessToast } from './index';
 import { normalizeLogs } from '../helpers/helpers';
 
+export const setLogsPagination = createAction('LOGS_PAGINATION');
+
 export const getLogsRequest = createAction('GET_LOGS_REQUEST');
 export const getLogsFailure = createAction('GET_LOGS_FAILURE');
 export const getLogsSuccess = createAction('GET_LOGS_SUCCESS');
 
-export const getLogs = () => async (dispatch) => {
+export const getLogs = config => async (dispatch) => {
     dispatch(getLogsRequest());
     try {
-        const logs = normalizeLogs(await apiClient.getQueryLog());
+        const { filter, lastRowTime: older_than } = config;
-        dispatch(getLogsSuccess(logs));
+        const logs = normalizeLogs(await apiClient.getQueryLog({ filter, older_than }));
+        dispatch(getLogsSuccess({ logs, ...config }));
     } catch (error) {
         dispatch(addErrorToast({ error }));
         dispatch(getLogsFailure(error));

client/src/api/Api.js

@@ -482,14 +482,18 @@ class Api {
     }
 
     // Query log
-    GET_QUERY_LOG = { path: 'querylog', method: 'GET' };
+    GET_QUERY_LOG = { path: 'querylog', method: 'POST' };
     QUERY_LOG_CONFIG = { path: 'querylog_config', method: 'POST' };
     QUERY_LOG_INFO = { path: 'querylog_info', method: 'GET' };
     QUERY_LOG_CLEAR = { path: 'querylog_clear', method: 'POST' };
 
-    getQueryLog() {
+    getQueryLog(data) {
         const { path, method } = this.GET_QUERY_LOG;
-        return this.makeRequest(path, method);
+        const config = {
+            data,
+            headers: { 'Content-Type': 'application/json' },
+        };
+        return this.makeRequest(path, method, config);
     }
 
     getQueryLogInfo() {

client/src/components/Logs/Logs.css

@@ -107,6 +107,11 @@
     border: 1px solid rgba(0, 40, 100, 0.12);
 }
 
+.logs__table .rt-thead.-filters select {
+    background: #fff url("data:image/svg+xml;base64,PHN2ZyB4bWxucz0naHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmcnIHZpZXdCb3g9JzAgMCAxMCA1Jz48cGF0aCBmaWxsPScjOTk5JyBkPSdNMCAwTDEwIDBMNSA1TDAgMCcvPjwvc3ZnPg==") no-repeat right 0.75rem center;
+    background-size: 8px 10px;
+}
+
 .logs__table .rt-thead.-filters input:focus,
 .logs__table .rt-thead.-filters select:focus {
     border-color: #1991eb;
@@ -130,6 +135,21 @@
     overflow: hidden;
 }
 
+.logs__input-wrap {
+    position: relative;
+}
+
+.logs__notice {
+    position: absolute;
+    z-index: 1;
+    top: 8px;
+    right: 10px;
+    margin-top: 3px;
+    font-size: 12px;
+    text-align: left;
+    color: #a5a5a5;
+}
+
 .logs__whois {
     display: inline;
 }

client/src/components/Logs/index.js

@@ -5,9 +5,14 @@ import escapeRegExp from 'lodash/escapeRegExp';
 import endsWith from 'lodash/endsWith';
 import { Trans, withNamespaces } from 'react-i18next';
 import { HashLink as Link } from 'react-router-hash-link';
+import debounce from 'lodash/debounce';
 
-import { formatTime, formatDateTime } from '../../helpers/helpers';
+import {
-import { SERVICES, FILTERED_STATUS } from '../../helpers/constants';
+    formatTime,
+    formatDateTime,
+    isValidQuestionType,
+} from '../../helpers/helpers';
+import { SERVICES, FILTERED_STATUS, DEBOUNCE_TIMEOUT } from '../../helpers/constants';
 import { getTrackerData } from '../../helpers/trackers/trackers';
 import { formatClientCell } from '../../helpers/formatClientCell';
 
@@ -16,8 +21,12 @@ import Card from '../ui/Card';
 import Loading from '../ui/Loading';
 import PopoverFiltered from '../ui/PopoverFilter';
 import Popover from '../ui/Popover';
+import Tooltip from '../ui/Tooltip';
 import './Logs.css';
 
+const TABLE_FIRST_PAGE = 0;
+const TABLE_DEFAULT_PAGE_SIZE = 50;
+const INITIAL_REQUEST_DATA = ['', {}, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE];
 const FILTERED_REASON = 'Filtered';
 const RESPONSE_FILTER = {
     ALL: 'all',
@@ -26,26 +35,36 @@ const RESPONSE_FILTER = {
 
 class Logs extends Component {
     componentDidMount() {
-        this.getLogs();
+        this.getLogs(...INITIAL_REQUEST_DATA);
         this.props.getFilteringStatus();
         this.props.getClients();
         this.props.getLogsConfig();
     }
 
-    componentDidUpdate(prevProps) {
+    getLogs = (lastRowTime, filter, page, pageSize) => {
-        // get logs when queryLog becomes enabled
-        if (this.props.queryLogs.enabled && !prevProps.queryLogs.enabled) {
-            this.props.getLogs();
-        }
-    }
-
-    getLogs = () => {
-        // get logs on initialization if queryLogIsEnabled
         if (this.props.queryLogs.enabled) {
-            this.props.getLogs();
+            this.props.getLogs({
+                lastRowTime, filter, page, pageSize,
+            });
         }
     };
 
+    refreshLogs = (lastRowTime, filter, page, pageSize, refreshLogs = true) => {
+        this.props.getLogs({
+            lastRowTime, filter, page, pageSize, refreshLogs,
+        });
+    };
+
+    handleLogsFiltering = debounce((lastRowTime, filter, page, pageSize, filtered) => {
+        this.props.getLogs({
+            lastRowTime,
+            filter,
+            page,
+            pageSize,
+            filtered,
+        });
+    }, DEBOUNCE_TIMEOUT);
+
     renderTooltip = (isFiltered, rule, filter, service) =>
         isFiltered && <PopoverFiltered rule={rule} filter={filter} service={service} />;
 
@@ -215,8 +234,64 @@ class Logs extends Component {
         );
     };
 
-    renderLogs(logs) {
+    getFilterInput = ({ filter, onChange }) => (
-        const { t } = this.props;
+        <Fragment>
+            <div className="logs__input-wrap">
+                <input
+                    type="text"
+                    className="form-control"
+                    onChange={event => onChange(event.target.value)}
+                    value={filter ? filter.value : ''}
+                />
+                <span className="logs__notice">
+                    <Tooltip text={this.props.t('query_log_strict_search')} type='tooltip-custom--logs' />
+                </span>
+            </div>
+        </Fragment>
+    );
+
+    getFilters = (filtered) => {
+        const filteredObj = filtered.reduce((acc, cur) => ({ ...acc, [cur.id]: cur.value }), {});
+        const {
+            domain, client, type, response,
+        } = filteredObj;
+
+        return {
+            domain: domain || '',
+            client: client || '',
+            question_type: isValidQuestionType(type) ? type.toUpperCase() : '',
+            response_status: response === RESPONSE_FILTER.FILTERED ? response : '',
+        };
+    };
+
+    fetchData = (state) => {
+        const {
+            filtered, pageSize, page, pages,
+        } = state;
+        const { allLogs } = this.props.queryLogs;
+        const isLastPage = pages && (page + 1 >= pages);
+        const isFiltersPresent = filtered.length > 0;
+        const filter = this.getFilters(filtered);
+
+        if (isFiltersPresent) {
+            this.handleLogsFiltering('', filter, page, pageSize, true);
+        } else if (isLastPage) {
+            const lastRow = allLogs[allLogs.length - 1];
+            const lastRowTime = (lastRow && lastRow.time) || '';
+            this.getLogs(lastRowTime, filter, page, pageSize);
+        } else {
+            this.props.setLogsPagination({ page, pageSize });
+        }
+    };
+
+    renderLogs() {
+        const { queryLogs, dashboard, t } = this.props;
+        const { processingClients } = dashboard;
+        const {
+            processingGetLogs, processingGetConfig, logs, pages,
+        } = queryLogs;
+        const isLoading = processingGetLogs || processingClients || processingGetConfig;
+
         const columns = [
             {
                 Header: t('time_table_header'),
@@ -230,6 +305,7 @@ class Logs extends Component {
                 accessor: 'domain',
                 minWidth: 180,
                 Cell: this.getDomainCell,
+                Filter: this.getFilterInput,
             },
             {
                 Header: t('type_table_header'),
@@ -251,7 +327,7 @@ class Logs extends Component {
                 },
                 Filter: ({ filter, onChange }) => (
                     <select
-                        className="form-control"
+                        className="form-control custom-select"
                         onChange={event => onChange(event.target.value)}
                         value={filter ? filter.value : RESPONSE_FILTER.ALL}
                     >
@@ -270,82 +346,83 @@ class Logs extends Component {
                 maxWidth: 240,
                 minWidth: 240,
                 Cell: this.getClientCell,
+                Filter: this.getFilterInput,
             },
         ];
 
-        if (logs) {
+        return (
-            return (
+            <ReactTable
-                <ReactTable
+                manual
-                    className="logs__table"
+                filterable
-                    filterable
+                minRows={5}
-                    data={logs}
+                pages={pages}
-                    columns={columns}
+                columns={columns}
-                    showPagination={true}
+                sortable={false}
-                    defaultPageSize={50}
+                data={logs || []}
-                    minRows={7}
+                loading={isLoading}
-                    previousText={t('previous_btn')}
+                showPageJump={false}
-                    nextText={t('next_btn')}
+                onFetchData={this.fetchData}
-                    loadingText={t('loading_table_status')}
+                onPageChange={page => this.setState({ page })}
-                    pageText={t('page_table_footer_text')}
+                className="logs__table"
-                    ofText={t('of_table_footer_text')}
+                showPagination={true}
-                    rowsText={t('rows_table_footer_text')}
+                defaultPageSize={TABLE_DEFAULT_PAGE_SIZE}
-                    noDataText={t('no_logs_found')}
+                previousText={t('previous_btn')}
-                    defaultFilterMethod={(filter, row) => {
+                nextText={t('next_btn')}
-                        const id = filter.pivotId || filter.id;
+                loadingText={t('loading_table_status')}
-                        return row[id] !== undefined
+                pageText={t('page_table_footer_text')}
-                            ? String(row[id]).indexOf(filter.value) !== -1
+                ofText={t('of_table_footer_text')}
-                            : true;
+                rowsText={t('rows_table_footer_text')}
-                    }}
+                noDataText={t('no_logs_found')}
-                    defaultSorted={[
+                defaultFilterMethod={(filter, row) => {
-                        {
+                    const id = filter.pivotId || filter.id;
-                            id: 'time',
+                    return row[id] !== undefined
-                            desc: true,
+                        ? String(row[id]).indexOf(filter.value) !== -1
-                        },
+                        : true;
-                    ]}
+                }}
-                    getTrProps={(_state, rowInfo) => {
+                defaultSorted={[
-                        if (!rowInfo) {
+                    {
-                            return {};
+                        id: 'time',
-                        }
+                        desc: true,
+                    },
+                ]}
+                getTrProps={(_state, rowInfo) => {
+                    if (!rowInfo) {
+                        return {};
+                    }
 
-                        const { reason } = rowInfo.original;
+                    const { reason } = rowInfo.original;
-
-                        if (this.checkFiltered(reason)) {
-                            return {
-                                className: 'red',
-                            };
-                        } else if (this.checkWhiteList(reason)) {
-                            return {
-                                className: 'green',
-                            };
-                        } else if (this.checkRewrite(reason)) {
-                            return {
-                                className: 'blue',
-                            };
-                        }
 
+                    if (this.checkFiltered(reason)) {
                         return {
-                            className: '',
+                            className: 'red',
                         };
-                    }}
+                    } else if (this.checkWhiteList(reason)) {
-                />
+                        return {
-            );
+                            className: 'green',
-        }
+                        };
+                    } else if (this.checkRewrite(reason)) {
+                        return {
+                            className: 'blue',
+                        };
+                    }
 
-        return null;
+                    return {
+                        className: '',
+                    };
+                }}
+            />
+        );
     }
 
     render() {
-        const { queryLogs, dashboard, t } = this.props;
+        const { queryLogs, t } = this.props;
-        const { enabled, processingGetLogs, processingGetConfig } = queryLogs;
+        const { enabled, processingGetConfig } = queryLogs;
-        const { processingClients } = dashboard;
-        const isDataReady =
-            !processingGetLogs && !processingGetConfig && !dashboard.processingClients;
 
         const refreshButton = enabled ? (
             <button
+                type="button"
                 className="btn btn-icon btn-outline-primary btn-sm ml-3"
-                type="submit"
+                onClick={() => this.refreshLogs(...INITIAL_REQUEST_DATA)}
-                onClick={this.getLogs}
             >
                 <svg className="icons">
                     <use xlinkHref="#refresh" />
@@ -357,22 +434,24 @@ class Logs extends Component {
 
         return (
             <Fragment>
-                <PageTitle title={t('query_log')} subtitle={t('last_dns_queries')}>
+                <PageTitle title={t('query_log')}>{refreshButton}</PageTitle>
-                    {refreshButton}
+                {enabled && processingGetConfig && <Loading />}
-                </PageTitle>
+                {enabled && !processingGetConfig && <Card>{this.renderLogs()}</Card>}
-                <Card>
+                {!enabled && !processingGetConfig && (
-                    {enabled && (processingGetLogs || processingClients || processingGetConfig) && (
+                    <Card>
-                        <Loading />
-                    )}
-                    {enabled && isDataReady && this.renderLogs(queryLogs.logs)}
-                    {!enabled && !processingGetConfig && (
                         <div className="lead text-center py-6">
-                            <Trans components={[<Link to="/settings#logs-config" key="0">link</Link>]}>
+                            <Trans
+                                components={[
+                                    <Link to="/settings#logs-config" key="0">
+                                        link
+                                    </Link>,
+                                ]}
+                            >
                                 query_log_disabled
                             </Trans>
                         </div>
-                    )}
+                    </Card>
-                </Card>
+                )}
             </Fragment>
         );
     }
@@ -388,6 +467,7 @@ Logs.propTypes = {
     addSuccessToast: PropTypes.func.isRequired,
     getClients: PropTypes.func.isRequired,
     getLogsConfig: PropTypes.func.isRequired,
+    setLogsPagination: PropTypes.func.isRequired,
     t: PropTypes.func.isRequired,
 };
 

client/src/components/ui/Popover.css

@@ -38,6 +38,7 @@
     position: absolute;
     bottom: calc(100% + 3px);
     left: 50%;
+    z-index: 1;
     min-width: 275px;
     padding: 10px 15px;
     font-size: 0.8rem;

client/src/components/ui/Tooltip.css

@@ -52,3 +52,23 @@
 .tooltip-custom--narrow:before {
     width: 220px;
 }
+
+.tooltip-custom--logs {
+    border-radius: 50%;
+    background-image: url("./svg/help-circle-gray.svg");
+    background-color: #fff;
+}
+
+.tooltip-custom--logs:before {
+    bottom: initial;
+    top: calc(100% + 10px);
+    right: -10px;
+    left: initial;
+    transform: none;
+}
+
+.tooltip-custom--logs:after {
+    top: 8px;
+    border-top: none;
+    border-bottom: 6px solid #585965;
+}

client/src/components/ui/svg/help-circle-gray.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#9aa0ac" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-help-circle"><circle cx="12" cy="12" r="10"></circle><path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3"></path><line x1="12" y1="17" x2="12" y2="17"></line></svg>

client/src/containers/Logs.js

@@ -1,7 +1,7 @@
 import { connect } from 'react-redux';
 import { addSuccessToast, getClients } from '../actions';
 import { getFilteringStatus, setRules } from '../actions/filtering';
-import { getLogs, getLogsConfig } from '../actions/queryLogs';
+import { getLogs, getLogsConfig, setLogsPagination } from '../actions/queryLogs';
 import Logs from '../components/Logs';
 
 const mapStateToProps = (state) => {
@@ -17,6 +17,7 @@ const mapDispatchToProps = {
     addSuccessToast,
     getClients,
     getLogsConfig,
+    setLogsPagination,
 };
 
 export default connect(

client/src/helpers/constants.js

@@ -274,3 +274,47 @@ export const WHOIS_ICONS = {
     netname: 'network',
     descr: '',
 };
+
+export const DNS_RECORD_TYPES = [
+    'A',
+    'AAAA',
+    'AFSDB',
+    'APL',
+    'CAA',
+    'CDNSKEY',
+    'CDS',
+    'CERT',
+    'CNAME',
+    'CSYNC',
+    'DHCID',
+    'DLV',
+    'DNAME',
+    'DNSKEY',
+    'DS',
+    'HIP',
+    'IPSECKEY',
+    'KEY',
+    'KX',
+    'LOC',
+    'MX',
+    'NAPTR',
+    'NS',
+    'NSEC',
+    'NSEC3',
+    'NSEC3PARAM',
+    'OPENPGPKEY',
+    'PTR',
+    'RRSIG',
+    'RP',
+    'SIG',
+    'SMIMEA',
+    'SOA',
+    'SRV',
+    'SSHFP',
+    'TA',
+    'TKEY',
+    'TLSA',
+    'TSIG',
+    'TXT',
+    'URI',
+];

client/src/helpers/helpers.js

@@ -14,6 +14,7 @@ import {
     STANDARD_WEB_PORT,
     STANDARD_HTTPS_PORT,
     CHECK_TIMEOUT,
+    DNS_RECORD_TYPES,
 } from './constants';
 
 export const formatTime = (time) => {
@@ -318,3 +319,5 @@ export const normalizeWhois = (whois) => {
 
     return whois;
 };
+
+export const isValidQuestionType = type => type && DNS_RECORD_TYPES.includes(type.toUpperCase());

client/src/reducers/queryLogs.js

@@ -4,11 +4,54 @@ import * as actions from '../actions/queryLogs';
 
 const queryLogs = handleActions(
     {
+        [actions.setLogsPagination]: (state, { payload }) => {
+            const { page, pageSize } = payload;
+            const { allLogs } = state;
+            const rowsStart = pageSize * page;
+            const rowsEnd = (pageSize * page) + pageSize;
+            const logsSlice = allLogs.slice(rowsStart, rowsEnd);
+            const pages = Math.ceil(allLogs.length / pageSize);
+
+            return {
+                ...state,
+                pages,
+                logs: logsSlice,
+            };
+        },
+
         [actions.getLogsRequest]: state => ({ ...state, processingGetLogs: true }),
         [actions.getLogsFailure]: state => ({ ...state, processingGetLogs: false }),
         [actions.getLogsSuccess]: (state, { payload }) => {
-            const newState = { ...state, logs: payload, processingGetLogs: false };
+            const {
-            return newState;
+                logs, lastRowTime, page, pageSize, filtered, refreshLogs,
+            } = payload;
+            let logsWithOffset = state.allLogs.length > 0 ? state.allLogs : logs;
+            let allLogs = logs;
+
+            if (lastRowTime) {
+                logsWithOffset = [...state.allLogs, ...logs];
+                allLogs = [...state.allLogs, ...logs];
+            }
+
+            if (filtered || refreshLogs) {
+                logsWithOffset = logs;
+                allLogs = logs;
+            }
+
+            const pages = Math.ceil(logsWithOffset.length / pageSize);
+            const total = logsWithOffset.length;
+            const rowsStart = pageSize * page;
+            const rowsEnd = (pageSize * page) + pageSize;
+            const logsSlice = logsWithOffset.slice(rowsStart, rowsEnd);
+
+            return {
+                ...state,
+                pages,
+                total,
+                allLogs,
+                logs: logsSlice,
+                processingGetLogs: false,
+            };
         },
 
         [actions.clearLogsRequest]: state => ({ ...state, processingClear: true }),
@@ -42,6 +85,10 @@ const queryLogs = handleActions(
         processingSetConfig: false,
         logs: [],
         interval: 1,
+        allLogs: [],
+        pages: 0,
+        offset: 0,
+        total: 0,
         enabled: true,
     },
 );