diff --git a/AGHTechDoc.md b/AGHTechDoc.md index d901a862..dafb44c0 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -1012,17 +1012,20 @@ Response: When a new DNS request is received and processed, we store information about this event in "query log". It is a file on disk in JSON format: { - "Question":"..."," - Answer":"...", + "IP":"127.0.0.1", // client IP + "T":"...", // response time + "QH":"...", // target host name without the last dot + "QT":"...", // question type + "QC":"...", // question class + "Answer":"...", "Result":{ "IsFiltered":true, "Reason":3, "Rule":"...", "FilterID":1 }, - "Time":"...", "Elapsed":12345, - "IP":"127.0.0.1" + "Upstream":"...", } @@ -1052,7 +1055,7 @@ Request: &filter_question_type=A | AAAA &filter_response_status= | filtered -If `older_than` value is set, server returns the next chunk of entries that are older than this time stamp. This setting is used for paging. UI sets the empty value on the first request and gets the latest log entries. To get the older entries, UI sets this value to the timestamp of the last (the oldest) entry from the previous response from Server. +`older_than` setting is used for paging. UI uses an empty value for `older_than` on the first request and gets the latest log entries. To get the older entries, UI sets `older_than` to the `oldest` value from the server's response. If "filter" settings are set, server returns only entries that match the specified request. @@ -1060,7 +1063,9 @@ For `filter.domain` and `filter.client` the server matches substrings by default Response: - [ + { + "oldest":"2006-01-02T15:04:05.999999999Z07:00" + "data":[ { "answer":[ { @@ -1085,6 +1090,7 @@ Response: } ... ] + } The most recent entries are at the top of list. diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 53f269a2..4365b6ec 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -401,4 +401,4 @@ "descr": "Description", "whois": "Whois", "filtering_rules_learn_more": "<0>Learn more about creating your own hosts blocklists." -} \ No newline at end of file +} diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index 09e93a60..35b4f7af 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -3,26 +3,100 @@ import { createAction } from 'redux-actions'; import apiClient from '../api/Api'; import { addErrorToast, addSuccessToast } from './index'; import { normalizeLogs } from '../helpers/helpers'; +import { TABLE_DEFAULT_PAGE_SIZE } from '../helpers/constants'; + +const getLogsWithParams = async (config) => { + const { older_than, filter, ...values } = config; + const rawLogs = await apiClient.getQueryLog({ ...filter, older_than }); + const { data, oldest } = rawLogs; + const logs = normalizeLogs(data); + + return { + logs, oldest, older_than, filter, ...values, + }; +}; + +export const getAdditionalLogsRequest = createAction('GET_ADDITIONAL_LOGS_REQUEST'); +export const getAdditionalLogsFailure = createAction('GET_ADDITIONAL_LOGS_FAILURE'); +export const getAdditionalLogsSuccess = createAction('GET_ADDITIONAL_LOGS_SUCCESS'); + +const checkFilteredLogs = async (data, filter, dispatch, total) => { + const { logs, oldest } = data; + const totalData = total || { logs }; + + const needToGetAdditionalLogs = (logs.length < TABLE_DEFAULT_PAGE_SIZE || + totalData.logs.length < TABLE_DEFAULT_PAGE_SIZE) && + oldest !== ''; + + if (needToGetAdditionalLogs) { + dispatch(getAdditionalLogsRequest()); + + try { + const additionalLogs = await getLogsWithParams({ older_than: oldest, filter }); + if (additionalLogs.logs.length > 0) { + return await checkFilteredLogs(additionalLogs, filter, dispatch, { + logs: [...totalData.logs, ...additionalLogs.logs], + oldest: additionalLogs.oldest, + }); + } + dispatch(getAdditionalLogsSuccess()); + return totalData; + } catch (error) { + dispatch(addErrorToast({ error })); + dispatch(getAdditionalLogsFailure(error)); + } + } + + dispatch(getAdditionalLogsSuccess()); + return totalData; +}; export const setLogsPagination = createAction('LOGS_PAGINATION'); -export const setLogsFilter = createAction('LOGS_FILTER'); +export const setLogsPage = createAction('SET_LOG_PAGE'); export const getLogsRequest = createAction('GET_LOGS_REQUEST'); export const getLogsFailure = createAction('GET_LOGS_FAILURE'); export const getLogsSuccess = createAction('GET_LOGS_SUCCESS'); -export const getLogs = config => async (dispatch) => { +export const getLogs = config => async (dispatch, getState) => { dispatch(getLogsRequest()); try { - const { filter, lastRowTime: older_than } = config; - const logs = normalizeLogs(await apiClient.getQueryLog({ ...filter, older_than })); - dispatch(getLogsSuccess({ logs, ...config })); + const { isFiltered, filter, page } = getState().queryLogs; + const data = await getLogsWithParams({ ...config, filter }); + + if (isFiltered) { + const additionalData = await checkFilteredLogs(data, filter, dispatch); + const updatedData = additionalData.logs ? { ...data, ...additionalData } : data; + dispatch(getLogsSuccess(updatedData)); + dispatch(setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE })); + } else { + dispatch(getLogsSuccess(data)); + } } catch (error) { dispatch(addErrorToast({ error })); dispatch(getLogsFailure(error)); } }; +export const setLogsFilterRequest = createAction('SET_LOGS_FILTER_REQUEST'); +export const setLogsFilterFailure = createAction('SET_LOGS_FILTER_FAILURE'); +export const setLogsFilterSuccess = createAction('SET_LOGS_FILTER_SUCCESS'); + +export const setLogsFilter = filter => async (dispatch) => { + dispatch(setLogsFilterRequest()); + try { + const data = await getLogsWithParams({ older_than: '', filter }); + const additionalData = await checkFilteredLogs(data, filter, dispatch); + const updatedData = additionalData.logs ? { ...data, ...additionalData } : data; + + dispatch(setLogsFilterSuccess({ ...updatedData, filter })); + dispatch(setLogsPage(0)); + } catch (error) { + dispatch(addErrorToast({ error })); + dispatch(setLogsFilterFailure(error)); + } +}; + export const clearLogsRequest = createAction('CLEAR_LOGS_REQUEST'); export const clearLogsFailure = createAction('CLEAR_LOGS_FAILURE'); export const clearLogsSuccess = createAction('CLEAR_LOGS_SUCCESS'); diff --git a/client/src/components/Logs/Filters/Form.js b/client/src/components/Logs/Filters/Form.js new file mode 100644 index 00000000..cedc6f5e --- /dev/null +++ b/client/src/components/Logs/Filters/Form.js @@ -0,0 +1,116 @@ +import React, { Fragment } from 'react'; +import PropTypes from 'prop-types'; +import { Field, reduxForm } from 'redux-form'; +import { withNamespaces, Trans } from 'react-i18next'; +import flow from 'lodash/flow'; + +import { renderField } from '../../../helpers/form'; +import { RESPONSE_FILTER } from '../../../helpers/constants'; +import Tooltip from '../../ui/Tooltip'; + +const renderFilterField = ({ + input, + id, + className, + placeholder, + type, + disabled, + autoComplete, + tooltip, + meta: { touched, error }, +}) => ( + +
+ + + + + {!disabled && + touched && + (error && {error})} +
+ +); + +const Form = (props) => { + const { + t, + handleChange, + } = props; + + return ( +
+
+
+ +
+
+ +
+
+ + + + +
+
+ +
+
+
+ ); +}; + +Form.propTypes = { + handleChange: PropTypes.func, + t: PropTypes.func.isRequired, +}; + +export default flow([ + withNamespaces(), + reduxForm({ + form: 'logsFilterForm', + }), +])(Form); diff --git a/client/src/components/Logs/Filters/index.js b/client/src/components/Logs/Filters/index.js new file mode 100644 index 00000000..ab654cbe --- /dev/null +++ b/client/src/components/Logs/Filters/index.js @@ -0,0 +1,52 @@ +import React, { Component } from 'react'; +import PropTypes from 'prop-types'; +import debounce from 'lodash/debounce'; +import classnames from 'classnames'; + +import { DEBOUNCE_FILTER_TIMEOUT, RESPONSE_FILTER } from '../../../helpers/constants'; +import { isValidQuestionType } from '../../../helpers/helpers'; +import Form from './Form'; +import Card from '../../ui/Card'; + +class Filters extends Component { + getFilters = ({ + filter_domain, filter_question_type, filter_response_status, filter_client, + }) => ({ + filter_domain: filter_domain || '', + filter_question_type: isValidQuestionType(filter_question_type) ? filter_question_type.toUpperCase() : '', + filter_response_status: filter_response_status === RESPONSE_FILTER.FILTERED ? filter_response_status : '', + filter_client: filter_client || '', + }); + + handleFormChange = debounce((values) => { + const filter = this.getFilters(values); + this.props.setLogsFilter(filter); + }, DEBOUNCE_FILTER_TIMEOUT); + + render() { + const { filter, processingAdditionalLogs } = this.props; + + const cardBodyClass = classnames({ + 'card-body': true, + 'card-body--loading': processingAdditionalLogs, + }); + + return ( + +
+ + ); + } +} + +Filters.propTypes = { + filter: PropTypes.object.isRequired, + setLogsFilter: PropTypes.func.isRequired, + processingGetLogs: PropTypes.bool.isRequired, + processingAdditionalLogs: PropTypes.bool.isRequired, +}; + +export default Filters; diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index e61b19d1..68b9cc61 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -5,46 +5,40 @@ import escapeRegExp from 'lodash/escapeRegExp'; import endsWith from 'lodash/endsWith'; import { Trans, withNamespaces } from 'react-i18next'; import { HashLink as Link } from 'react-router-hash-link'; -import debounce from 'lodash/debounce'; import { formatTime, formatDateTime, - isValidQuestionType, } from '../../helpers/helpers'; -import { SERVICES, FILTERED_STATUS, DEBOUNCE_TIMEOUT, DEFAULT_LOGS_FILTER } from '../../helpers/constants'; +import { SERVICES, FILTERED_STATUS, TABLE_DEFAULT_PAGE_SIZE } from '../../helpers/constants'; import { getTrackerData } from '../../helpers/trackers/trackers'; import { formatClientCell } from '../../helpers/formatClientCell'; +import Filters from './Filters'; import PageTitle from '../ui/PageTitle'; import Card from '../ui/Card'; import Loading from '../ui/Loading'; import PopoverFiltered from '../ui/PopoverFilter'; import Popover from '../ui/Popover'; -import Tooltip from '../ui/Tooltip'; import './Logs.css'; const TABLE_FIRST_PAGE = 0; -const TABLE_DEFAULT_PAGE_SIZE = 50; -const INITIAL_REQUEST_DATA = ['', DEFAULT_LOGS_FILTER, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; +const INITIAL_REQUEST_DATA = ['', TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; const FILTERED_REASON = 'Filtered'; -const RESPONSE_FILTER = { - ALL: 'all', - FILTERED: 'filtered', -}; class Logs extends Component { componentDidMount() { + this.props.setLogsPage(TABLE_FIRST_PAGE); this.getLogs(...INITIAL_REQUEST_DATA); this.props.getFilteringStatus(); this.props.getClients(); this.props.getLogsConfig(); } - getLogs = (lastRowTime, filter, page, pageSize, filtered) => { + getLogs = (older_than, page) => { if (this.props.queryLogs.enabled) { this.props.getLogs({ - lastRowTime, filter, page, pageSize, filtered, + older_than, page, pageSize: TABLE_DEFAULT_PAGE_SIZE, }); } }; @@ -53,16 +47,6 @@ class Logs extends Component { window.location.reload(); }; - handleLogsFiltering = debounce((lastRowTime, filter, page, pageSize, filtered) => { - this.props.getLogs({ - lastRowTime, - filter, - page, - pageSize, - filtered, - }); - }, DEBOUNCE_TIMEOUT); - renderTooltip = (isFiltered, rule, filter, service) => isFiltered && ; @@ -232,72 +216,26 @@ class Logs extends Component { ); }; - getFilterInput = ({ filter, onChange }) => ( - -
- onChange(event.target.value)} - value={filter ? filter.value : ''} - /> - - - -
- - ); - - getFilters = (filtered) => { - const filteredObj = filtered.reduce((acc, cur) => ({ ...acc, [cur.id]: cur.value }), {}); - const { - domain, client, type, response, - } = filteredObj; - - return { - filter_domain: domain || '', - filter_client: client || '', - filter_question_type: isValidQuestionType(type) ? type.toUpperCase() : '', - filter_response_status: response === RESPONSE_FILTER.FILTERED ? response : '', - }; - }; - fetchData = (state) => { - const { pageSize, page, pages } = state; - const { allLogs, filter } = this.props.queryLogs; + const { pages } = state; + const { oldest, page } = this.props.queryLogs; const isLastPage = pages && (page + 1 === pages); if (isLastPage) { - const lastRow = allLogs[allLogs.length - 1]; - const lastRowTime = (lastRow && lastRow.time) || ''; - this.getLogs(lastRowTime, filter, page, pageSize, true); - } else { - this.props.setLogsPagination({ page, pageSize }); + this.getLogs(oldest, page); } }; - handleFilterChange = (filtered) => { - const filters = this.getFilters(filtered); - this.props.setLogsFilter(filters); - this.handleLogsFiltering('', filters, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE, true); - } - - showTotalPagesCount = (pages) => { - const { total, isEntireLog } = this.props.queryLogs; - const showEllipsis = !isEntireLog && total >= 500; - - return ( - - {pages || 1}{showEllipsis && '…' } - - ); - } + changePage = (page) => { + this.props.setLogsPage(page); + this.props.setLogsPagination({ page, pageSize: TABLE_DEFAULT_PAGE_SIZE }); + }; renderLogs() { const { queryLogs, dashboard, t } = this.props; const { processingClients } = dashboard; const { - processingGetLogs, processingGetConfig, logs, pages, + processingGetLogs, processingGetConfig, logs, pages, page, } = queryLogs; const isLoading = processingGetLogs || processingClients || processingGetConfig; @@ -306,7 +244,6 @@ class Logs extends Component { Header: t('time_table_header'), accessor: 'time', maxWidth: 100, - filterable: false, Cell: this.getTimeCell, }, { @@ -314,7 +251,6 @@ class Logs extends Component { accessor: 'domain', minWidth: 180, Cell: this.getDomainCell, - Filter: this.getFilterInput, }, { Header: t('type_table_header'), @@ -326,28 +262,6 @@ class Logs extends Component { accessor: 'response', minWidth: 250, Cell: this.getResponseCell, - filterMethod: (filter, row) => { - if (filter.value === RESPONSE_FILTER.FILTERED) { - // eslint-disable-next-line no-underscore-dangle - const { reason } = row._original; - return this.checkFiltered(reason) || this.checkWhiteList(reason); - } - return true; - }, - Filter: ({ filter, onChange }) => ( - - ), }, { Header: t('client_table_header'), @@ -355,34 +269,36 @@ class Logs extends Component { maxWidth: 240, minWidth: 240, Cell: this.getClientCell, - Filter: this.getFilterInput, }, ]; return ( false} defaultFilterMethod={(filter, row) => { const id = filter.pivotId || filter.id; return row[id] !== undefined @@ -426,7 +342,9 @@ class Logs extends Component { render() { const { queryLogs, t } = this.props; - const { enabled, processingGetConfig } = queryLogs; + const { + enabled, processingGetConfig, processingAdditionalLogs, processingGetLogs, + } = queryLogs; const refreshButton = enabled ? (