Pull request: 2504 querylog interval

Merge in DNS/adguard-home from 2504-querylog-ivl to master

Updates #2504.

Squashed commit of the following:

commit 5d15a6f735cd195fc81c8af909b56fbc7db1fe21
Merge: 8cd5c30d 97073d0d
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 18:45:10 2021 +0300

    Merge branch 'master' into 2504-querylog-ivl

commit 8cd5c30de6f72d4b12162dbc9e3d90132795fe94
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 18:35:50 2021 +0300

    client: fix fmt

commit e95d462c31d886bacec0735acc567fec7c962149
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 17:58:06 2021 +0300

    home: imp code

commit 48737b249c52a997a4f34dac45fbaf699477b007
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 17:23:18 2021 +0300

    home: imp duration

commit 44f5dc3d3ada5120d74caa24cace9a253b8f15d3
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 16:55:31 2021 +0300

    home: imp code, docs

commit bb2826521b7e5d248ce2ab686528219c312b8ba2
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 16:11:40 2021 +0300

    all: imp code, docs

commit d688aed1f340807a8bac8807c263956b0fc16f5b
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jul 1 13:49:42 2021 +0300

    all: change querylog interval setting format
This commit is contained in:
Eugene Burkov 2021-07-01 18:50:28 +03:00
parent 97073d0d9e
commit e113b276e7
19 changed files with 357 additions and 79 deletions

View File

@ -15,6 +15,7 @@ and this project adheres to
### Added
- New possible value of `6h` for `querylog_interval` setting ([#2504]).
- Blocking access using client IDs ([#2624], [#3162]).
- `source` directives support in `/etc/network/interfaces` on Linux ([#3257]).
- RFC 9000 support in DNS-over-QUIC.
@ -40,6 +41,7 @@ and this project adheres to
### Changed
- `querylog_interval` setting is now formatted in hours.
- Query log search now supports internationalized domains ([#3012]).
- Internationalized domains are now shown decoded in the query log with the
original encoded version shown in request details ([#3013]).
@ -82,6 +84,7 @@ released by then.
[#2439]: https://github.com/AdguardTeam/AdGuardHome/issues/2439
[#2441]: https://github.com/AdguardTeam/AdGuardHome/issues/2441
[#2443]: https://github.com/AdguardTeam/AdGuardHome/issues/2443
[#2504]: https://github.com/AdguardTeam/AdGuardHome/issues/2504
[#2624]: https://github.com/AdguardTeam/AdGuardHome/issues/2624
[#2763]: https://github.com/AdguardTeam/AdGuardHome/issues/2763
[#3012]: https://github.com/AdguardTeam/AdGuardHome/issues/3012

View File

@ -484,6 +484,7 @@
"encryption_key_source_content": "Paste the private key contents",
"stats_params": "Statistics configuration",
"config_successfully_saved": "Configuration successfully saved",
"interval_6_hour": "6 hours",
"interval_24_hour": "24 hours",
"interval_days": "{{count}} day",
"interval_days_plural": "{{count}} days",

View File

@ -4,26 +4,33 @@ import { Field, reduxForm } from 'redux-form';
import { Trans, withTranslation } from 'react-i18next';
import flow from 'lodash/flow';
import { CheckboxField, renderRadioField, toNumber } from '../../../helpers/form';
import { CheckboxField, renderRadioField, toFloatNumber } from '../../../helpers/form';
import { FORM_NAME, QUERY_LOG_INTERVALS_DAYS } from '../../../helpers/constants';
import '../FormButton.css';
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => {
const title = interval === 1 ? t('interval_24_hour') : t('interval_days', { count: interval });
const getIntervalTitle = (interval, t) => {
switch (interval) {
case 0.25:
return t('interval_6_hour');
case 1:
return t('interval_24_hour');
default:
return t('interval_days', { count: interval });
}
};
return (
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => (
<Field
key={interval}
name="interval"
type="radio"
component={renderRadioField}
value={interval}
placeholder={title}
placeholder={getIntervalTitle(interval, t)}
normalize={toNumber}
disabled={processing}
/>
);
});
));
const Form = (props) => {
const {
@ -56,7 +63,7 @@ const Form = (props) => {
</label>
<div className="form__group form__group--settings">
<div className="custom-controls-stacked">
{getIntervalFields(processing, t, toNumber)}
{getIntervalFields(processing, t, toFloatNumber)}
</div>
</div>
<div className="mt-5">

View File

@ -357,7 +357,7 @@ export const NOT_FILTERED = 'NotFiltered';
export const STATS_INTERVALS_DAYS = [0, 1, 7, 30, 90];
export const QUERY_LOG_INTERVALS_DAYS = [1, 7, 30, 90];
export const QUERY_LOG_INTERVALS_DAYS = [0.25, 1, 7, 30, 90];
export const FILTERS_INTERVALS_HOURS = [0, 1, 12, 24, 72, 168];

View File

@ -276,6 +276,12 @@ export const ip4ToInt = (ip) => {
*/
export const toNumber = (value) => value && parseInt(value, 10);
/**
* @param value {string}
* @returns {*|number}
*/
export const toFloatNumber = (value) => value && parseFloat(value, 10);
/**
* @param value {string}
* @returns {boolean}

View File

@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
@ -104,7 +105,8 @@ type dnsConfig struct {
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
// QueryLogInterval is the interval for query log's files rotation.
QueryLogInterval Duration `yaml:"querylog_interval"`
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
@ -185,7 +187,7 @@ var config = configuration{
},
FilteringEnabled: true, // whether or not use filter lists
FiltersUpdateIntervalHours: 24,
UpstreamTimeout: Duration{dnsforward.DefaultTimeout},
UpstreamTimeout: Duration{Duration: dnsforward.DefaultTimeout},
LocalDomainName: "lan",
ResolveClients: true,
UsePrivateRDNS: true,
@ -212,7 +214,7 @@ func initConfig() {
config.DNS.QueryLogEnabled = true
config.DNS.QueryLogFileEnabled = true
config.DNS.QueryLogInterval = 90
config.DNS.QueryLogInterval = Duration{Duration: 90 * 24 * time.Hour}
config.DNS.QueryLogMemSize = 1000
config.DNS.CacheSize = 4 * 1024 * 1024
@ -281,7 +283,7 @@ func parseConfig() error {
}
if config.DNS.UpstreamTimeout.Duration == 0 {
config.DNS.UpstreamTimeout = Duration{dnsforward.DefaultTimeout}
config.DNS.UpstreamTimeout = Duration{Duration: dnsforward.DefaultTimeout}
}
return nil
@ -328,7 +330,7 @@ func (c *configuration) write() error {
Context.queryLog.WriteDiskConfig(&dc)
config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogFileEnabled = dc.FileEnabled
config.DNS.QueryLogInterval = dc.RotationIvl
config.DNS.QueryLogInterval = Duration{Duration: dc.RotationIvl}
config.DNS.QueryLogMemSize = dc.MemSize
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
}

View File

@ -48,7 +48,7 @@ func initDNSServer() error {
HTTPRegister: httpRegister,
FindClient: Context.clients.findMultiple,
BaseDir: baseDir,
RotationIvl: config.DNS.QueryLogInterval,
RotationIvl: config.DNS.QueryLogInterval.Duration,
MemSize: config.DNS.QueryLogMemSize,
Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,

View File

@ -12,6 +12,35 @@ type Duration struct {
time.Duration
}
// String implements the fmt.Stringer interface for Duration. It wraps
// time.Duration.String method and additionally cuts off non-leading zero values
// of minutes and seconds. Some values which are differ between the
// implementations:
//
// Duration: "1m", time.Duration: "1m0s"
// Duration: "1h", time.Duration: "1h0m0s"
// Duration: "1h1m", time.Duration: "1h1m0s"
//
func (d Duration) String() (str string) {
str = d.Duration.String()
secs := d.Seconds()
var secsInt int
if secsInt = int(secs); float64(secsInt) != secs || secsInt%60 != 0 {
return str
}
const (
tailMin = len(`0s`)
tailMinSec = len(`0m0s`)
)
if (secsInt%3600)/60 != 0 {
return str[:len(str)-tailMin]
}
return str[:len(str)-tailMinSec]
}
// MarshalText implements the encoding.TextMarshaler interface for Duration.
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
@ -19,6 +48,8 @@ func (d Duration) MarshalText() (text []byte, err error) {
// UnmarshalText implements the encoding.TextUnmarshaler interface for
// *Duration.
//
// TODO(e.burkov): Make it able to parse larger units like days.
func (d *Duration) UnmarshalText(b []byte) (err error) {
defer func() { err = errors.Annotate(err, "unmarshalling duration: %w") }()

View File

@ -12,6 +12,50 @@ import (
yaml "gopkg.in/yaml.v2"
)
func TestDuration_String(t *testing.T) {
testCases := []struct {
name string
val time.Duration
}{{
name: "1s",
val: time.Second,
}, {
name: "1m",
val: time.Minute,
}, {
name: "1h",
val: time.Hour,
}, {
name: "1m1s",
val: time.Minute + time.Second,
}, {
name: "1h1m",
val: time.Hour + time.Minute,
}, {
name: "1h0m1s",
val: time.Hour + time.Second,
}, {
name: "1ms",
val: time.Millisecond,
}, {
name: "1h0m0.001s",
val: time.Hour + time.Millisecond,
}, {
name: "1.001s",
val: time.Second + time.Millisecond,
}, {
name: "1m1.001s",
val: time.Minute + time.Second + time.Millisecond,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
d := Duration{Duration: tc.val}
assert.Equal(t, tc.name, d.String())
})
}
}
// durationEncodingTester is a helper struct to simplify testing different
// Duration marshalling and unmarshalling cases.
type durationEncodingTester struct {

View File

@ -9,6 +9,7 @@ import (
"runtime"
"strconv"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/errors"
@ -19,7 +20,7 @@ import (
)
// currentSchemaVersion is the current schema version.
const currentSchemaVersion = 11
const currentSchemaVersion = 12
// These aliases are provided for convenience.
type (
@ -82,6 +83,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema8to9,
upgradeSchema9to10,
upgradeSchema10to11,
upgradeSchema11to12,
}
n := 0
@ -647,6 +649,46 @@ func upgradeSchema10to11(diskConf yobj) (err error) {
return nil
}
// upgradeSchema11to12 performs the following changes:
//
// # BEFORE:
// 'querylog_interval': 90
//
// # AFTER:
// 'querylog_interval': '2160h'
//
func upgradeSchema11to12(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 11 to 12")
diskConf["schema_version"] = 12
dnsVal, ok := diskConf["dns"]
if !ok {
return nil
}
var dns yobj
dns, ok = dnsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
}
const field = "querylog_interval"
// Set the initial value from home.initConfig function.
qlogIvl := 90
qlogIvlVal, ok := dns[field]
if ok {
qlogIvl, ok = qlogIvlVal.(int)
if !ok {
return fmt.Errorf("unexpected type of %s: %T", field, qlogIvlVal)
}
}
dns[field] = Duration{Duration: time.Duration(qlogIvl) * 24 * time.Hour}
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging
// package.
func funcName() string {

View File

@ -2,6 +2,7 @@ package home
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -415,3 +416,98 @@ func TestUpgradeSchema10to11(t *testing.T) {
check(t, conf)
})
}
func TestUpgradeSchema11to12(t *testing.T) {
testCases := []struct {
ivl any
want any
wantErr string
name string
}{{
ivl: 1,
want: Duration{Duration: 24 * time.Hour},
wantErr: "",
name: "success",
}, {
ivl: 0.25,
want: 0,
wantErr: "unexpected type of querylog_interval: float64",
name: "fail",
}}
for _, tc := range testCases {
conf := yobj{
"dns": yobj{
"querylog_interval": tc.ivl,
},
"schema_version": 11,
}
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema11to12(conf)
if tc.wantErr != "" {
require.Error(t, err)
assert.Equal(t, tc.wantErr, err.Error())
return
}
require.NoError(t, err)
require.Equal(t, conf["schema_version"], 12)
dnsVal, ok := conf["dns"]
require.True(t, ok)
var newDNSConf yobj
newDNSConf, ok = dnsVal.(yobj)
require.True(t, ok)
var newIvl Duration
newIvl, ok = newDNSConf["querylog_interval"].(Duration)
require.True(t, ok)
assert.Equal(t, tc.want, newIvl)
})
}
t.Run("no_dns", func(t *testing.T) {
err := upgradeSchema11to12(yobj{})
assert.NoError(t, err)
})
t.Run("bad_dns", func(t *testing.T) {
err := upgradeSchema11to12(yobj{
"dns": 0,
})
require.Error(t, err)
assert.Equal(t, "unexpected type of dns: int", err.Error())
})
t.Run("no_field", func(t *testing.T) {
conf := yobj{
"dns": yobj{},
}
err := upgradeSchema11to12(conf)
require.NoError(t, err)
dns, ok := conf["dns"]
require.True(t, ok)
var dnsVal yobj
dnsVal, ok = dns.(yobj)
require.True(t, ok)
var ivl interface{}
ivl, ok = dnsVal["querylog_interval"]
require.True(t, ok)
var ivlVal Duration
ivlVal, ok = ivl.(Duration)
require.True(t, ok)
assert.Equal(t, 90*24*time.Hour, ivlVal.Duration)
})
}

View File

@ -17,7 +17,9 @@ import (
type qlogConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"`
// Use float64 here to support fractional numbers and not mess the API
// users by changing the units.
Interval float64 `json:"interval"`
AnonymizeClientIP bool `json:"anonymize_client_ip"`
}
@ -71,7 +73,7 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = l.conf.Enabled
resp.Interval = l.conf.RotationIvl
resp.Interval = l.conf.RotationIvl.Hours() / 24
resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
jsonVal, err := json.Marshal(resp)
@ -95,7 +97,8 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
return
}
if req.Exists("interval") && !checkInterval(d.Interval) {
ivl := time.Duration(24*d.Interval) * time.Hour
if req.Exists("interval") && !checkInterval(ivl) {
httpError(r, w, http.StatusBadRequest, "Unsupported interval")
return
}
@ -107,7 +110,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
conf.Enabled = d.Enabled
}
if req.Exists("interval") {
conf.RotationIvl = d.Interval
conf.RotationIvl = ivl
}
if req.Exists("anonymize_client_ip") {
conf.AnonymizeClientIP = d.AnonymizeClientIP

View File

@ -100,8 +100,17 @@ func (l *queryLog) Close() {
_ = l.flushLogBuffer(true)
}
func checkInterval(days uint32) bool {
return days == 1 || days == 7 || days == 30 || days == 90
func checkInterval(ivl time.Duration) (ok bool) {
// The constants for possible values of query log's rotation interval.
const (
quarterDay = 6 * time.Hour
day = 24 * time.Hour
week = day * 7
month = day * 30
threeMonths = day * 90
)
return ivl == quarterDay || ivl == day || ivl == week || ivl == month || ivl == threeMonths
}
func (l *queryLog) WriteDiskConfig(c *Config) {

View File

@ -26,7 +26,7 @@ func TestQueryLog(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
RotationIvl: 1,
RotationIvl: 24 * time.Hour,
MemSize: 100,
BaseDir: t.TempDir(),
})
@ -128,7 +128,7 @@ func TestQueryLog(t *testing.T) {
func TestQueryLogOffsetLimit(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
RotationIvl: 1,
RotationIvl: 24 * time.Hour,
MemSize: 100,
BaseDir: t.TempDir(),
})
@ -153,33 +153,34 @@ func TestQueryLogOffsetLimit(t *testing.T) {
testCases := []struct {
name string
want string
wantLen int
offset int
limit int
wantLen int
want string
}{{
name: "page_1",
want: firstPageDomain,
wantLen: 10,
offset: 0,
limit: 10,
wantLen: 10,
want: firstPageDomain,
}, {
name: "page_2",
want: secondPageDomain,
wantLen: 10,
offset: 10,
limit: 10,
wantLen: 10,
want: secondPageDomain,
}, {
name: "page_2.5",
want: secondPageDomain,
wantLen: 5,
offset: 15,
limit: 10,
wantLen: 5,
want: secondPageDomain,
}, {
name: "page_3",
want: "",
wantLen: 0,
offset: 20,
limit: 10,
wantLen: 0,
}}
for _, tc := range testCases {
@ -202,7 +203,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
RotationIvl: 1,
RotationIvl: 24 * time.Hour,
MemSize: 100,
BaseDir: t.TempDir(),
})
@ -230,7 +231,7 @@ func TestQueryLogFileDisabled(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: false,
RotationIvl: 1,
RotationIvl: 24 * time.Hour,
MemSize: 2,
BaseDir: t.TempDir(),
})

View File

@ -41,10 +41,17 @@ type Config struct {
// BaseDir is the base directory for log files.
BaseDir string
// RotationIvl is the interval for log rotation, in days. After that
// period, the old log file will be renamed, NOT deleted, so the actual
// log retention time is twice the interval.
RotationIvl uint32
// RotationIvl is the interval for log rotation. After that period, the
// old log file will be renamed, NOT deleted, so the actual log
// retention time is twice the interval. The value must be one of:
//
// 6 * time.Hour
// 24 * time.Hour
// 7 * 24 * time.Hour
// 30 * 24 * time.Hour
// 90 * 24 * time.Hour
//
RotationIvl time.Duration
// MemSize is the number of entries kept in a memory buffer before they
// are flushed to disk.
@ -118,7 +125,7 @@ func newQueryLog(conf Config) (l *queryLog) {
"querylog: warning: unsupported rotation interval %d, setting to 1 day",
conf.RotationIvl,
)
l.conf.RotationIvl = 1
l.conf.RotationIvl = 24 * time.Hour
}
return l

View File

@ -104,43 +104,52 @@ func (l *queryLog) rotate() error {
return nil
}
func (l *queryLog) readFileFirstTimeValue() int64 {
f, err := os.Open(l.logFile)
func (l *queryLog) readFileFirstTimeValue() (first time.Time, err error) {
var f *os.File
f, err = os.Open(l.logFile)
if err != nil {
return -1
return time.Time{}, err
}
defer func() {
derr := f.Close()
if derr != nil {
log.Error("querylog: closing file: %s", derr)
}
}()
buf := make([]byte, 500)
r, err := f.Read(buf)
defer func() { err = errors.WithDeferred(err, f.Close()) }()
buf := make([]byte, 512)
var r int
r, err = f.Read(buf)
if err != nil {
return -1
return time.Time{}, err
}
buf = buf[:r]
val := readJSONValue(string(buf), `"T":"`)
val := readJSONValue(string(buf[:r]), `"T":"`)
t, err := time.Parse(time.RFC3339Nano, val)
if err != nil {
return -1
return time.Time{}, err
}
log.Debug("querylog: the oldest log entry: %s", val)
return t.Unix()
return t, nil
}
func (l *queryLog) periodicRotate() {
intervalSeconds := uint64(l.conf.RotationIvl) * 24 * 60 * 60
defer log.OnPanic("querylog: rotating")
var err error
for {
oldest := l.readFileFirstTimeValue()
if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) {
_ = l.rotate()
var oldest time.Time
oldest, err = l.readFileFirstTimeValue()
if err != nil {
log.Debug("%s", err)
}
if oldest.Add(l.conf.RotationIvl).After(time.Now()) {
err = l.rotate()
if err != nil {
log.Debug("%s", err)
}
}
// What?
time.Sleep(24 * time.Hour)
}
}

View File

@ -37,7 +37,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
l := newQueryLog(Config{
FindClient: findClient,
BaseDir: t.TempDir(),
RotationIvl: 1,
RotationIvl: 24 * time.Hour,
MemSize: 100,
Enabled: true,
FileEnabled: true,

View File

@ -4,6 +4,16 @@
## v0.107: API changes
### New possible value of `"interval"` field in `QueryLogConfig`
* The value of `"interval"` field in `POST /control/querylog_config` and `GET
/control/querylog_info` methods could now take the value of `0.25`. It's
equal to 6 hours.
* All the possible values of `"interval"` field are enumerated.
* The type of `"interval"` field is now `number` instead of `integer`.
### Client IDs in Access Settings
* The `POST /control/access/set` HTTP API now accepts client IDs in

View File

@ -2008,8 +2008,15 @@
'type': 'boolean'
'description': 'Is query log enabled'
'interval':
'type': 'integer'
'description': 'Time period to keep data (1 | 7 | 30 | 90)'
'description': >
Time period for query log rotation.
'type': 'number'
'enum':
- 0.25
- 1
- 7
- 30
- 90
'anonymize_client_ip':
'type': 'boolean'
'description': "Anonymize clients' IP addresses"