Pull request: 2504 querylog interval
Merge in DNS/adguard-home from 2504-querylog-ivl to master
Updates #2504.
Squashed commit of the following:
commit 5d15a6f735cd195fc81c8af909b56fbc7db1fe21
Merge: 8cd5c30d 97073d0d
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 18:45:10 2021 +0300
Merge branch 'master' into 2504-querylog-ivl
commit 8cd5c30de6f72d4b12162dbc9e3d90132795fe94
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 18:35:50 2021 +0300
client: fix fmt
commit e95d462c31d886bacec0735acc567fec7c962149
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 17:58:06 2021 +0300
home: imp code
commit 48737b249c52a997a4f34dac45fbaf699477b007
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 17:23:18 2021 +0300
home: imp duration
commit 44f5dc3d3ada5120d74caa24cace9a253b8f15d3
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 16:55:31 2021 +0300
home: imp code, docs
commit bb2826521b7e5d248ce2ab686528219c312b8ba2
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 16:11:40 2021 +0300
all: imp code, docs
commit d688aed1f340807a8bac8807c263956b0fc16f5b
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 13:49:42 2021 +0300
all: change querylog interval setting format
This commit is contained in:
parent
97073d0d9e
commit
e113b276e7
|
@ -15,6 +15,7 @@ and this project adheres to
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- New possible value of `6h` for `querylog_interval` setting ([#2504]).
|
||||||
- Blocking access using client IDs ([#2624], [#3162]).
|
- Blocking access using client IDs ([#2624], [#3162]).
|
||||||
- `source` directives support in `/etc/network/interfaces` on Linux ([#3257]).
|
- `source` directives support in `/etc/network/interfaces` on Linux ([#3257]).
|
||||||
- RFC 9000 support in DNS-over-QUIC.
|
- RFC 9000 support in DNS-over-QUIC.
|
||||||
|
@ -40,6 +41,7 @@ and this project adheres to
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
- `querylog_interval` setting is now formatted in hours.
|
||||||
- Query log search now supports internationalized domains ([#3012]).
|
- Query log search now supports internationalized domains ([#3012]).
|
||||||
- Internationalized domains are now shown decoded in the query log with the
|
- Internationalized domains are now shown decoded in the query log with the
|
||||||
original encoded version shown in request details ([#3013]).
|
original encoded version shown in request details ([#3013]).
|
||||||
|
@ -82,6 +84,7 @@ released by then.
|
||||||
[#2439]: https://github.com/AdguardTeam/AdGuardHome/issues/2439
|
[#2439]: https://github.com/AdguardTeam/AdGuardHome/issues/2439
|
||||||
[#2441]: https://github.com/AdguardTeam/AdGuardHome/issues/2441
|
[#2441]: https://github.com/AdguardTeam/AdGuardHome/issues/2441
|
||||||
[#2443]: https://github.com/AdguardTeam/AdGuardHome/issues/2443
|
[#2443]: https://github.com/AdguardTeam/AdGuardHome/issues/2443
|
||||||
|
[#2504]: https://github.com/AdguardTeam/AdGuardHome/issues/2504
|
||||||
[#2624]: https://github.com/AdguardTeam/AdGuardHome/issues/2624
|
[#2624]: https://github.com/AdguardTeam/AdGuardHome/issues/2624
|
||||||
[#2763]: https://github.com/AdguardTeam/AdGuardHome/issues/2763
|
[#2763]: https://github.com/AdguardTeam/AdGuardHome/issues/2763
|
||||||
[#3012]: https://github.com/AdguardTeam/AdGuardHome/issues/3012
|
[#3012]: https://github.com/AdguardTeam/AdGuardHome/issues/3012
|
||||||
|
|
|
@ -484,6 +484,7 @@
|
||||||
"encryption_key_source_content": "Paste the private key contents",
|
"encryption_key_source_content": "Paste the private key contents",
|
||||||
"stats_params": "Statistics configuration",
|
"stats_params": "Statistics configuration",
|
||||||
"config_successfully_saved": "Configuration successfully saved",
|
"config_successfully_saved": "Configuration successfully saved",
|
||||||
|
"interval_6_hour": "6 hours",
|
||||||
"interval_24_hour": "24 hours",
|
"interval_24_hour": "24 hours",
|
||||||
"interval_days": "{{count}} day",
|
"interval_days": "{{count}} day",
|
||||||
"interval_days_plural": "{{count}} days",
|
"interval_days_plural": "{{count}} days",
|
||||||
|
|
|
@ -4,26 +4,33 @@ import { Field, reduxForm } from 'redux-form';
|
||||||
import { Trans, withTranslation } from 'react-i18next';
|
import { Trans, withTranslation } from 'react-i18next';
|
||||||
import flow from 'lodash/flow';
|
import flow from 'lodash/flow';
|
||||||
|
|
||||||
import { CheckboxField, renderRadioField, toNumber } from '../../../helpers/form';
|
import { CheckboxField, renderRadioField, toFloatNumber } from '../../../helpers/form';
|
||||||
import { FORM_NAME, QUERY_LOG_INTERVALS_DAYS } from '../../../helpers/constants';
|
import { FORM_NAME, QUERY_LOG_INTERVALS_DAYS } from '../../../helpers/constants';
|
||||||
import '../FormButton.css';
|
import '../FormButton.css';
|
||||||
|
|
||||||
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => {
|
const getIntervalTitle = (interval, t) => {
|
||||||
const title = interval === 1 ? t('interval_24_hour') : t('interval_days', { count: interval });
|
switch (interval) {
|
||||||
|
case 0.25:
|
||||||
|
return t('interval_6_hour');
|
||||||
|
case 1:
|
||||||
|
return t('interval_24_hour');
|
||||||
|
default:
|
||||||
|
return t('interval_days', { count: interval });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => (
|
||||||
<Field
|
<Field
|
||||||
key={interval}
|
key={interval}
|
||||||
name="interval"
|
name="interval"
|
||||||
type="radio"
|
type="radio"
|
||||||
component={renderRadioField}
|
component={renderRadioField}
|
||||||
value={interval}
|
value={interval}
|
||||||
placeholder={title}
|
placeholder={getIntervalTitle(interval, t)}
|
||||||
normalize={toNumber}
|
normalize={toNumber}
|
||||||
disabled={processing}
|
disabled={processing}
|
||||||
/>
|
/>
|
||||||
);
|
));
|
||||||
});
|
|
||||||
|
|
||||||
const Form = (props) => {
|
const Form = (props) => {
|
||||||
const {
|
const {
|
||||||
|
@ -56,7 +63,7 @@ const Form = (props) => {
|
||||||
</label>
|
</label>
|
||||||
<div className="form__group form__group--settings">
|
<div className="form__group form__group--settings">
|
||||||
<div className="custom-controls-stacked">
|
<div className="custom-controls-stacked">
|
||||||
{getIntervalFields(processing, t, toNumber)}
|
{getIntervalFields(processing, t, toFloatNumber)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="mt-5">
|
<div className="mt-5">
|
||||||
|
|
|
@ -357,7 +357,7 @@ export const NOT_FILTERED = 'NotFiltered';
|
||||||
|
|
||||||
export const STATS_INTERVALS_DAYS = [0, 1, 7, 30, 90];
|
export const STATS_INTERVALS_DAYS = [0, 1, 7, 30, 90];
|
||||||
|
|
||||||
export const QUERY_LOG_INTERVALS_DAYS = [1, 7, 30, 90];
|
export const QUERY_LOG_INTERVALS_DAYS = [0.25, 1, 7, 30, 90];
|
||||||
|
|
||||||
export const FILTERS_INTERVALS_HOURS = [0, 1, 12, 24, 72, 168];
|
export const FILTERS_INTERVALS_HOURS = [0, 1, 12, 24, 72, 168];
|
||||||
|
|
||||||
|
|
|
@ -276,6 +276,12 @@ export const ip4ToInt = (ip) => {
|
||||||
*/
|
*/
|
||||||
export const toNumber = (value) => value && parseInt(value, 10);
|
export const toNumber = (value) => value && parseInt(value, 10);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param value {string}
|
||||||
|
* @returns {*|number}
|
||||||
|
*/
|
||||||
|
export const toFloatNumber = (value) => value && parseFloat(value, 10);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param value {string}
|
* @param value {string}
|
||||||
* @returns {boolean}
|
* @returns {boolean}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
|
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
|
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
|
||||||
|
@ -104,7 +105,8 @@ type dnsConfig struct {
|
||||||
|
|
||||||
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
|
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
|
||||||
QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
|
QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
|
||||||
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
|
// QueryLogInterval is the interval for query log's files rotation.
|
||||||
|
QueryLogInterval Duration `yaml:"querylog_interval"`
|
||||||
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
|
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
|
||||||
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
|
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
|
||||||
|
|
||||||
|
@ -185,7 +187,7 @@ var config = configuration{
|
||||||
},
|
},
|
||||||
FilteringEnabled: true, // whether or not use filter lists
|
FilteringEnabled: true, // whether or not use filter lists
|
||||||
FiltersUpdateIntervalHours: 24,
|
FiltersUpdateIntervalHours: 24,
|
||||||
UpstreamTimeout: Duration{dnsforward.DefaultTimeout},
|
UpstreamTimeout: Duration{Duration: dnsforward.DefaultTimeout},
|
||||||
LocalDomainName: "lan",
|
LocalDomainName: "lan",
|
||||||
ResolveClients: true,
|
ResolveClients: true,
|
||||||
UsePrivateRDNS: true,
|
UsePrivateRDNS: true,
|
||||||
|
@ -212,7 +214,7 @@ func initConfig() {
|
||||||
|
|
||||||
config.DNS.QueryLogEnabled = true
|
config.DNS.QueryLogEnabled = true
|
||||||
config.DNS.QueryLogFileEnabled = true
|
config.DNS.QueryLogFileEnabled = true
|
||||||
config.DNS.QueryLogInterval = 90
|
config.DNS.QueryLogInterval = Duration{Duration: 90 * 24 * time.Hour}
|
||||||
config.DNS.QueryLogMemSize = 1000
|
config.DNS.QueryLogMemSize = 1000
|
||||||
|
|
||||||
config.DNS.CacheSize = 4 * 1024 * 1024
|
config.DNS.CacheSize = 4 * 1024 * 1024
|
||||||
|
@ -281,7 +283,7 @@ func parseConfig() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.DNS.UpstreamTimeout.Duration == 0 {
|
if config.DNS.UpstreamTimeout.Duration == 0 {
|
||||||
config.DNS.UpstreamTimeout = Duration{dnsforward.DefaultTimeout}
|
config.DNS.UpstreamTimeout = Duration{Duration: dnsforward.DefaultTimeout}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -328,7 +330,7 @@ func (c *configuration) write() error {
|
||||||
Context.queryLog.WriteDiskConfig(&dc)
|
Context.queryLog.WriteDiskConfig(&dc)
|
||||||
config.DNS.QueryLogEnabled = dc.Enabled
|
config.DNS.QueryLogEnabled = dc.Enabled
|
||||||
config.DNS.QueryLogFileEnabled = dc.FileEnabled
|
config.DNS.QueryLogFileEnabled = dc.FileEnabled
|
||||||
config.DNS.QueryLogInterval = dc.RotationIvl
|
config.DNS.QueryLogInterval = Duration{Duration: dc.RotationIvl}
|
||||||
config.DNS.QueryLogMemSize = dc.MemSize
|
config.DNS.QueryLogMemSize = dc.MemSize
|
||||||
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
|
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ func initDNSServer() error {
|
||||||
HTTPRegister: httpRegister,
|
HTTPRegister: httpRegister,
|
||||||
FindClient: Context.clients.findMultiple,
|
FindClient: Context.clients.findMultiple,
|
||||||
BaseDir: baseDir,
|
BaseDir: baseDir,
|
||||||
RotationIvl: config.DNS.QueryLogInterval,
|
RotationIvl: config.DNS.QueryLogInterval.Duration,
|
||||||
MemSize: config.DNS.QueryLogMemSize,
|
MemSize: config.DNS.QueryLogMemSize,
|
||||||
Enabled: config.DNS.QueryLogEnabled,
|
Enabled: config.DNS.QueryLogEnabled,
|
||||||
FileEnabled: config.DNS.QueryLogFileEnabled,
|
FileEnabled: config.DNS.QueryLogFileEnabled,
|
||||||
|
|
|
@ -12,6 +12,35 @@ type Duration struct {
|
||||||
time.Duration
|
time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// String implements the fmt.Stringer interface for Duration. It wraps
|
||||||
|
// time.Duration.String method and additionally cuts off non-leading zero values
|
||||||
|
// of minutes and seconds. Some values which are differ between the
|
||||||
|
// implementations:
|
||||||
|
//
|
||||||
|
// Duration: "1m", time.Duration: "1m0s"
|
||||||
|
// Duration: "1h", time.Duration: "1h0m0s"
|
||||||
|
// Duration: "1h1m", time.Duration: "1h1m0s"
|
||||||
|
//
|
||||||
|
func (d Duration) String() (str string) {
|
||||||
|
str = d.Duration.String()
|
||||||
|
secs := d.Seconds()
|
||||||
|
var secsInt int
|
||||||
|
if secsInt = int(secs); float64(secsInt) != secs || secsInt%60 != 0 {
|
||||||
|
return str
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
tailMin = len(`0s`)
|
||||||
|
tailMinSec = len(`0m0s`)
|
||||||
|
)
|
||||||
|
|
||||||
|
if (secsInt%3600)/60 != 0 {
|
||||||
|
return str[:len(str)-tailMin]
|
||||||
|
}
|
||||||
|
|
||||||
|
return str[:len(str)-tailMinSec]
|
||||||
|
}
|
||||||
|
|
||||||
// MarshalText implements the encoding.TextMarshaler interface for Duration.
|
// MarshalText implements the encoding.TextMarshaler interface for Duration.
|
||||||
func (d Duration) MarshalText() (text []byte, err error) {
|
func (d Duration) MarshalText() (text []byte, err error) {
|
||||||
return []byte(d.String()), nil
|
return []byte(d.String()), nil
|
||||||
|
@ -19,6 +48,8 @@ func (d Duration) MarshalText() (text []byte, err error) {
|
||||||
|
|
||||||
// UnmarshalText implements the encoding.TextUnmarshaler interface for
|
// UnmarshalText implements the encoding.TextUnmarshaler interface for
|
||||||
// *Duration.
|
// *Duration.
|
||||||
|
//
|
||||||
|
// TODO(e.burkov): Make it able to parse larger units like days.
|
||||||
func (d *Duration) UnmarshalText(b []byte) (err error) {
|
func (d *Duration) UnmarshalText(b []byte) (err error) {
|
||||||
defer func() { err = errors.Annotate(err, "unmarshalling duration: %w") }()
|
defer func() { err = errors.Annotate(err, "unmarshalling duration: %w") }()
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,50 @@ import (
|
||||||
yaml "gopkg.in/yaml.v2"
|
yaml "gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDuration_String(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
val time.Duration
|
||||||
|
}{{
|
||||||
|
name: "1s",
|
||||||
|
val: time.Second,
|
||||||
|
}, {
|
||||||
|
name: "1m",
|
||||||
|
val: time.Minute,
|
||||||
|
}, {
|
||||||
|
name: "1h",
|
||||||
|
val: time.Hour,
|
||||||
|
}, {
|
||||||
|
name: "1m1s",
|
||||||
|
val: time.Minute + time.Second,
|
||||||
|
}, {
|
||||||
|
name: "1h1m",
|
||||||
|
val: time.Hour + time.Minute,
|
||||||
|
}, {
|
||||||
|
name: "1h0m1s",
|
||||||
|
val: time.Hour + time.Second,
|
||||||
|
}, {
|
||||||
|
name: "1ms",
|
||||||
|
val: time.Millisecond,
|
||||||
|
}, {
|
||||||
|
name: "1h0m0.001s",
|
||||||
|
val: time.Hour + time.Millisecond,
|
||||||
|
}, {
|
||||||
|
name: "1.001s",
|
||||||
|
val: time.Second + time.Millisecond,
|
||||||
|
}, {
|
||||||
|
name: "1m1.001s",
|
||||||
|
val: time.Minute + time.Second + time.Millisecond,
|
||||||
|
}}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
d := Duration{Duration: tc.val}
|
||||||
|
assert.Equal(t, tc.name, d.String())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// durationEncodingTester is a helper struct to simplify testing different
|
// durationEncodingTester is a helper struct to simplify testing different
|
||||||
// Duration marshalling and unmarshalling cases.
|
// Duration marshalling and unmarshalling cases.
|
||||||
type durationEncodingTester struct {
|
type durationEncodingTester struct {
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||||
"github.com/AdguardTeam/golibs/errors"
|
"github.com/AdguardTeam/golibs/errors"
|
||||||
|
@ -19,7 +20,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// currentSchemaVersion is the current schema version.
|
// currentSchemaVersion is the current schema version.
|
||||||
const currentSchemaVersion = 11
|
const currentSchemaVersion = 12
|
||||||
|
|
||||||
// These aliases are provided for convenience.
|
// These aliases are provided for convenience.
|
||||||
type (
|
type (
|
||||||
|
@ -82,6 +83,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
|
||||||
upgradeSchema8to9,
|
upgradeSchema8to9,
|
||||||
upgradeSchema9to10,
|
upgradeSchema9to10,
|
||||||
upgradeSchema10to11,
|
upgradeSchema10to11,
|
||||||
|
upgradeSchema11to12,
|
||||||
}
|
}
|
||||||
|
|
||||||
n := 0
|
n := 0
|
||||||
|
@ -647,6 +649,46 @@ func upgradeSchema10to11(diskConf yobj) (err error) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// upgradeSchema11to12 performs the following changes:
|
||||||
|
//
|
||||||
|
// # BEFORE:
|
||||||
|
// 'querylog_interval': 90
|
||||||
|
//
|
||||||
|
// # AFTER:
|
||||||
|
// 'querylog_interval': '2160h'
|
||||||
|
//
|
||||||
|
func upgradeSchema11to12(diskConf yobj) (err error) {
|
||||||
|
log.Printf("Upgrade yaml: 11 to 12")
|
||||||
|
diskConf["schema_version"] = 12
|
||||||
|
|
||||||
|
dnsVal, ok := diskConf["dns"]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var dns yobj
|
||||||
|
dns, ok = dnsVal.(yobj)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
|
||||||
|
}
|
||||||
|
|
||||||
|
const field = "querylog_interval"
|
||||||
|
|
||||||
|
// Set the initial value from home.initConfig function.
|
||||||
|
qlogIvl := 90
|
||||||
|
qlogIvlVal, ok := dns[field]
|
||||||
|
if ok {
|
||||||
|
qlogIvl, ok = qlogIvlVal.(int)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("unexpected type of %s: %T", field, qlogIvlVal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dns[field] = Duration{Duration: time.Duration(qlogIvl) * 24 * time.Hour}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// TODO(a.garipov): Replace with log.Output when we port it to our logging
|
// TODO(a.garipov): Replace with log.Output when we port it to our logging
|
||||||
// package.
|
// package.
|
||||||
func funcName() string {
|
func funcName() string {
|
||||||
|
|
|
@ -2,6 +2,7 @@ package home
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -415,3 +416,98 @@ func TestUpgradeSchema10to11(t *testing.T) {
|
||||||
check(t, conf)
|
check(t, conf)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUpgradeSchema11to12(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
ivl any
|
||||||
|
want any
|
||||||
|
wantErr string
|
||||||
|
name string
|
||||||
|
}{{
|
||||||
|
ivl: 1,
|
||||||
|
want: Duration{Duration: 24 * time.Hour},
|
||||||
|
wantErr: "",
|
||||||
|
name: "success",
|
||||||
|
}, {
|
||||||
|
ivl: 0.25,
|
||||||
|
want: 0,
|
||||||
|
wantErr: "unexpected type of querylog_interval: float64",
|
||||||
|
name: "fail",
|
||||||
|
}}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
conf := yobj{
|
||||||
|
"dns": yobj{
|
||||||
|
"querylog_interval": tc.ivl,
|
||||||
|
},
|
||||||
|
"schema_version": 11,
|
||||||
|
}
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := upgradeSchema11to12(conf)
|
||||||
|
|
||||||
|
if tc.wantErr != "" {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, tc.wantErr, err.Error())
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, conf["schema_version"], 12)
|
||||||
|
|
||||||
|
dnsVal, ok := conf["dns"]
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
var newDNSConf yobj
|
||||||
|
newDNSConf, ok = dnsVal.(yobj)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
var newIvl Duration
|
||||||
|
newIvl, ok = newDNSConf["querylog_interval"].(Duration)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
assert.Equal(t, tc.want, newIvl)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("no_dns", func(t *testing.T) {
|
||||||
|
err := upgradeSchema11to12(yobj{})
|
||||||
|
|
||||||
|
assert.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("bad_dns", func(t *testing.T) {
|
||||||
|
err := upgradeSchema11to12(yobj{
|
||||||
|
"dns": 0,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, "unexpected type of dns: int", err.Error())
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no_field", func(t *testing.T) {
|
||||||
|
conf := yobj{
|
||||||
|
"dns": yobj{},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := upgradeSchema11to12(conf)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
dns, ok := conf["dns"]
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
var dnsVal yobj
|
||||||
|
dnsVal, ok = dns.(yobj)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
var ivl interface{}
|
||||||
|
ivl, ok = dnsVal["querylog_interval"]
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
var ivlVal Duration
|
||||||
|
ivlVal, ok = ivl.(Duration)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
assert.Equal(t, 90*24*time.Hour, ivlVal.Duration)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
@ -17,7 +17,9 @@ import (
|
||||||
|
|
||||||
type qlogConfig struct {
|
type qlogConfig struct {
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
Interval uint32 `json:"interval"`
|
// Use float64 here to support fractional numbers and not mess the API
|
||||||
|
// users by changing the units.
|
||||||
|
Interval float64 `json:"interval"`
|
||||||
AnonymizeClientIP bool `json:"anonymize_client_ip"`
|
AnonymizeClientIP bool `json:"anonymize_client_ip"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,7 +73,7 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
|
||||||
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
|
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
|
||||||
resp := qlogConfig{}
|
resp := qlogConfig{}
|
||||||
resp.Enabled = l.conf.Enabled
|
resp.Enabled = l.conf.Enabled
|
||||||
resp.Interval = l.conf.RotationIvl
|
resp.Interval = l.conf.RotationIvl.Hours() / 24
|
||||||
resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
|
resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
|
||||||
|
|
||||||
jsonVal, err := json.Marshal(resp)
|
jsonVal, err := json.Marshal(resp)
|
||||||
|
@ -95,7 +97,8 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.Exists("interval") && !checkInterval(d.Interval) {
|
ivl := time.Duration(24*d.Interval) * time.Hour
|
||||||
|
if req.Exists("interval") && !checkInterval(ivl) {
|
||||||
httpError(r, w, http.StatusBadRequest, "Unsupported interval")
|
httpError(r, w, http.StatusBadRequest, "Unsupported interval")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -107,7 +110,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||||
conf.Enabled = d.Enabled
|
conf.Enabled = d.Enabled
|
||||||
}
|
}
|
||||||
if req.Exists("interval") {
|
if req.Exists("interval") {
|
||||||
conf.RotationIvl = d.Interval
|
conf.RotationIvl = ivl
|
||||||
}
|
}
|
||||||
if req.Exists("anonymize_client_ip") {
|
if req.Exists("anonymize_client_ip") {
|
||||||
conf.AnonymizeClientIP = d.AnonymizeClientIP
|
conf.AnonymizeClientIP = d.AnonymizeClientIP
|
||||||
|
|
|
@ -100,8 +100,17 @@ func (l *queryLog) Close() {
|
||||||
_ = l.flushLogBuffer(true)
|
_ = l.flushLogBuffer(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkInterval(days uint32) bool {
|
func checkInterval(ivl time.Duration) (ok bool) {
|
||||||
return days == 1 || days == 7 || days == 30 || days == 90
|
// The constants for possible values of query log's rotation interval.
|
||||||
|
const (
|
||||||
|
quarterDay = 6 * time.Hour
|
||||||
|
day = 24 * time.Hour
|
||||||
|
week = day * 7
|
||||||
|
month = day * 30
|
||||||
|
threeMonths = day * 90
|
||||||
|
)
|
||||||
|
|
||||||
|
return ivl == quarterDay || ivl == day || ivl == week || ivl == month || ivl == threeMonths
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *queryLog) WriteDiskConfig(c *Config) {
|
func (l *queryLog) WriteDiskConfig(c *Config) {
|
||||||
|
|
|
@ -26,7 +26,7 @@ func TestQueryLog(t *testing.T) {
|
||||||
l := newQueryLog(Config{
|
l := newQueryLog(Config{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
FileEnabled: true,
|
FileEnabled: true,
|
||||||
RotationIvl: 1,
|
RotationIvl: 24 * time.Hour,
|
||||||
MemSize: 100,
|
MemSize: 100,
|
||||||
BaseDir: t.TempDir(),
|
BaseDir: t.TempDir(),
|
||||||
})
|
})
|
||||||
|
@ -128,7 +128,7 @@ func TestQueryLog(t *testing.T) {
|
||||||
func TestQueryLogOffsetLimit(t *testing.T) {
|
func TestQueryLogOffsetLimit(t *testing.T) {
|
||||||
l := newQueryLog(Config{
|
l := newQueryLog(Config{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
RotationIvl: 1,
|
RotationIvl: 24 * time.Hour,
|
||||||
MemSize: 100,
|
MemSize: 100,
|
||||||
BaseDir: t.TempDir(),
|
BaseDir: t.TempDir(),
|
||||||
})
|
})
|
||||||
|
@ -153,33 +153,34 @@ func TestQueryLogOffsetLimit(t *testing.T) {
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
name string
|
name string
|
||||||
|
want string
|
||||||
|
wantLen int
|
||||||
offset int
|
offset int
|
||||||
limit int
|
limit int
|
||||||
wantLen int
|
|
||||||
want string
|
|
||||||
}{{
|
}{{
|
||||||
name: "page_1",
|
name: "page_1",
|
||||||
|
want: firstPageDomain,
|
||||||
|
wantLen: 10,
|
||||||
offset: 0,
|
offset: 0,
|
||||||
limit: 10,
|
limit: 10,
|
||||||
wantLen: 10,
|
|
||||||
want: firstPageDomain,
|
|
||||||
}, {
|
}, {
|
||||||
name: "page_2",
|
name: "page_2",
|
||||||
|
want: secondPageDomain,
|
||||||
|
wantLen: 10,
|
||||||
offset: 10,
|
offset: 10,
|
||||||
limit: 10,
|
limit: 10,
|
||||||
wantLen: 10,
|
|
||||||
want: secondPageDomain,
|
|
||||||
}, {
|
}, {
|
||||||
name: "page_2.5",
|
name: "page_2.5",
|
||||||
|
want: secondPageDomain,
|
||||||
|
wantLen: 5,
|
||||||
offset: 15,
|
offset: 15,
|
||||||
limit: 10,
|
limit: 10,
|
||||||
wantLen: 5,
|
|
||||||
want: secondPageDomain,
|
|
||||||
}, {
|
}, {
|
||||||
name: "page_3",
|
name: "page_3",
|
||||||
|
want: "",
|
||||||
|
wantLen: 0,
|
||||||
offset: 20,
|
offset: 20,
|
||||||
limit: 10,
|
limit: 10,
|
||||||
wantLen: 0,
|
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
|
@ -202,7 +203,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
|
||||||
l := newQueryLog(Config{
|
l := newQueryLog(Config{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
FileEnabled: true,
|
FileEnabled: true,
|
||||||
RotationIvl: 1,
|
RotationIvl: 24 * time.Hour,
|
||||||
MemSize: 100,
|
MemSize: 100,
|
||||||
BaseDir: t.TempDir(),
|
BaseDir: t.TempDir(),
|
||||||
})
|
})
|
||||||
|
@ -230,7 +231,7 @@ func TestQueryLogFileDisabled(t *testing.T) {
|
||||||
l := newQueryLog(Config{
|
l := newQueryLog(Config{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
FileEnabled: false,
|
FileEnabled: false,
|
||||||
RotationIvl: 1,
|
RotationIvl: 24 * time.Hour,
|
||||||
MemSize: 2,
|
MemSize: 2,
|
||||||
BaseDir: t.TempDir(),
|
BaseDir: t.TempDir(),
|
||||||
})
|
})
|
||||||
|
|
|
@ -41,10 +41,17 @@ type Config struct {
|
||||||
// BaseDir is the base directory for log files.
|
// BaseDir is the base directory for log files.
|
||||||
BaseDir string
|
BaseDir string
|
||||||
|
|
||||||
// RotationIvl is the interval for log rotation, in days. After that
|
// RotationIvl is the interval for log rotation. After that period, the
|
||||||
// period, the old log file will be renamed, NOT deleted, so the actual
|
// old log file will be renamed, NOT deleted, so the actual log
|
||||||
// log retention time is twice the interval.
|
// retention time is twice the interval. The value must be one of:
|
||||||
RotationIvl uint32
|
//
|
||||||
|
// 6 * time.Hour
|
||||||
|
// 24 * time.Hour
|
||||||
|
// 7 * 24 * time.Hour
|
||||||
|
// 30 * 24 * time.Hour
|
||||||
|
// 90 * 24 * time.Hour
|
||||||
|
//
|
||||||
|
RotationIvl time.Duration
|
||||||
|
|
||||||
// MemSize is the number of entries kept in a memory buffer before they
|
// MemSize is the number of entries kept in a memory buffer before they
|
||||||
// are flushed to disk.
|
// are flushed to disk.
|
||||||
|
@ -118,7 +125,7 @@ func newQueryLog(conf Config) (l *queryLog) {
|
||||||
"querylog: warning: unsupported rotation interval %d, setting to 1 day",
|
"querylog: warning: unsupported rotation interval %d, setting to 1 day",
|
||||||
conf.RotationIvl,
|
conf.RotationIvl,
|
||||||
)
|
)
|
||||||
l.conf.RotationIvl = 1
|
l.conf.RotationIvl = 24 * time.Hour
|
||||||
}
|
}
|
||||||
|
|
||||||
return l
|
return l
|
||||||
|
|
|
@ -104,43 +104,52 @@ func (l *queryLog) rotate() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *queryLog) readFileFirstTimeValue() int64 {
|
func (l *queryLog) readFileFirstTimeValue() (first time.Time, err error) {
|
||||||
f, err := os.Open(l.logFile)
|
var f *os.File
|
||||||
|
f, err = os.Open(l.logFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return -1
|
return time.Time{}, err
|
||||||
}
|
}
|
||||||
defer func() {
|
|
||||||
derr := f.Close()
|
|
||||||
if derr != nil {
|
|
||||||
log.Error("querylog: closing file: %s", derr)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
buf := make([]byte, 500)
|
defer func() { err = errors.WithDeferred(err, f.Close()) }()
|
||||||
r, err := f.Read(buf)
|
|
||||||
|
buf := make([]byte, 512)
|
||||||
|
var r int
|
||||||
|
r, err = f.Read(buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return -1
|
return time.Time{}, err
|
||||||
}
|
}
|
||||||
buf = buf[:r]
|
|
||||||
|
|
||||||
val := readJSONValue(string(buf), `"T":"`)
|
val := readJSONValue(string(buf[:r]), `"T":"`)
|
||||||
t, err := time.Parse(time.RFC3339Nano, val)
|
t, err := time.Parse(time.RFC3339Nano, val)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return -1
|
return time.Time{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("querylog: the oldest log entry: %s", val)
|
log.Debug("querylog: the oldest log entry: %s", val)
|
||||||
return t.Unix()
|
|
||||||
|
return t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *queryLog) periodicRotate() {
|
func (l *queryLog) periodicRotate() {
|
||||||
intervalSeconds := uint64(l.conf.RotationIvl) * 24 * 60 * 60
|
defer log.OnPanic("querylog: rotating")
|
||||||
|
|
||||||
|
var err error
|
||||||
for {
|
for {
|
||||||
oldest := l.readFileFirstTimeValue()
|
var oldest time.Time
|
||||||
if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) {
|
oldest, err = l.readFileFirstTimeValue()
|
||||||
_ = l.rotate()
|
if err != nil {
|
||||||
|
log.Debug("%s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if oldest.Add(l.conf.RotationIvl).After(time.Now()) {
|
||||||
|
err = l.rotate()
|
||||||
|
if err != nil {
|
||||||
|
log.Debug("%s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What?
|
||||||
time.Sleep(24 * time.Hour)
|
time.Sleep(24 * time.Hour)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,7 +37,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
|
||||||
l := newQueryLog(Config{
|
l := newQueryLog(Config{
|
||||||
FindClient: findClient,
|
FindClient: findClient,
|
||||||
BaseDir: t.TempDir(),
|
BaseDir: t.TempDir(),
|
||||||
RotationIvl: 1,
|
RotationIvl: 24 * time.Hour,
|
||||||
MemSize: 100,
|
MemSize: 100,
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
FileEnabled: true,
|
FileEnabled: true,
|
||||||
|
|
|
@ -4,6 +4,16 @@
|
||||||
|
|
||||||
## v0.107: API changes
|
## v0.107: API changes
|
||||||
|
|
||||||
|
### New possible value of `"interval"` field in `QueryLogConfig`
|
||||||
|
|
||||||
|
* The value of `"interval"` field in `POST /control/querylog_config` and `GET
|
||||||
|
/control/querylog_info` methods could now take the value of `0.25`. It's
|
||||||
|
equal to 6 hours.
|
||||||
|
|
||||||
|
* All the possible values of `"interval"` field are enumerated.
|
||||||
|
|
||||||
|
* The type of `"interval"` field is now `number` instead of `integer`.
|
||||||
|
|
||||||
### Client IDs in Access Settings
|
### Client IDs in Access Settings
|
||||||
|
|
||||||
* The `POST /control/access/set` HTTP API now accepts client IDs in
|
* The `POST /control/access/set` HTTP API now accepts client IDs in
|
||||||
|
|
|
@ -2008,8 +2008,15 @@
|
||||||
'type': 'boolean'
|
'type': 'boolean'
|
||||||
'description': 'Is query log enabled'
|
'description': 'Is query log enabled'
|
||||||
'interval':
|
'interval':
|
||||||
'type': 'integer'
|
'description': >
|
||||||
'description': 'Time period to keep data (1 | 7 | 30 | 90)'
|
Time period for query log rotation.
|
||||||
|
'type': 'number'
|
||||||
|
'enum':
|
||||||
|
- 0.25
|
||||||
|
- 1
|
||||||
|
- 7
|
||||||
|
- 30
|
||||||
|
- 90
|
||||||
'anonymize_client_ip':
|
'anonymize_client_ip':
|
||||||
'type': 'boolean'
|
'type': 'boolean'
|
||||||
'description': "Anonymize clients' IP addresses"
|
'description': "Anonymize clients' IP addresses"
|
||||||
|
|
Loading…
Reference in New Issue