From 32d1f385ff4252fdda5d81928caaa09052b62451 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 28 May 2020 15:29:36 +0300 Subject: [PATCH] + config: new setting "querylog_file_enabled" Close #876 Squashed commit of the following: commit f83f60a7340d8a3f6de7ecfebb426e47d19e83d8 Merge: cfb72869 391e6199 Author: Simon Zolin Date: Thu May 28 15:17:21 2020 +0300 Merge remote-tracking branch 'origin/master' into 876-logs commit cfb72869f7cf0bf59a478ab8c7920c273e2fa5f9 Author: Simon Zolin Date: Thu May 28 12:50:02 2020 +0300 tests commit 35376e4f450cf66507d733c931b7ed27eff1f36c Author: Simon Zolin Date: Wed May 27 18:15:12 2020 +0300 fix commit 0cfb802d73db52a4b09c459a68a8a18918447b76 Author: Simon Zolin Date: Wed May 27 16:49:52 2020 +0300 tests commit 03ca280b6aed3a4880a9d4f4cd18bf47b1c742f6 Author: Simon Zolin Date: Wed May 27 15:32:27 2020 +0300 + config: new setting "querylog_file_enabled" - query log will be written to a file --- home/config.go | 13 ++++++++----- home/dns.go | 1 + querylog/qlog.go | 22 +++++++++++++--------- querylog/qlog_test.go | 37 +++++++++++++++++++++++++++++++------ querylog/querylog.go | 13 +++---------- querylog/querylog_file.go | 4 ++++ 6 files changed, 60 insertions(+), 30 deletions(-) diff --git a/home/config.go b/home/config.go index 35940ebd..f453a732 100644 --- a/home/config.go +++ b/home/config.go @@ -78,10 +78,11 @@ type dnsConfig struct { // time interval for statistics (in days) StatsInterval uint32 `yaml:"statistics_interval"` - QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled - QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) - QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk - AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats + QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled + QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file + QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) + QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk + AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats dnsforward.FilteringConfig `yaml:",inline"` @@ -138,6 +139,7 @@ func initConfig() { config.WebSessionTTLHours = 30 * 24 config.DNS.QueryLogEnabled = true + config.DNS.QueryLogFileEnabled = true config.DNS.QueryLogInterval = 90 config.DNS.QueryLogMemSize = 1000 @@ -239,9 +241,10 @@ func (c *configuration) write() error { } if Context.queryLog != nil { - dc := querylog.DiskConfig{} + dc := querylog.Config{} Context.queryLog.WriteDiskConfig(&dc) config.DNS.QueryLogEnabled = dc.Enabled + config.DNS.QueryLogFileEnabled = dc.FileEnabled config.DNS.QueryLogInterval = dc.Interval config.DNS.QueryLogMemSize = dc.MemSize config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP diff --git a/home/dns.go b/home/dns.go index 36ee9ec1..db05f99a 100644 --- a/home/dns.go +++ b/home/dns.go @@ -40,6 +40,7 @@ func initDNSServer() error { } conf := querylog.Config{ Enabled: config.DNS.QueryLogEnabled, + FileEnabled: config.DNS.QueryLogFileEnabled, BaseDir: baseDir, Interval: config.DNS.QueryLogInterval, MemSize: config.DNS.QueryLogMemSize, diff --git a/querylog/qlog.go b/querylog/qlog.go index e85da3b9..27bcc01e 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -73,11 +73,8 @@ func checkInterval(days uint32) bool { return days == 1 || days == 7 || days == 30 || days == 90 } -func (l *queryLog) WriteDiskConfig(dc *DiskConfig) { - dc.Enabled = l.conf.Enabled - dc.Interval = l.conf.Interval - dc.MemSize = l.conf.MemSize - dc.AnonymizeClientIP = l.conf.AnonymizeClientIP +func (l *queryLog) WriteDiskConfig(c *Config) { + *c = *l.conf } // Clear memory buffer and remove log files @@ -152,7 +149,14 @@ func (l *queryLog) Add(params AddParams) { l.bufferLock.Lock() l.buffer = append(l.buffer, &entry) needFlush := false - if !l.flushPending { + + if !l.conf.FileEnabled { + if len(l.buffer) > int(l.conf.MemSize) { + // writing to file is disabled - just remove the oldest entry from array + l.buffer = l.buffer[1:] + } + + } else if !l.flushPending { needFlush = len(l.buffer) >= int(l.conf.MemSize) if needFlush { l.flushPending = true @@ -162,8 +166,8 @@ func (l *queryLog) Add(params AddParams) { // if buffer needs to be flushed to disk, do it now if needFlush { - // write to file - // do it in separate goroutine -- we are stalling DNS response this whole time - go l.flushLogBuffer(false) // nolint + go func() { + _ = l.flushLogBuffer(false) + }() } } diff --git a/querylog/qlog_test.go b/querylog/qlog_test.go index a5392fc0..921d2870 100644 --- a/querylog/qlog_test.go +++ b/querylog/qlog_test.go @@ -22,9 +22,10 @@ func prepareTestDir() string { // Check adding and loading (with filtering) entries from disk and memory func TestQueryLog(t *testing.T) { conf := Config{ - Enabled: true, - Interval: 1, - MemSize: 100, + Enabled: true, + FileEnabled: true, + Interval: 1, + MemSize: 100, } conf.BaseDir = prepareTestDir() defer func() { _ = os.RemoveAll(conf.BaseDir) }() @@ -158,9 +159,10 @@ func TestQueryLogOffsetLimit(t *testing.T) { func TestQueryLogMaxFileScanEntries(t *testing.T) { conf := Config{ - Enabled: true, - Interval: 1, - MemSize: 100, + Enabled: true, + FileEnabled: true, + Interval: 1, + MemSize: 100, } conf.BaseDir = prepareTestDir() defer func() { _ = os.RemoveAll(conf.BaseDir) }() @@ -183,6 +185,29 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) { assert.Equal(t, 10, len(entries)) } +func TestQueryLogFileDisabled(t *testing.T) { + conf := Config{ + Enabled: true, + FileEnabled: false, + Interval: 1, + MemSize: 2, + } + conf.BaseDir = prepareTestDir() + defer func() { _ = os.RemoveAll(conf.BaseDir) }() + l := newQueryLog(conf) + + addEntry(l, "example1.org", "1.1.1.1", "2.2.2.1") + addEntry(l, "example2.org", "1.1.1.1", "2.2.2.1") + addEntry(l, "example3.org", "1.1.1.1", "2.2.2.1") + // the oldest entry is now removed from mem buffer + + params := newSearchParams() + ll, _ := l.search(params) + assert.Equal(t, 2, len(ll)) + assert.Equal(t, "example3.org", ll[0].QHost) + assert.Equal(t, "example2.org", ll[1].QHost) +} + func addEntry(l *queryLog, host, answerStr, client string) { q := dns.Msg{} q.Question = append(q.Question, dns.Question{ diff --git a/querylog/querylog.go b/querylog/querylog.go index 0e079ec3..2fa7ac9d 100644 --- a/querylog/querylog.go +++ b/querylog/querylog.go @@ -9,14 +9,6 @@ import ( "github.com/miekg/dns" ) -// DiskConfig - configuration settings that are stored on disk -type DiskConfig struct { - Enabled bool - Interval uint32 - MemSize uint32 - AnonymizeClientIP bool -} - // QueryLog - main interface type QueryLog interface { Start() @@ -28,12 +20,13 @@ type QueryLog interface { Add(params AddParams) // WriteDiskConfig - write configuration - WriteDiskConfig(dc *DiskConfig) + WriteDiskConfig(c *Config) } // Config - configuration object type Config struct { - Enabled bool + Enabled bool // enable the module + FileEnabled bool // write logs to file BaseDir string // directory where log file is stored Interval uint32 // interval to rotate logs (in days) MemSize uint32 // number of entries kept in memory before they are flushed to disk diff --git a/querylog/querylog_file.go b/querylog/querylog_file.go index 1eeeea7c..e1544fff 100644 --- a/querylog/querylog_file.go +++ b/querylog/querylog_file.go @@ -11,6 +11,10 @@ import ( // flushLogBuffer flushes the current buffer to file and resets the current buffer func (l *queryLog) flushLogBuffer(fullFlush bool) error { + if !l.conf.FileEnabled { + return nil + } + l.fileFlushLock.Lock() defer l.fileFlushLock.Unlock()