+ config: new setting "querylog_file_enabled"

Close #876

Squashed commit of the following:

commit f83f60a7340d8a3f6de7ecfebb426e47d19e83d8
Merge: cfb72869 391e6199
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu May 28 15:17:21 2020 +0300

    Merge remote-tracking branch 'origin/master' into 876-logs

commit cfb72869f7cf0bf59a478ab8c7920c273e2fa5f9
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu May 28 12:50:02 2020 +0300

    tests

commit 35376e4f450cf66507d733c931b7ed27eff1f36c
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 18:15:12 2020 +0300

    fix

commit 0cfb802d73db52a4b09c459a68a8a18918447b76
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 16:49:52 2020 +0300

    tests

commit 03ca280b6aed3a4880a9d4f4cd18bf47b1c742f6
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed May 27 15:32:27 2020 +0300

    + config: new setting "querylog_file_enabled" - query log will be written to a file

home/config.go

@@ -78,10 +78,11 @@ type dnsConfig struct {
 	// time interval for statistics (in days)
 	StatsInterval uint32 `yaml:"statistics_interval"`
 
-	QueryLogEnabled   bool   `yaml:"querylog_enabled"`     // if true, query log is enabled
+	QueryLogEnabled     bool   `yaml:"querylog_enabled"`      // if true, query log is enabled
-	QueryLogInterval  uint32 `yaml:"querylog_interval"`    // time interval for query log (in days)
+	QueryLogFileEnabled bool   `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
-	QueryLogMemSize   uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+	QueryLogInterval    uint32 `yaml:"querylog_interval"`     // time interval for query log (in days)
-	AnonymizeClientIP bool   `yaml:"anonymize_client_ip"`  // anonymize clients' IP addresses in logs and stats
+	QueryLogMemSize     uint32 `yaml:"querylog_size_memory"`  // number of entries kept in memory before they are flushed to disk
+	AnonymizeClientIP   bool   `yaml:"anonymize_client_ip"`   // anonymize clients' IP addresses in logs and stats
 
 	dnsforward.FilteringConfig `yaml:",inline"`
 
@@ -138,6 +139,7 @@ func initConfig() {
 	config.WebSessionTTLHours = 30 * 24
 
 	config.DNS.QueryLogEnabled = true
+	config.DNS.QueryLogFileEnabled = true
 	config.DNS.QueryLogInterval = 90
 	config.DNS.QueryLogMemSize = 1000
 
@@ -239,9 +241,10 @@ func (c *configuration) write() error {
 	}
 
 	if Context.queryLog != nil {
-		dc := querylog.DiskConfig{}
+		dc := querylog.Config{}
 		Context.queryLog.WriteDiskConfig(&dc)
 		config.DNS.QueryLogEnabled = dc.Enabled
+		config.DNS.QueryLogFileEnabled = dc.FileEnabled
 		config.DNS.QueryLogInterval = dc.Interval
 		config.DNS.QueryLogMemSize = dc.MemSize
 		config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP

home/dns.go

@@ -40,6 +40,7 @@ func initDNSServer() error {
 	}
 	conf := querylog.Config{
 		Enabled:           config.DNS.QueryLogEnabled,
+		FileEnabled:       config.DNS.QueryLogFileEnabled,
 		BaseDir:           baseDir,
 		Interval:          config.DNS.QueryLogInterval,
 		MemSize:           config.DNS.QueryLogMemSize,

querylog/qlog.go

@@ -73,11 +73,8 @@ func checkInterval(days uint32) bool {
 	return days == 1 || days == 7 || days == 30 || days == 90
 }
 
-func (l *queryLog) WriteDiskConfig(dc *DiskConfig) {
+func (l *queryLog) WriteDiskConfig(c *Config) {
-	dc.Enabled = l.conf.Enabled
+	*c = *l.conf
-	dc.Interval = l.conf.Interval
-	dc.MemSize = l.conf.MemSize
-	dc.AnonymizeClientIP = l.conf.AnonymizeClientIP
 }
 
 // Clear memory buffer and remove log files
@@ -152,7 +149,14 @@ func (l *queryLog) Add(params AddParams) {
 	l.bufferLock.Lock()
 	l.buffer = append(l.buffer, &entry)
 	needFlush := false
-	if !l.flushPending {
+
+	if !l.conf.FileEnabled {
+		if len(l.buffer) > int(l.conf.MemSize) {
+			// writing to file is disabled - just remove the oldest entry from array
+			l.buffer = l.buffer[1:]
+		}
+
+	} else if !l.flushPending {
 		needFlush = len(l.buffer) >= int(l.conf.MemSize)
 		if needFlush {
 			l.flushPending = true
@@ -162,8 +166,8 @@ func (l *queryLog) Add(params AddParams) {
 
 	// if buffer needs to be flushed to disk, do it now
 	if needFlush {
-		// write to file
+		go func() {
-		// do it in separate goroutine -- we are stalling DNS response this whole time
+			_ = l.flushLogBuffer(false)
-		go l.flushLogBuffer(false) // nolint
+		}()
 	}
 }

querylog/qlog_test.go

@@ -22,9 +22,10 @@ func prepareTestDir() string {
 // Check adding and loading (with filtering) entries from disk and memory
 func TestQueryLog(t *testing.T) {
 	conf := Config{
-		Enabled:  true,
+		Enabled:     true,
-		Interval: 1,
+		FileEnabled: true,
-		MemSize:  100,
+		Interval:    1,
+		MemSize:     100,
 	}
 	conf.BaseDir = prepareTestDir()
 	defer func() { _ = os.RemoveAll(conf.BaseDir) }()
@@ -158,9 +159,10 @@ func TestQueryLogOffsetLimit(t *testing.T) {
 
 func TestQueryLogMaxFileScanEntries(t *testing.T) {
 	conf := Config{
-		Enabled:  true,
+		Enabled:     true,
-		Interval: 1,
+		FileEnabled: true,
-		MemSize:  100,
+		Interval:    1,
+		MemSize:     100,
 	}
 	conf.BaseDir = prepareTestDir()
 	defer func() { _ = os.RemoveAll(conf.BaseDir) }()
@@ -183,6 +185,29 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
 	assert.Equal(t, 10, len(entries))
 }
 
+func TestQueryLogFileDisabled(t *testing.T) {
+	conf := Config{
+		Enabled:     true,
+		FileEnabled: false,
+		Interval:    1,
+		MemSize:     2,
+	}
+	conf.BaseDir = prepareTestDir()
+	defer func() { _ = os.RemoveAll(conf.BaseDir) }()
+	l := newQueryLog(conf)
+
+	addEntry(l, "example1.org", "1.1.1.1", "2.2.2.1")
+	addEntry(l, "example2.org", "1.1.1.1", "2.2.2.1")
+	addEntry(l, "example3.org", "1.1.1.1", "2.2.2.1")
+	// the oldest entry is now removed from mem buffer
+
+	params := newSearchParams()
+	ll, _ := l.search(params)
+	assert.Equal(t, 2, len(ll))
+	assert.Equal(t, "example3.org", ll[0].QHost)
+	assert.Equal(t, "example2.org", ll[1].QHost)
+}
+
 func addEntry(l *queryLog, host, answerStr, client string) {
 	q := dns.Msg{}
 	q.Question = append(q.Question, dns.Question{

querylog/querylog.go

@@ -9,14 +9,6 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DiskConfig - configuration settings that are stored on disk
-type DiskConfig struct {
-	Enabled           bool
-	Interval          uint32
-	MemSize           uint32
-	AnonymizeClientIP bool
-}
-
 // QueryLog - main interface
 type QueryLog interface {
 	Start()
@@ -28,12 +20,13 @@ type QueryLog interface {
 	Add(params AddParams)
 
 	// WriteDiskConfig - write configuration
-	WriteDiskConfig(dc *DiskConfig)
+	WriteDiskConfig(c *Config)
 }
 
 // Config - configuration object
 type Config struct {
-	Enabled           bool
+	Enabled           bool   // enable the module
+	FileEnabled       bool   // write logs to file
 	BaseDir           string // directory where log file is stored
 	Interval          uint32 // interval to rotate logs (in days)
 	MemSize           uint32 // number of entries kept in memory before they are flushed to disk

querylog/querylog_file.go

@@ -11,6 +11,10 @@ import (
 
 // flushLogBuffer flushes the current buffer to file and resets the current buffer
 func (l *queryLog) flushLogBuffer(fullFlush bool) error {
+	if !l.conf.FileEnabled {
+		return nil
+	}
+
 	l.fileFlushLock.Lock()
 	defer l.fileFlushLock.Unlock()