* querylog: move code to a separate package

+ config: "querylog_interval" setting
/control/querylog_config, /control/querylog_info
+ POST /control/querylog_clear

AGHTechDoc.md

@@ -42,6 +42,9 @@ Contents:
 	* API: Clear statistics data
 	* API: Set statistics parameters
 	* API: Get statistics parameters
+* Query logs
+	* API: Set querylog parameters
+	* API: Get querylog parameters
 
 
 ## First startup
@@ -976,3 +979,37 @@ Response:
 	{
 		"interval": 1 | 7 | 30 | 90
 	}
+
+
+## Query logs
+
+### API: Set querylog parameters
+
+Request:
+
+	POST /control/querylog_config
+
+	{
+		"enabled": true | false
+		"interval": 1 | 7 | 30 | 90
+	}
+
+Response:
+
+	200 OK
+
+
+### API: Get querylog parameters
+
+Request:
+
+	GET /control/querylog_info
+
+Response:
+
+	200 OK
+
+	{
+		"enabled": true | false
+		"interval": 1 | 7 | 30 | 90
+	}

dnsforward/dnsforward.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
+	"github.com/AdguardTeam/AdGuardHome/querylog"
 	"github.com/AdguardTeam/AdGuardHome/stats"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
@@ -40,7 +41,7 @@ const (
 type Server struct {
 	dnsProxy  *proxy.Proxy         // DNS proxy instance
 	dnsFilter *dnsfilter.Dnsfilter // DNS filter instance
-	queryLog  *queryLog            // Query log instance
+	queryLog  querylog.QueryLog    // Query log instance
 	stats     stats.Stats
 
 	AllowedClients         map[string]bool // IP addresses of whitelist clients
@@ -54,16 +55,11 @@ type Server struct {
 }
 
 // NewServer creates a new instance of the dnsforward.Server
-// baseDir is the base directory for query logs
 // Note: this function must be called only once
-func NewServer(baseDir string, stats stats.Stats) *Server {
+func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
-	s := &Server{
+	s := &Server{}
-		queryLog: newQueryLog(baseDir),
-	}
 	s.stats = stats
-
+	s.queryLog = queryLog
-	log.Printf("Start DNS server periodic jobs")
-	go s.queryLog.periodicQueryLogRotate()
 	return s
 }
 
@@ -75,6 +71,7 @@ type FilteringConfig struct {
 	BlockingMode       string   `yaml:"blocking_mode"`        // mode how to answer filtered requests
 	BlockedResponseTTL uint32   `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
 	QueryLogEnabled    bool     `yaml:"querylog_enabled"`     // if true, query log is enabled
+	QueryLogInterval   uint32   `yaml:"querylog_interval"`    // time interval for query log (in days)
 	Ratelimit          int      `yaml:"ratelimit"`            // max number of requests per second from a given IP (0 to disable)
 	RatelimitWhitelist []string `yaml:"ratelimit_whitelist"`  // a list of whitelisted client IP addresses
 	RefuseAny          bool     `yaml:"refuse_any"`           // if true, refuse ANY requests
@@ -303,8 +300,7 @@ func (s *Server) stopInternal() error {
 		s.dnsFilter = nil
 	}
 
-	// flush remainder to file
+	return nil
-	return s.queryLog.flushLogBuffer(true)
 }
 
 // IsRunning returns true if the DNS server is running
@@ -343,13 +339,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	s.RUnlock()
 }
 
-// GetQueryLog returns a map with the current query log ready to be converted to a JSON
-func (s *Server) GetQueryLog() []map[string]interface{} {
-	s.RLock()
-	defer s.RUnlock()
-	return s.queryLog.getQueryLog()
-}
-
 // Return TRUE if this client should be blocked
 func (s *Server) isBlockedIP(ip string) bool {
 	if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 {
@@ -469,12 +458,12 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
 	}
 
 	elapsed := time.Since(start)
-	if s.conf.QueryLogEnabled && shouldLog {
+	if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil {
 		upstreamAddr := ""
 		if d.Upstream != nil {
 			upstreamAddr = d.Upstream.Address()
 		}
-		_ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
+		s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
 	}
 
 	s.updateStats(d, elapsed, *res)

dnsforward/dnsforward_test.go

@@ -10,7 +10,6 @@ import (
 	"encoding/pem"
 	"math/big"
 	"net"
-	"os"
 	"sync"
 	"testing"
 	"time"
@@ -18,18 +17,15 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/miekg/dns"
-	"github.com/stretchr/testify/assert"
 )
 
 const (
 	tlsServerName     = "testdns.adguard.com"
-	dataDir           = "testData"
 	testMessagesCount = 10
 )
 
 func TestServer(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -45,10 +41,6 @@ func TestServer(t *testing.T) {
 	}
 	assertGoogleAResponse(t, reply)
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	// message over TCP
 	req = createGoogleATestMessage()
 	addr = s.dnsProxy.Addr("tcp")
@@ -59,10 +51,6 @@ func TestServer(t *testing.T) {
 	}
 	assertGoogleAResponse(t, reply)
 
-	// check query log and stats again
-	log = s.GetQueryLog()
-	assert.Equal(t, 2, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -72,7 +60,6 @@ func TestServer(t *testing.T) {
 func TestServerWithProtectionDisabled(t *testing.T) {
 	s := createTestServer(t)
 	s.conf.ProtectionEnabled = false
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -88,10 +75,6 @@ func TestServerWithProtectionDisabled(t *testing.T) {
 	}
 	assertGoogleAResponse(t, reply)
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -102,7 +85,6 @@ func TestDotServer(t *testing.T) {
 	// Prepare the proxy server
 	_, certPem, keyPem := createServerTLSConfig(t)
 	s := createTestServer(t)
-	defer removeDataDir(t)
 
 	s.conf.TLSConfig = TLSConfig{
 		TLSListenAddr:        &net.TCPAddr{Port: 0},
@@ -143,7 +125,6 @@ func TestDotServer(t *testing.T) {
 
 func TestServerRace(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -168,7 +149,6 @@ func TestServerRace(t *testing.T) {
 func TestSafeSearch(t *testing.T) {
 	s := createTestServer(t)
 	s.conf.SafeSearchEnabled = true
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -210,7 +190,6 @@ func TestSafeSearch(t *testing.T) {
 
 func TestInvalidRequest(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -229,11 +208,6 @@ func TestInvalidRequest(t *testing.T) {
 		t.Fatalf("got a response to an invalid query")
 	}
 
-	// check query log and stats
-	// invalid requests aren't written to the query log
-	log := s.GetQueryLog()
-	assert.Equal(t, 0, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -242,7 +216,6 @@ func TestInvalidRequest(t *testing.T) {
 
 func TestBlockedRequest(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -267,10 +240,6 @@ func TestBlockedRequest(t *testing.T) {
 		t.Fatalf("Wrong response: %s", reply.String())
 	}
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -280,7 +249,6 @@ func TestBlockedRequest(t *testing.T) {
 func TestNullBlockedRequest(t *testing.T) {
 	s := createTestServer(t)
 	s.conf.FilteringConfig.BlockingMode = "null_ip"
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -312,10 +280,6 @@ func TestNullBlockedRequest(t *testing.T) {
 		t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
 	}
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -324,7 +288,6 @@ func TestNullBlockedRequest(t *testing.T) {
 
 func TestBlockedByHosts(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -356,10 +319,6 @@ func TestBlockedByHosts(t *testing.T) {
 		t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
 	}
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -368,7 +327,6 @@ func TestBlockedByHosts(t *testing.T) {
 
 func TestBlockedBySafeBrowsing(t *testing.T) {
 	s := createTestServer(t)
-	defer removeDataDir(t)
 	err := s.Start(nil)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
@@ -411,10 +369,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
 		t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
 	}
 
-	// check query log and stats
-	log := s.GetQueryLog()
-	assert.Equal(t, 1, len(log), "Log size")
-
 	err = s.Stop()
 	if err != nil {
 		t.Fatalf("DNS server failed to stop: %s", err)
@@ -422,7 +376,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
 }
 
 func createTestServer(t *testing.T) *Server {
-	s := NewServer(createDataDir(t), nil)
+	s := NewServer(nil, nil)
 	s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
 	s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
 
@@ -489,21 +443,6 @@ func createServerTLSConfig(t *testing.T) (*tls.Config, []byte, []byte) {
 	return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem
 }
 
-func createDataDir(t *testing.T) string {
-	err := os.MkdirAll(dataDir, 0755)
-	if err != nil {
-		t.Fatalf("Cannot create %s: %s", dataDir, err)
-	}
-	return dataDir
-}
-
-func removeDataDir(t *testing.T) {
-	err := os.RemoveAll(dataDir)
-	if err != nil {
-		t.Fatalf("Cannot remove %s: %s", dataDir, err)
-	}
-}
-
 func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) {
 	defer func() {
 		g.Done()
@@ -607,7 +546,6 @@ func TestIsBlockedIPAllowed(t *testing.T) {
 	s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
 
 	err := s.Start(nil)
-	defer removeDataDir(t)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
 	}
@@ -631,7 +569,6 @@ func TestIsBlockedIPDisallowed(t *testing.T) {
 	s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
 
 	err := s.Start(nil)
-	defer removeDataDir(t)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
 	}
@@ -655,7 +592,6 @@ func TestIsBlockedIPBlockedDomain(t *testing.T) {
 	s.conf.BlockedHosts = []string{"host1", "host2"}
 
 	err := s.Start(nil)
-	defer removeDataDir(t)
 	if err != nil {
 		t.Fatalf("Failed to start server: %s", err)
 	}

home/config.go

@@ -12,6 +12,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/dhcpd"
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
+	"github.com/AdguardTeam/AdGuardHome/querylog"
 	"github.com/AdguardTeam/AdGuardHome/stats"
 	"github.com/AdguardTeam/golibs/file"
 	"github.com/AdguardTeam/golibs/log"
@@ -70,6 +71,7 @@ type configuration struct {
 	transport        *http.Transport
 	client           *http.Client
 	stats            stats.Stats
+	queryLog         querylog.QueryLog
 
 	// cached version.json to avoid hammering github.io for each page reload
 	versionCheckJSON     []byte
@@ -175,6 +177,7 @@ var config = configuration{
 			BlockingMode:       "nxdomain", // mode how to answer filtered requests
 			BlockedResponseTTL: 10,         // in seconds
 			QueryLogEnabled:    true,
+			QueryLogInterval:   1,
 			Ratelimit:          20,
 			RefuseAny:          true,
 			BootstrapDNS:       defaultBootstrap,
@@ -274,6 +277,10 @@ func parseConfig() error {
 		config.DNS.StatsInterval = 1
 	}
 
+	if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
+		config.DNS.QueryLogInterval = 1
+	}
+
 	for _, cy := range config.Clients {
 		cli := Client{
 			Name:                cy.Name,

home/control.go

@@ -146,35 +146,6 @@ func handleProtectionDisable(w http.ResponseWriter, r *http.Request) {
 	httpUpdateConfigReloadDNSReturnOK(w, r)
 }
 
-// -----
-// stats
-// -----
-func handleQueryLogEnable(w http.ResponseWriter, r *http.Request) {
-	config.DNS.QueryLogEnabled = true
-	httpUpdateConfigReloadDNSReturnOK(w, r)
-}
-
-func handleQueryLogDisable(w http.ResponseWriter, r *http.Request) {
-	config.DNS.QueryLogEnabled = false
-	httpUpdateConfigReloadDNSReturnOK(w, r)
-}
-
-func handleQueryLog(w http.ResponseWriter, r *http.Request) {
-	data := config.dnsServer.GetQueryLog()
-
-	jsonVal, err := json.Marshal(data)
-	if err != nil {
-		httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	_, err = w.Write(jsonVal)
-	if err != nil {
-		httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
-	}
-}
-
 // -----------------------
 // upstreams configuration
 // -----------------------
@@ -570,9 +541,6 @@ func registerControlHandlers() {
 	httpRegister(http.MethodGet, "/control/status", handleStatus)
 	httpRegister(http.MethodPost, "/control/enable_protection", handleProtectionEnable)
 	httpRegister(http.MethodPost, "/control/disable_protection", handleProtectionDisable)
-	httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
-	httpRegister(http.MethodPost, "/control/querylog_enable", handleQueryLogEnable)
-	httpRegister(http.MethodPost, "/control/querylog_disable", handleQueryLogDisable)
 	httpRegister(http.MethodPost, "/control/set_upstreams_config", handleSetUpstreamConfig)
 	httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS)
 	httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage)
@@ -611,6 +579,7 @@ func registerControlHandlers() {
 	RegisterClientsHandlers()
 	registerRewritesHandlers()
 	RegisterBlockedServicesHandlers()
+	RegisterQueryLogHandlers()
 	RegisterStatsHandlers()
 
 	http.HandleFunc("/dns-query", postInstall(handleDOH))

home/control_querylog.go

@@ -0,0 +1,91 @@
+package home
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/AdguardTeam/AdGuardHome/querylog"
+)
+
+func handleQueryLog(w http.ResponseWriter, r *http.Request) {
+	data := config.queryLog.GetData()
+
+	jsonVal, err := json.Marshal(data)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
+	}
+}
+
+func handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
+	config.queryLog.Clear()
+	returnOK(w)
+}
+
+type qlogConfig struct {
+	Enabled  bool   `json:"enabled"`
+	Interval uint32 `json:"interval"`
+}
+
+// Get configuration
+func handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
+	resp := qlogConfig{}
+	resp.Enabled = config.DNS.QueryLogEnabled
+	resp.Interval = config.DNS.QueryLogInterval
+
+	jsonVal, err := json.Marshal(resp)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "json encode: %s", err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "http write: %s", err)
+	}
+}
+
+// Set configuration
+func handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
+
+	reqData := qlogConfig{}
+	err := json.NewDecoder(r.Body).Decode(&reqData)
+	if err != nil {
+		httpError(w, http.StatusBadRequest, "json decode: %s", err)
+		return
+	}
+
+	if !checkQueryLogInterval(reqData.Interval) {
+		httpError(w, http.StatusBadRequest, "Unsupported interval")
+		return
+	}
+
+	config.DNS.QueryLogEnabled = reqData.Enabled
+	config.DNS.QueryLogInterval = reqData.Interval
+	_ = config.write()
+
+	conf := querylog.Config{
+		Interval: config.DNS.QueryLogInterval * 24,
+	}
+	config.queryLog.Configure(conf)
+
+	returnOK(w)
+}
+
+func checkQueryLogInterval(i uint32) bool {
+	return i == 1 || i == 7 || i == 30 || i == 90
+}
+
+// RegisterQueryLogHandlers - register handlers
+func RegisterQueryLogHandlers() {
+	httpRegister(http.MethodGet, "/control/querylog", handleQueryLog)
+	httpRegister(http.MethodGet, "/control/querylog_info", handleQueryLogInfo)
+	httpRegister(http.MethodPost, "/control/querylog_clear", handleQueryLogClear)
+	httpRegister(http.MethodPost, "/control/querylog_config", handleQueryLogConfig)
+}

home/dns.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
+	"github.com/AdguardTeam/AdGuardHome/querylog"
 	"github.com/AdguardTeam/AdGuardHome/stats"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
@@ -40,7 +41,12 @@ func initDNSServer(baseDir string) {
 	if err != nil {
 		log.Fatal("Couldn't initialize statistics module")
 	}
-	config.dnsServer = dnsforward.NewServer(baseDir, config.stats)
+	conf := querylog.Config{
+		BaseDir:  baseDir,
+		Interval: config.DNS.QueryLogInterval * 24,
+	}
+	config.queryLog = querylog.New(conf)
+	config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
 
 	initRDNS()
 }
@@ -186,6 +192,7 @@ func stopDNSServer() error {
 	}
 
 	config.stats.Close()
+	config.queryLog.Close()
 
 	return nil
 }

querylog/qlog.go

@@ -1,8 +1,9 @@
-package dnsforward
+package querylog
 
 import (
 	"fmt"
 	"net"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -16,15 +17,13 @@ import (
 
 const (
 	logBufferCap     = 5000            // maximum capacity of logBuffer before it's flushed to disk
-	queryLogTimeLimit      = time.Hour * 24  // how far in the past we care about querylogs
-	queryLogRotationPeriod = time.Hour * 24  // rotate the log every 24 hours
 	queryLogFileName = "querylog.json" // .gz added during compression
 	queryLogSize     = 5000            // maximum API response for /querylog
-	queryLogTopSize        = 500             // Keep in memory only top N values
 )
 
 // queryLog is a structure that writes and reads the DNS query log
 type queryLog struct {
+	conf    Config
 	logFile string // path to the log file
 
 	logBufferLock sync.RWMutex
@@ -32,16 +31,53 @@ type queryLog struct {
 	fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
 	flushPending  bool       // don't start another goroutine while the previous one is still running
 
-	queryLogCache []*logEntry
+	cache []*logEntry
-	queryLogLock  sync.RWMutex
+	lock  sync.RWMutex
 }
 
 // newQueryLog creates a new instance of the query log
-func newQueryLog(baseDir string) *queryLog {
+func newQueryLog(conf Config) *queryLog {
-	l := &queryLog{
+	l := queryLog{}
-		logFile: filepath.Join(baseDir, queryLogFileName),
+	l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
+	l.conf = conf
+	go l.periodicQueryLogRotate()
+	go l.fillFromFile()
+	return &l
 }
-	return l
+
+func (l *queryLog) Close() {
+	_ = l.flushLogBuffer(true)
+}
+
+func (l *queryLog) Configure(conf Config) {
+	l.conf = conf
+}
+
+// Clear memory buffer and remove the file
+func (l *queryLog) Clear() {
+	l.fileFlushLock.Lock()
+	defer l.fileFlushLock.Unlock()
+
+	l.logBufferLock.Lock()
+	l.logBuffer = nil
+	l.flushPending = false
+	l.logBufferLock.Unlock()
+
+	l.lock.Lock()
+	l.cache = nil
+	l.lock.Unlock()
+
+	err := os.Remove(l.logFile + ".1")
+	if err != nil {
+		log.Error("file remove: %s: %s", l.logFile+".1", err)
+	}
+
+	err = os.Remove(l.logFile)
+	if err != nil {
+		log.Error("file remove: %s: %s", l.logFile, err)
+	}
+
+	log.Debug("Query log: cleared")
 }
 
 type logEntry struct {
@@ -54,17 +90,28 @@ type logEntry struct {
 	Upstream string `json:",omitempty"` // if empty, means it was cached
 }
 
-func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) *logEntry {
+// getIPString is a helper function that extracts IP address from net.Addr
+func getIPString(addr net.Addr) string {
+	switch addr := addr.(type) {
+	case *net.UDPAddr:
+		return addr.IP.String()
+	case *net.TCPAddr:
+		return addr.IP.String()
+	}
+	return ""
+}
+
+func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) {
 	var q []byte
 	var a []byte
 	var err error
-	ip := GetIPString(addr)
+	ip := getIPString(addr)
 
 	if question != nil {
 		q, err = question.Pack()
 		if err != nil {
 			log.Printf("failed to pack question for querylog: %s", err)
-			return nil
+			return
 		}
 	}
 
@@ -72,7 +119,7 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
 		a, err = answer.Pack()
 		if err != nil {
 			log.Printf("failed to pack answer for querylog: %s", err)
-			return nil
+			return
 		}
 	}
 
@@ -101,13 +148,13 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
 		}
 	}
 	l.logBufferLock.Unlock()
-	l.queryLogLock.Lock()
+	l.lock.Lock()
-	l.queryLogCache = append(l.queryLogCache, &entry)
+	l.cache = append(l.cache, &entry)
-	if len(l.queryLogCache) > queryLogSize {
+	if len(l.cache) > queryLogSize {
-		toremove := len(l.queryLogCache) - queryLogSize
+		toremove := len(l.cache) - queryLogSize
-		l.queryLogCache = l.queryLogCache[toremove:]
+		l.cache = l.cache[toremove:]
 	}
-	l.queryLogLock.Unlock()
+	l.lock.Unlock()
 
 	// if buffer needs to be flushed to disk, do it now
 	if needFlush {
@@ -115,16 +162,14 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil
 		// do it in separate goroutine -- we are stalling DNS response this whole time
 		go l.flushLogBuffer(false) // nolint
 	}
-
-	return &entry
 }
 
 // getQueryLogJson returns a map with the current query log ready to be converted to a JSON
-func (l *queryLog) getQueryLog() []map[string]interface{} {
+func (l *queryLog) GetData() []map[string]interface{} {
-	l.queryLogLock.RLock()
+	l.lock.RLock()
-	values := make([]*logEntry, len(l.queryLogCache))
+	values := make([]*logEntry, len(l.cache))
-	copy(values, l.queryLogCache)
+	copy(values, l.cache)
-	l.queryLogLock.RUnlock()
+	l.lock.RUnlock()
 
 	// reverse it so that newest is first
 	for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 {

querylog/querylog.go

@@ -0,0 +1,33 @@
+package querylog
+
+import (
+	"net"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
+	"github.com/miekg/dns"
+)
+
+// QueryLog - main interface
+type QueryLog interface {
+	Close()
+
+	// Set new configuration at runtime
+	// Currently only 'Interval' field is supported.
+	Configure(conf Config)
+
+	Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
+	GetData() []map[string]interface{}
+	Clear()
+}
+
+// Config - configuration object
+type Config struct {
+	BaseDir  string // directory where log file is stored
+	Interval uint32 // interval to rotate logs (in hours)
+}
+
+// New - create instance
+func New(conf Config) QueryLog {
+	return newQueryLog(conf)
+}

querylog/querylog_file.go

@@ -1,4 +1,4 @@
-package dnsforward
+package querylog
 
 import (
 	"bytes"
@@ -11,6 +11,7 @@ import (
 
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/go-test/deep"
+	"github.com/miekg/dns"
 )
 
 var (
@@ -170,7 +171,7 @@ func (l *queryLog) rotateQueryLog() error {
 }
 
 func (l *queryLog) periodicQueryLogRotate() {
-	for range time.Tick(queryLogRotationPeriod) {
+	for range time.Tick(time.Duration(l.conf.Interval) * time.Hour) {
 		err := l.rotateQueryLog()
 		if err != nil {
 			log.Error("Failed to rotate querylog: %s", err)
@@ -178,3 +179,152 @@ func (l *queryLog) periodicQueryLogRotate() {
 		}
 	}
 }
+
+// Reader is the DB reader context
+type Reader struct {
+	f   *os.File
+	jd  *json.Decoder
+	now time.Time
+	ql  *queryLog
+
+	files []string
+	ifile int
+
+	count uint64 // returned elements counter
+}
+
+// OpenReader locks the file and returns reader object or nil on error
+func (l *queryLog) OpenReader() *Reader {
+	r := Reader{}
+	r.ql = l
+	r.now = time.Now()
+
+	return &r
+}
+
+// Close closes the reader
+func (r *Reader) Close() {
+	elapsed := time.Since(r.now)
+	var perunit time.Duration
+	if r.count > 0 {
+		perunit = elapsed / time.Duration(r.count)
+	}
+	log.Debug("querylog: read %d entries in %v, %v/entry",
+		r.count, elapsed, perunit)
+
+	if r.f != nil {
+		r.f.Close()
+	}
+}
+
+// BeginRead starts reading
+func (r *Reader) BeginRead() {
+	r.files = []string{
+		r.ql.logFile,
+		r.ql.logFile + ".1",
+	}
+}
+
+// Next returns the next entry or nil if reading is finished
+func (r *Reader) Next() *logEntry { // nolint
+	var err error
+	for {
+		// open file if needed
+		if r.f == nil {
+			if r.ifile == len(r.files) {
+				return nil
+			}
+			fn := r.files[r.ifile]
+			r.f, err = os.Open(fn)
+			if err != nil {
+				log.Error("Failed to open file \"%s\": %s", fn, err)
+				r.ifile++
+				continue
+			}
+		}
+
+		// open decoder if needed
+		if r.jd == nil {
+			r.jd = json.NewDecoder(r.f)
+		}
+
+		// check if there's data
+		if !r.jd.More() {
+			r.jd = nil
+			r.f.Close()
+			r.f = nil
+			r.ifile++
+			continue
+		}
+
+		// read data
+		var entry logEntry
+		err = r.jd.Decode(&entry)
+		if err != nil {
+			log.Error("Failed to decode: %s", err)
+			// next entry can be fine, try more
+			continue
+		}
+		r.count++
+		return &entry
+	}
+}
+
+// Total returns the total number of items
+func (r *Reader) Total() int {
+	return 0
+}
+
+// Fill cache from file
+func (l *queryLog) fillFromFile() {
+	now := time.Now()
+	validFrom := now.Unix() - int64(l.conf.Interval*60*60)
+	r := l.OpenReader()
+	if r == nil {
+		return
+	}
+
+	r.BeginRead()
+
+	for {
+		entry := r.Next()
+		if entry == nil {
+			break
+		}
+
+		if entry.Time.Unix() < validFrom {
+			continue
+		}
+
+		if len(entry.Question) == 0 {
+			log.Printf("entry question is absent, skipping")
+			continue
+		}
+
+		if entry.Time.After(now) {
+			log.Printf("t %v vs %v is in the future, ignoring", entry.Time, now)
+			continue
+		}
+
+		q := new(dns.Msg)
+		if err := q.Unpack(entry.Question); err != nil {
+			log.Printf("failed to unpack dns message question: %s", err)
+			continue
+		}
+
+		if len(q.Question) != 1 {
+			log.Printf("malformed dns message, has no questions, skipping")
+			continue
+		}
+
+		l.lock.Lock()
+		l.cache = append(l.cache, entry)
+		if len(l.cache) > queryLogSize {
+			toremove := len(l.cache) - queryLogSize
+			l.cache = l.cache[toremove:]
+		}
+		l.lock.Unlock()
+	}
+
+	r.Close()
+}

querylog/querylog_test.go

@@ -0,0 +1,43 @@
+package querylog
+
+import (
+	"net"
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestQueryLog(t *testing.T) {
+	conf := Config{
+		Interval: 1,
+	}
+	l := New(conf)
+
+	q := dns.Msg{}
+	q.Question = append(q.Question, dns.Question{
+		Name:   "example.org.",
+		Qtype:  dns.TypeA,
+		Qclass: dns.ClassINET,
+	})
+
+	a := dns.Msg{}
+	a.Question = append(a.Question, q.Question[0])
+	answer := new(dns.A)
+	answer.Hdr = dns.RR_Header{
+		Name:   q.Question[0].Name,
+		Rrtype: dns.TypeA,
+		Class:  dns.ClassINET,
+	}
+	answer.A = net.IP{1, 2, 3, 4}
+	a.Answer = append(a.Answer, answer)
+
+	res := dnsfilter.Result{}
+	l.Add(&q, &a, &res, 0, nil, "upstream")
+
+	d := l.GetData()
+	m := d[0]
+	mq := m["question"].(map[string]interface{})
+	assert.True(t, mq["host"].(string) == "example.org")
+}