diff --git a/AGHTechDoc.md b/AGHTechDoc.md index 4b2833d7..e2c88d12 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -42,6 +42,9 @@ Contents: * API: Clear statistics data * API: Set statistics parameters * API: Get statistics parameters +* Query logs + * API: Set querylog parameters + * API: Get querylog parameters ## First startup @@ -976,3 +979,37 @@ Response: { "interval": 1 | 7 | 30 | 90 } + + +## Query logs + +### API: Set querylog parameters + +Request: + + POST /control/querylog_config + + { + "enabled": true | false + "interval": 1 | 7 | 30 | 90 + } + +Response: + + 200 OK + + +### API: Get querylog parameters + +Request: + + GET /control/querylog_info + +Response: + + 200 OK + + { + "enabled": true | false + "interval": 1 | 7 | 30 | 90 + } diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index 0fffd0a5..ee1c1e0d 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -11,6 +11,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/dnsfilter" + "github.com/AdguardTeam/AdGuardHome/querylog" "github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/upstream" @@ -40,7 +41,7 @@ const ( type Server struct { dnsProxy *proxy.Proxy // DNS proxy instance dnsFilter *dnsfilter.Dnsfilter // DNS filter instance - queryLog *queryLog // Query log instance + queryLog querylog.QueryLog // Query log instance stats stats.Stats AllowedClients map[string]bool // IP addresses of whitelist clients @@ -54,16 +55,11 @@ type Server struct { } // NewServer creates a new instance of the dnsforward.Server -// baseDir is the base directory for query logs // Note: this function must be called only once -func NewServer(baseDir string, stats stats.Stats) *Server { - s := &Server{ - queryLog: newQueryLog(baseDir), - } +func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server { + s := &Server{} s.stats = stats - - log.Printf("Start DNS server periodic jobs") - go s.queryLog.periodicQueryLogRotate() + s.queryLog = queryLog return s } @@ -75,6 +71,7 @@ type FilteringConfig struct { BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600) QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled + QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) Ratelimit int `yaml:"ratelimit"` // max number of requests per second from a given IP (0 to disable) RatelimitWhitelist []string `yaml:"ratelimit_whitelist"` // a list of whitelisted client IP addresses RefuseAny bool `yaml:"refuse_any"` // if true, refuse ANY requests @@ -303,8 +300,7 @@ func (s *Server) stopInternal() error { s.dnsFilter = nil } - // flush remainder to file - return s.queryLog.flushLogBuffer(true) + return nil } // IsRunning returns true if the DNS server is running @@ -343,13 +339,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.RUnlock() } -// GetQueryLog returns a map with the current query log ready to be converted to a JSON -func (s *Server) GetQueryLog() []map[string]interface{} { - s.RLock() - defer s.RUnlock() - return s.queryLog.getQueryLog() -} - // Return TRUE if this client should be blocked func (s *Server) isBlockedIP(ip string) bool { if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 { @@ -469,12 +458,12 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error { } elapsed := time.Since(start) - if s.conf.QueryLogEnabled && shouldLog { + if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil { upstreamAddr := "" if d.Upstream != nil { upstreamAddr = d.Upstream.Address() } - _ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr) + s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr) } s.updateStats(d, elapsed, *res) diff --git a/dnsforward/dnsforward_test.go b/dnsforward/dnsforward_test.go index 740a43b8..92a1e01b 100644 --- a/dnsforward/dnsforward_test.go +++ b/dnsforward/dnsforward_test.go @@ -10,7 +10,6 @@ import ( "encoding/pem" "math/big" "net" - "os" "sync" "testing" "time" @@ -18,18 +17,15 @@ import ( "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/miekg/dns" - "github.com/stretchr/testify/assert" ) const ( tlsServerName = "testdns.adguard.com" - dataDir = "testData" testMessagesCount = 10 ) func TestServer(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -45,10 +41,6 @@ func TestServer(t *testing.T) { } assertGoogleAResponse(t, reply) - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - // message over TCP req = createGoogleATestMessage() addr = s.dnsProxy.Addr("tcp") @@ -59,10 +51,6 @@ func TestServer(t *testing.T) { } assertGoogleAResponse(t, reply) - // check query log and stats again - log = s.GetQueryLog() - assert.Equal(t, 2, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -72,7 +60,6 @@ func TestServer(t *testing.T) { func TestServerWithProtectionDisabled(t *testing.T) { s := createTestServer(t) s.conf.ProtectionEnabled = false - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -88,10 +75,6 @@ func TestServerWithProtectionDisabled(t *testing.T) { } assertGoogleAResponse(t, reply) - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -102,7 +85,6 @@ func TestDotServer(t *testing.T) { // Prepare the proxy server _, certPem, keyPem := createServerTLSConfig(t) s := createTestServer(t) - defer removeDataDir(t) s.conf.TLSConfig = TLSConfig{ TLSListenAddr: &net.TCPAddr{Port: 0}, @@ -143,7 +125,6 @@ func TestDotServer(t *testing.T) { func TestServerRace(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -168,7 +149,6 @@ func TestServerRace(t *testing.T) { func TestSafeSearch(t *testing.T) { s := createTestServer(t) s.conf.SafeSearchEnabled = true - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -210,7 +190,6 @@ func TestSafeSearch(t *testing.T) { func TestInvalidRequest(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -229,11 +208,6 @@ func TestInvalidRequest(t *testing.T) { t.Fatalf("got a response to an invalid query") } - // check query log and stats - // invalid requests aren't written to the query log - log := s.GetQueryLog() - assert.Equal(t, 0, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -242,7 +216,6 @@ func TestInvalidRequest(t *testing.T) { func TestBlockedRequest(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -267,10 +240,6 @@ func TestBlockedRequest(t *testing.T) { t.Fatalf("Wrong response: %s", reply.String()) } - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -280,7 +249,6 @@ func TestBlockedRequest(t *testing.T) { func TestNullBlockedRequest(t *testing.T) { s := createTestServer(t) s.conf.FilteringConfig.BlockingMode = "null_ip" - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -312,10 +280,6 @@ func TestNullBlockedRequest(t *testing.T) { t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) } - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -324,7 +288,6 @@ func TestNullBlockedRequest(t *testing.T) { func TestBlockedByHosts(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -356,10 +319,6 @@ func TestBlockedByHosts(t *testing.T) { t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) } - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -368,7 +327,6 @@ func TestBlockedByHosts(t *testing.T) { func TestBlockedBySafeBrowsing(t *testing.T) { s := createTestServer(t) - defer removeDataDir(t) err := s.Start(nil) if err != nil { t.Fatalf("Failed to start server: %s", err) @@ -411,10 +369,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) { t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0]) } - // check query log and stats - log := s.GetQueryLog() - assert.Equal(t, 1, len(log), "Log size") - err = s.Stop() if err != nil { t.Fatalf("DNS server failed to stop: %s", err) @@ -422,7 +376,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) { } func createTestServer(t *testing.T) *Server { - s := NewServer(createDataDir(t), nil) + s := NewServer(nil, nil) s.conf.UDPListenAddr = &net.UDPAddr{Port: 0} s.conf.TCPListenAddr = &net.TCPAddr{Port: 0} @@ -489,21 +443,6 @@ func createServerTLSConfig(t *testing.T) (*tls.Config, []byte, []byte) { return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem } -func createDataDir(t *testing.T) string { - err := os.MkdirAll(dataDir, 0755) - if err != nil { - t.Fatalf("Cannot create %s: %s", dataDir, err) - } - return dataDir -} - -func removeDataDir(t *testing.T) { - err := os.RemoveAll(dataDir) - if err != nil { - t.Fatalf("Cannot remove %s: %s", dataDir, err) - } -} - func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) { defer func() { g.Done() @@ -607,7 +546,6 @@ func TestIsBlockedIPAllowed(t *testing.T) { s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"} err := s.Start(nil) - defer removeDataDir(t) if err != nil { t.Fatalf("Failed to start server: %s", err) } @@ -631,7 +569,6 @@ func TestIsBlockedIPDisallowed(t *testing.T) { s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"} err := s.Start(nil) - defer removeDataDir(t) if err != nil { t.Fatalf("Failed to start server: %s", err) } @@ -655,7 +592,6 @@ func TestIsBlockedIPBlockedDomain(t *testing.T) { s.conf.BlockedHosts = []string{"host1", "host2"} err := s.Start(nil) - defer removeDataDir(t) if err != nil { t.Fatalf("Failed to start server: %s", err) } diff --git a/home/config.go b/home/config.go index 4edc168f..9323b1ce 100644 --- a/home/config.go +++ b/home/config.go @@ -12,6 +12,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/dhcpd" "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsforward" + "github.com/AdguardTeam/AdGuardHome/querylog" "github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/golibs/file" "github.com/AdguardTeam/golibs/log" @@ -70,6 +71,7 @@ type configuration struct { transport *http.Transport client *http.Client stats stats.Stats + queryLog querylog.QueryLog // cached version.json to avoid hammering github.io for each page reload versionCheckJSON []byte @@ -175,6 +177,7 @@ var config = configuration{ BlockingMode: "nxdomain", // mode how to answer filtered requests BlockedResponseTTL: 10, // in seconds QueryLogEnabled: true, + QueryLogInterval: 1, Ratelimit: 20, RefuseAny: true, BootstrapDNS: defaultBootstrap, @@ -274,6 +277,10 @@ func parseConfig() error { config.DNS.StatsInterval = 1 } + if !checkQueryLogInterval(config.DNS.QueryLogInterval) { + config.DNS.QueryLogInterval = 1 + } + for _, cy := range config.Clients { cli := Client{ Name: cy.Name, diff --git a/home/control.go b/home/control.go index 5ac71d6d..27187626 100644 --- a/home/control.go +++ b/home/control.go @@ -146,35 +146,6 @@ func handleProtectionDisable(w http.ResponseWriter, r *http.Request) { httpUpdateConfigReloadDNSReturnOK(w, r) } -// ----- -// stats -// ----- -func handleQueryLogEnable(w http.ResponseWriter, r *http.Request) { - config.DNS.QueryLogEnabled = true - httpUpdateConfigReloadDNSReturnOK(w, r) -} - -func handleQueryLogDisable(w http.ResponseWriter, r *http.Request) { - config.DNS.QueryLogEnabled = false - httpUpdateConfigReloadDNSReturnOK(w, r) -} - -func handleQueryLog(w http.ResponseWriter, r *http.Request) { - data := config.dnsServer.GetQueryLog() - - jsonVal, err := json.Marshal(data) - if err != nil { - httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err) - return - } - - w.Header().Set("Content-Type", "application/json") - _, err = w.Write(jsonVal) - if err != nil { - httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err) - } -} - // ----------------------- // upstreams configuration // ----------------------- @@ -570,9 +541,6 @@ func registerControlHandlers() { httpRegister(http.MethodGet, "/control/status", handleStatus) httpRegister(http.MethodPost, "/control/enable_protection", handleProtectionEnable) httpRegister(http.MethodPost, "/control/disable_protection", handleProtectionDisable) - httpRegister(http.MethodGet, "/control/querylog", handleQueryLog) - httpRegister(http.MethodPost, "/control/querylog_enable", handleQueryLogEnable) - httpRegister(http.MethodPost, "/control/querylog_disable", handleQueryLogDisable) httpRegister(http.MethodPost, "/control/set_upstreams_config", handleSetUpstreamConfig) httpRegister(http.MethodPost, "/control/test_upstream_dns", handleTestUpstreamDNS) httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage) @@ -611,6 +579,7 @@ func registerControlHandlers() { RegisterClientsHandlers() registerRewritesHandlers() RegisterBlockedServicesHandlers() + RegisterQueryLogHandlers() RegisterStatsHandlers() http.HandleFunc("/dns-query", postInstall(handleDOH)) diff --git a/home/control_querylog.go b/home/control_querylog.go new file mode 100644 index 00000000..43ac3869 --- /dev/null +++ b/home/control_querylog.go @@ -0,0 +1,91 @@ +package home + +import ( + "encoding/json" + "net/http" + + "github.com/AdguardTeam/AdGuardHome/querylog" +) + +func handleQueryLog(w http.ResponseWriter, r *http.Request) { + data := config.queryLog.GetData() + + jsonVal, err := json.Marshal(data) + if err != nil { + httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err) + return + } + + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(jsonVal) + if err != nil { + httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err) + } +} + +func handleQueryLogClear(w http.ResponseWriter, r *http.Request) { + config.queryLog.Clear() + returnOK(w) +} + +type qlogConfig struct { + Enabled bool `json:"enabled"` + Interval uint32 `json:"interval"` +} + +// Get configuration +func handleQueryLogInfo(w http.ResponseWriter, r *http.Request) { + resp := qlogConfig{} + resp.Enabled = config.DNS.QueryLogEnabled + resp.Interval = config.DNS.QueryLogInterval + + jsonVal, err := json.Marshal(resp) + if err != nil { + httpError(w, http.StatusInternalServerError, "json encode: %s", err) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(jsonVal) + if err != nil { + httpError(w, http.StatusInternalServerError, "http write: %s", err) + } +} + +// Set configuration +func handleQueryLogConfig(w http.ResponseWriter, r *http.Request) { + + reqData := qlogConfig{} + err := json.NewDecoder(r.Body).Decode(&reqData) + if err != nil { + httpError(w, http.StatusBadRequest, "json decode: %s", err) + return + } + + if !checkQueryLogInterval(reqData.Interval) { + httpError(w, http.StatusBadRequest, "Unsupported interval") + return + } + + config.DNS.QueryLogEnabled = reqData.Enabled + config.DNS.QueryLogInterval = reqData.Interval + _ = config.write() + + conf := querylog.Config{ + Interval: config.DNS.QueryLogInterval * 24, + } + config.queryLog.Configure(conf) + + returnOK(w) +} + +func checkQueryLogInterval(i uint32) bool { + return i == 1 || i == 7 || i == 30 || i == 90 +} + +// RegisterQueryLogHandlers - register handlers +func RegisterQueryLogHandlers() { + httpRegister(http.MethodGet, "/control/querylog", handleQueryLog) + httpRegister(http.MethodGet, "/control/querylog_info", handleQueryLogInfo) + httpRegister(http.MethodPost, "/control/querylog_clear", handleQueryLogClear) + httpRegister(http.MethodPost, "/control/querylog_config", handleQueryLogConfig) +} diff --git a/home/dns.go b/home/dns.go index b1d1b0ca..53958bd5 100644 --- a/home/dns.go +++ b/home/dns.go @@ -9,6 +9,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsforward" + "github.com/AdguardTeam/AdGuardHome/querylog" "github.com/AdguardTeam/AdGuardHome/stats" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/upstream" @@ -40,7 +41,12 @@ func initDNSServer(baseDir string) { if err != nil { log.Fatal("Couldn't initialize statistics module") } - config.dnsServer = dnsforward.NewServer(baseDir, config.stats) + conf := querylog.Config{ + BaseDir: baseDir, + Interval: config.DNS.QueryLogInterval * 24, + } + config.queryLog = querylog.New(conf) + config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog) initRDNS() } @@ -186,6 +192,7 @@ func stopDNSServer() error { } config.stats.Close() + config.queryLog.Close() return nil } diff --git a/dnsforward/querylog.go b/querylog/qlog.go similarity index 72% rename from dnsforward/querylog.go rename to querylog/qlog.go index fbbeb7f2..cfc0604c 100644 --- a/dnsforward/querylog.go +++ b/querylog/qlog.go @@ -1,8 +1,9 @@ -package dnsforward +package querylog import ( "fmt" "net" + "os" "path/filepath" "strconv" "strings" @@ -15,16 +16,14 @@ import ( ) const ( - logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk - queryLogTimeLimit = time.Hour * 24 // how far in the past we care about querylogs - queryLogRotationPeriod = time.Hour * 24 // rotate the log every 24 hours - queryLogFileName = "querylog.json" // .gz added during compression - queryLogSize = 5000 // maximum API response for /querylog - queryLogTopSize = 500 // Keep in memory only top N values + logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk + queryLogFileName = "querylog.json" // .gz added during compression + queryLogSize = 5000 // maximum API response for /querylog ) // queryLog is a structure that writes and reads the DNS query log type queryLog struct { + conf Config logFile string // path to the log file logBufferLock sync.RWMutex @@ -32,16 +31,53 @@ type queryLog struct { fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread flushPending bool // don't start another goroutine while the previous one is still running - queryLogCache []*logEntry - queryLogLock sync.RWMutex + cache []*logEntry + lock sync.RWMutex } // newQueryLog creates a new instance of the query log -func newQueryLog(baseDir string) *queryLog { - l := &queryLog{ - logFile: filepath.Join(baseDir, queryLogFileName), +func newQueryLog(conf Config) *queryLog { + l := queryLog{} + l.logFile = filepath.Join(conf.BaseDir, queryLogFileName) + l.conf = conf + go l.periodicQueryLogRotate() + go l.fillFromFile() + return &l +} + +func (l *queryLog) Close() { + _ = l.flushLogBuffer(true) +} + +func (l *queryLog) Configure(conf Config) { + l.conf = conf +} + +// Clear memory buffer and remove the file +func (l *queryLog) Clear() { + l.fileFlushLock.Lock() + defer l.fileFlushLock.Unlock() + + l.logBufferLock.Lock() + l.logBuffer = nil + l.flushPending = false + l.logBufferLock.Unlock() + + l.lock.Lock() + l.cache = nil + l.lock.Unlock() + + err := os.Remove(l.logFile + ".1") + if err != nil { + log.Error("file remove: %s: %s", l.logFile+".1", err) } - return l + + err = os.Remove(l.logFile) + if err != nil { + log.Error("file remove: %s: %s", l.logFile, err) + } + + log.Debug("Query log: cleared") } type logEntry struct { @@ -54,17 +90,28 @@ type logEntry struct { Upstream string `json:",omitempty"` // if empty, means it was cached } -func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) *logEntry { +// getIPString is a helper function that extracts IP address from net.Addr +func getIPString(addr net.Addr) string { + switch addr := addr.(type) { + case *net.UDPAddr: + return addr.IP.String() + case *net.TCPAddr: + return addr.IP.String() + } + return "" +} + +func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) { var q []byte var a []byte var err error - ip := GetIPString(addr) + ip := getIPString(addr) if question != nil { q, err = question.Pack() if err != nil { log.Printf("failed to pack question for querylog: %s", err) - return nil + return } } @@ -72,7 +119,7 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil a, err = answer.Pack() if err != nil { log.Printf("failed to pack answer for querylog: %s", err) - return nil + return } } @@ -101,13 +148,13 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil } } l.logBufferLock.Unlock() - l.queryLogLock.Lock() - l.queryLogCache = append(l.queryLogCache, &entry) - if len(l.queryLogCache) > queryLogSize { - toremove := len(l.queryLogCache) - queryLogSize - l.queryLogCache = l.queryLogCache[toremove:] + l.lock.Lock() + l.cache = append(l.cache, &entry) + if len(l.cache) > queryLogSize { + toremove := len(l.cache) - queryLogSize + l.cache = l.cache[toremove:] } - l.queryLogLock.Unlock() + l.lock.Unlock() // if buffer needs to be flushed to disk, do it now if needFlush { @@ -115,16 +162,14 @@ func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfil // do it in separate goroutine -- we are stalling DNS response this whole time go l.flushLogBuffer(false) // nolint } - - return &entry } // getQueryLogJson returns a map with the current query log ready to be converted to a JSON -func (l *queryLog) getQueryLog() []map[string]interface{} { - l.queryLogLock.RLock() - values := make([]*logEntry, len(l.queryLogCache)) - copy(values, l.queryLogCache) - l.queryLogLock.RUnlock() +func (l *queryLog) GetData() []map[string]interface{} { + l.lock.RLock() + values := make([]*logEntry, len(l.cache)) + copy(values, l.cache) + l.lock.RUnlock() // reverse it so that newest is first for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 { diff --git a/querylog/querylog.go b/querylog/querylog.go new file mode 100644 index 00000000..c995183b --- /dev/null +++ b/querylog/querylog.go @@ -0,0 +1,33 @@ +package querylog + +import ( + "net" + "time" + + "github.com/AdguardTeam/AdGuardHome/dnsfilter" + "github.com/miekg/dns" +) + +// QueryLog - main interface +type QueryLog interface { + Close() + + // Set new configuration at runtime + // Currently only 'Interval' field is supported. + Configure(conf Config) + + Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) + GetData() []map[string]interface{} + Clear() +} + +// Config - configuration object +type Config struct { + BaseDir string // directory where log file is stored + Interval uint32 // interval to rotate logs (in hours) +} + +// New - create instance +func New(conf Config) QueryLog { + return newQueryLog(conf) +} diff --git a/dnsforward/querylog_file.go b/querylog/querylog_file.go similarity index 56% rename from dnsforward/querylog_file.go rename to querylog/querylog_file.go index e990fdec..6f6f887a 100644 --- a/dnsforward/querylog_file.go +++ b/querylog/querylog_file.go @@ -1,4 +1,4 @@ -package dnsforward +package querylog import ( "bytes" @@ -11,6 +11,7 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/go-test/deep" + "github.com/miekg/dns" ) var ( @@ -170,7 +171,7 @@ func (l *queryLog) rotateQueryLog() error { } func (l *queryLog) periodicQueryLogRotate() { - for range time.Tick(queryLogRotationPeriod) { + for range time.Tick(time.Duration(l.conf.Interval) * time.Hour) { err := l.rotateQueryLog() if err != nil { log.Error("Failed to rotate querylog: %s", err) @@ -178,3 +179,152 @@ func (l *queryLog) periodicQueryLogRotate() { } } } + +// Reader is the DB reader context +type Reader struct { + f *os.File + jd *json.Decoder + now time.Time + ql *queryLog + + files []string + ifile int + + count uint64 // returned elements counter +} + +// OpenReader locks the file and returns reader object or nil on error +func (l *queryLog) OpenReader() *Reader { + r := Reader{} + r.ql = l + r.now = time.Now() + + return &r +} + +// Close closes the reader +func (r *Reader) Close() { + elapsed := time.Since(r.now) + var perunit time.Duration + if r.count > 0 { + perunit = elapsed / time.Duration(r.count) + } + log.Debug("querylog: read %d entries in %v, %v/entry", + r.count, elapsed, perunit) + + if r.f != nil { + r.f.Close() + } +} + +// BeginRead starts reading +func (r *Reader) BeginRead() { + r.files = []string{ + r.ql.logFile, + r.ql.logFile + ".1", + } +} + +// Next returns the next entry or nil if reading is finished +func (r *Reader) Next() *logEntry { // nolint + var err error + for { + // open file if needed + if r.f == nil { + if r.ifile == len(r.files) { + return nil + } + fn := r.files[r.ifile] + r.f, err = os.Open(fn) + if err != nil { + log.Error("Failed to open file \"%s\": %s", fn, err) + r.ifile++ + continue + } + } + + // open decoder if needed + if r.jd == nil { + r.jd = json.NewDecoder(r.f) + } + + // check if there's data + if !r.jd.More() { + r.jd = nil + r.f.Close() + r.f = nil + r.ifile++ + continue + } + + // read data + var entry logEntry + err = r.jd.Decode(&entry) + if err != nil { + log.Error("Failed to decode: %s", err) + // next entry can be fine, try more + continue + } + r.count++ + return &entry + } +} + +// Total returns the total number of items +func (r *Reader) Total() int { + return 0 +} + +// Fill cache from file +func (l *queryLog) fillFromFile() { + now := time.Now() + validFrom := now.Unix() - int64(l.conf.Interval*60*60) + r := l.OpenReader() + if r == nil { + return + } + + r.BeginRead() + + for { + entry := r.Next() + if entry == nil { + break + } + + if entry.Time.Unix() < validFrom { + continue + } + + if len(entry.Question) == 0 { + log.Printf("entry question is absent, skipping") + continue + } + + if entry.Time.After(now) { + log.Printf("t %v vs %v is in the future, ignoring", entry.Time, now) + continue + } + + q := new(dns.Msg) + if err := q.Unpack(entry.Question); err != nil { + log.Printf("failed to unpack dns message question: %s", err) + continue + } + + if len(q.Question) != 1 { + log.Printf("malformed dns message, has no questions, skipping") + continue + } + + l.lock.Lock() + l.cache = append(l.cache, entry) + if len(l.cache) > queryLogSize { + toremove := len(l.cache) - queryLogSize + l.cache = l.cache[toremove:] + } + l.lock.Unlock() + } + + r.Close() +} diff --git a/querylog/querylog_test.go b/querylog/querylog_test.go new file mode 100644 index 00000000..8da84183 --- /dev/null +++ b/querylog/querylog_test.go @@ -0,0 +1,43 @@ +package querylog + +import ( + "net" + "testing" + + "github.com/AdguardTeam/AdGuardHome/dnsfilter" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" +) + +func TestQueryLog(t *testing.T) { + conf := Config{ + Interval: 1, + } + l := New(conf) + + q := dns.Msg{} + q.Question = append(q.Question, dns.Question{ + Name: "example.org.", + Qtype: dns.TypeA, + Qclass: dns.ClassINET, + }) + + a := dns.Msg{} + a.Question = append(a.Question, q.Question[0]) + answer := new(dns.A) + answer.Hdr = dns.RR_Header{ + Name: q.Question[0].Name, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + } + answer.A = net.IP{1, 2, 3, 4} + a.Answer = append(a.Answer, answer) + + res := dnsfilter.Result{} + l.Add(&q, &a, &res, 0, nil, "upstream") + + d := l.GetData() + m := d[0] + mq := m["question"].(map[string]interface{}) + assert.True(t, mq["host"].(string) == "example.org") +}