Pull request: 3978 Query Log ECS

Merge in DNS/adguard-home from 3978-ecs-ip to master

Updates #3978.

Squashed commit of the following:

commit 915b94afa4b6d90169f73d4fa171bc81bcc267a7
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 17:46:40 2022 +0300

    all: rm dot

commit 2dd2ed081b199de7e5d8269dae5d08d53b5eea6d
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 17:42:45 2022 +0300

    client: imp txt

commit 8d5a23df739f0b650f9f3870141fd83e8fa0c1e0
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 14:36:04 2022 +0300

    client: imp text

commit 69c856749a20144822ef3f1f67c5f3e3c24f5374
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 14:24:56 2022 +0300

    client: imp description

commit cd0150128ad29d1874492735a5d621c0803ad0bd
Merge: 28181fbc e0b557ed
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Mar 2 21:02:16 2022 +0300

    Merge branch 'master' into 3978-ecs-ip

commit 28181fbc79eb22e7fd13cbd1d5a3c040af9fa2a4
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Mar 2 20:45:50 2022 +0300

    client: show ecs

commit cdc5e7f8c4155b798426d815eed0da547ef6efb7
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 20:15:56 2022 +0300

    openapi: fix milestone

commit 404d6d822fa1ba4ed4cd41d92d4c1b805342fe55
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 20:08:21 2022 +0300

    all: fix deps, docs

commit 8fb80526f1e251d3b7b193c53a4a6dee0e22c145
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 19:39:34 2022 +0300

    all: add querylog ecs backend

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to
 
 ### Added
 
+- EDNS Client-Subnet information in the request details section of a query log
+  record ([#3978]).
 - Support for hostnames for plain UDP upstream servers using the `udp://` scheme
   ([#4166]).
 - Logs are now collected by default on FreeBSD and OpenBSD when AdGuard Home is
@@ -84,8 +86,10 @@ In this release, the schema version has changed from 12 to 13.
 [#3367]: https://github.com/AdguardTeam/AdGuardHome/issues/3367
 [#3381]: https://github.com/AdguardTeam/AdGuardHome/issues/3381
 [#3503]: https://github.com/AdguardTeam/AdGuardHome/issues/3503
+[#3978]: https://github.com/AdguardTeam/AdGuardHome/issues/3978
 [#4166]: https://github.com/AdguardTeam/AdGuardHome/issues/4166
 [#4213]: https://github.com/AdguardTeam/AdGuardHome/issues/4213
+[#4216]: https://github.com/AdguardTeam/AdGuardHome/issues/4216
 [#4221]: https://github.com/AdguardTeam/AdGuardHome/issues/4221
 [#4238]: https://github.com/AdguardTeam/AdGuardHome/issues/4238
 

client/src/__locales/en.json

@@ -287,7 +287,7 @@
     "form_enter_rate_limit": "Enter rate limit",
     "rate_limit": "Rate limit",
     "edns_enable": "Enable EDNS client subnet",
-    "edns_cs_desc": "Send clients' subnets to the DNS servers.",
+    "edns_cs_desc": "Add the EDNS Client Subnet option (ECS) to upstream requests and log the values sent by the clients in the query log.",
     "rate_limit_desc": "The number of requests per second allowed per client. Setting it to 0 means no limit.",
     "blocking_ipv4_desc": "IP address to be returned for a blocked A request",
     "blocking_ipv6_desc": "IP address to be returned for a blocked AAAA request",
@@ -502,6 +502,7 @@
     "interval_days": "{{count}} day",
     "interval_days_plural": "{{count}} days",
     "domain": "Domain",
+    "ecs": "ECS",
     "punycode": "Punycode",
     "answer": "Answer",
     "filter_added_successfully": "The list has been successfully added",

client/src/components/Logs/Cells/DomainCell.js

@@ -20,6 +20,7 @@ const DomainCell = ({
     time,
     tracker,
     type,
+    ecs,
 }) => {
     const { t } = useTranslation();
     const dnssec_enabled = useSelector((state) => state.dnsConfig.dnssec_enabled);
@@ -56,6 +57,13 @@ const DomainCell = ({
         };
     }
 
+    if (ecs) {
+        requestDetailsObj = {
+            ...requestDetailsObj,
+            ecs,
+        };
+    }
+
     requestDetailsObj = {
         ...requestDetailsObj,
         type_table_header: type,
@@ -168,6 +176,7 @@ DomainCell.propTypes = {
     time: propTypes.string.isRequired,
     type: propTypes.string.isRequired,
     tracker: propTypes.object,
+    ecs: propTypes.string,
 };
 
 export default DomainCell;

client/src/components/Logs/Cells/index.js

@@ -238,6 +238,7 @@ Row.propTypes = {
         type: propTypes.string.isRequired,
         client_proto: propTypes.string.isRequired,
         client_id: propTypes.string,
+        ecs: propTypes.string,
         client_info: propTypes.shape({
             name: propTypes.string.isRequired,
             whois: propTypes.shape({

client/src/helpers/helpers.js

@@ -76,6 +76,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
         original_answer,
         upstream,
         cached,
+        ecs,
     } = log;
 
     const { name: domain, unicode_name: unicodeName, type } = question;
@@ -118,6 +119,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
         elapsedMs,
         upstream,
         cached,
+        ecs,
     };
 });
 

internal/dnsforward/stats.go

@@ -41,55 +41,65 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
 	// uninitialized while in use.  This can happen after proxy server has been
 	// stopped, but its workers haven't yet exited.
 	if shouldLog && s.queryLog != nil {
-		p := &querylog.AddParams{
+		s.logQuery(dctx, pctx, elapsed, ip)
-			Question:          msg,
-			Answer:            pctx.Res,
-			OrigAnswer:        dctx.origResp,
-			Result:            dctx.result,
-			Elapsed:           elapsed,
-			ClientID:          dctx.clientID,
-			ClientIP:          ip,
-			AuthenticatedData: dctx.responseAD,
-		}
-
-		switch pctx.Proto {
-		case proxy.ProtoHTTPS:
-			p.ClientProto = querylog.ClientProtoDoH
-		case proxy.ProtoQUIC:
-			p.ClientProto = querylog.ClientProtoDoQ
-		case proxy.ProtoTLS:
-			p.ClientProto = querylog.ClientProtoDoT
-		case proxy.ProtoDNSCrypt:
-			p.ClientProto = querylog.ClientProtoDNSCrypt
-		default:
-			// Consider this a plain DNS-over-UDP or DNS-over-TCP request.
-		}
-
-		if pctx.Upstream != nil {
-			p.Upstream = pctx.Upstream.Address()
-		} else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" {
-			p.Upstream = pctx.CachedUpstreamAddr
-			p.Cached = true
-		}
-
-		s.queryLog.Add(p)
 	}
 
-	s.updateStats(dctx, elapsed, *dctx.result, ip)
+	if s.stats != nil {
+		s.updateStats(dctx, elapsed, *dctx.result, ip)
+	}
 
 	return resultCodeSuccess
 }
 
+// logQuery pushes the request details into the query log.
+func (s *Server) logQuery(
+	dctx *dnsContext,
+	pctx *proxy.DNSContext,
+	elapsed time.Duration,
+	ip net.IP,
+) {
+	p := &querylog.AddParams{
+		Question:          pctx.Req,
+		ReqECS:            pctx.ReqECS,
+		Answer:            pctx.Res,
+		OrigAnswer:        dctx.origResp,
+		Result:            dctx.result,
+		Elapsed:           elapsed,
+		ClientID:          dctx.clientID,
+		ClientIP:          ip,
+		AuthenticatedData: dctx.responseAD,
+	}
+
+	switch pctx.Proto {
+	case proxy.ProtoHTTPS:
+		p.ClientProto = querylog.ClientProtoDoH
+	case proxy.ProtoQUIC:
+		p.ClientProto = querylog.ClientProtoDoQ
+	case proxy.ProtoTLS:
+		p.ClientProto = querylog.ClientProtoDoT
+	case proxy.ProtoDNSCrypt:
+		p.ClientProto = querylog.ClientProtoDNSCrypt
+	default:
+		// Consider this a plain DNS-over-UDP or DNS-over-TCP request.
+	}
+
+	if pctx.Upstream != nil {
+		p.Upstream = pctx.Upstream.Address()
+	} else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" {
+		p.Upstream = pctx.CachedUpstreamAddr
+		p.Cached = true
+	}
+
+	s.queryLog.Add(p)
+}
+
+// updatesStats writes the request into statistics.
 func (s *Server) updateStats(
 	ctx *dnsContext,
 	elapsed time.Duration,
 	res filtering.Result,
 	clientIP net.IP,
 ) {
-	if s.stats == nil {
-		return
-	}
-
 	pctx := ctx.proxyCtx
 	e := stats.Entry{}
 	e.Domain = strings.ToLower(pctx.Req.Question[0].Name)

internal/querylog/decode.go

@@ -14,7 +14,7 @@ import (
 	"github.com/miekg/dns"
 )
 
-type logEntryHandler (func(t json.Token, ent *logEntry) error)
+type logEntryHandler func(t json.Token, ent *logEntry) error
 
 var logEntryHandlers = map[string]logEntryHandler{
 	"CID": func(t json.Token, ent *logEntry) error {
@@ -109,6 +109,16 @@ var logEntryHandlers = map[string]logEntryHandler{
 
 		return err
 	},
+	"ECS": func(t json.Token, ent *logEntry) error {
+		v, ok := t.(string)
+		if !ok {
+			return nil
+		}
+
+		ent.ReqECS = v
+
+		return nil
+	},
 	"Cached": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(bool)
 		if !ok {

internal/querylog/decode_test.go

@@ -32,6 +32,7 @@ func TestDecodeLogEntry(t *testing.T) {
 			`"QT":"A",` +
 			`"QC":"IN",` +
 			`"CP":"",` +
+			`"ECS":"1.2.3.0/24",` +
 			`"Answer":"` + ansStr + `",` +
 			`"Cached":true,` +
 			`"AD":true,` +
@@ -58,6 +59,7 @@ func TestDecodeLogEntry(t *testing.T) {
 			QClass:      "IN",
 			ClientID:    "cli42",
 			ClientProto: "",
+			ReqECS:      "1.2.3.0/24",
 			Answer:      ans,
 			Cached:      true,
 			Result: filtering.Result{

internal/querylog/json.go

@@ -78,6 +78,10 @@ func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (json
 		jsonEntry["client_id"] = entry.ClientID
 	}
 
+	if entry.ReqECS != "" {
+		jsonEntry["ecs"] = entry.ReqECS
+	}
+
 	if len(entry.Result.Rules) > 0 {
 		if r := entry.Result.Rules[0]; len(r.Text) > 0 {
 			jsonEntry["rule"] = r.Text

internal/querylog/qlog.go

@@ -81,6 +81,8 @@ type logEntry struct {
 	QType  string `json:"QT"`
 	QClass string `json:"QC"`
 
+	ReqECS string `json:"ECS,omitempty"`
+
 	ClientID    string      `json:"CID,omitempty"`
 	ClientProto ClientProto `json:"CP"`
 
@@ -189,6 +191,10 @@ func (l *queryLog) Add(params *AddParams) {
 		AuthenticatedData: params.AuthenticatedData,
 	}
 
+	if params.ReqECS != nil {
+		entry.ReqECS = params.ReqECS.String()
+	}
+
 	if params.Answer != nil {
 		var a []byte
 		a, err = params.Answer.Pack()

internal/querylog/querylog.go

@@ -77,6 +77,10 @@ type Config struct {
 type AddParams struct {
 	Question *dns.Msg
 
+	// ReqECS is the IP network extracted from EDNS Client-Subnet option of a
+	// request.
+	ReqECS *net.IPNet
+
 	// Answer is the response which is sent to the client, if any.
 	Answer *dns.Msg
 

internal/querylog/searchcriterion.go

@@ -99,24 +99,10 @@ func (c *searchCriterion) quickMatch(line string, findClient quickMatchClientFun
 		}
 
 		if c.strict {
-			return ctDomainOrClientCaseStrict(
+			return ctDomainOrClientCaseStrict(c.value, c.asciiVal, clientID, name, host, ip)
-				c.value,
-				c.asciiVal,
-				clientID,
-				name,
-				host,
-				ip,
-			)
 		}
 
-		return ctDomainOrClientCaseNonStrict(
+		return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
-			c.value,
-			c.asciiVal,
-			clientID,
-			name,
-			host,
-			ip,
-		)
 	case ctFilteringStatus:
 		// Go on, as we currently don't do quick matches against
 		// filtering statuses.

openapi/CHANGELOG.md

@@ -2,7 +2,12 @@
 
 <!-- TODO(a.garipov): Reformat in accordance with the KeepAChangelog spec. -->
 
-## v0.107.3: API changes
+## v0.108.0: API changes
+
+### The new optional field `"ecs"` in `QueryLogItem`
+
+* The new optional field `"ecs"` in `GET /control/querylog` contains the IP
+  network from an EDNS Client-Subnet option from the request message if any.
 
 ### The new possible status code in `/install/configure` response.
 
@@ -10,6 +15,8 @@
   `POST /install/configure` which means that the specified password does not
   meet the strength requirements.
 
+## v0.107.3: API changes
+
 ### The new field `"version"` in `AddressesInfo`
 
 * The new field `"version"` in `GET /install/get_addresses` is the version of

openapi/openapi.yaml

@@ -1905,6 +1905,12 @@
           - 'doq'
           - 'dnscrypt'
           - ''
+        'ecs':
+          'type': 'string'
+          'example': '192.168.0.0/16'
+          'description': >
+            The IP network defined by an EDNS Client-Subnet option in the
+            request message if any.
         'elapsedMs':
           'type': 'string'
           'example': '54.023928'