Merge: + new query logs API

* commit '59c4a2886a97143e3a36912ec895dc1a06be88cc':
  openapi
  If there are no more older entries, `"oldest":""` is returned.
  fix search by "whitelisted", "rewritten"
  doc
  fix whois test
  + "dot"
  openapi
  * minor
  + client_proto
  * openapi
  + new query logs API

AGHTechDoc.md

@@ -1198,8 +1198,9 @@ When a new DNS request is received and processed, we store information about thi
 	"QH":"...", // target host name without the last dot
 	"QT":"...", // question type
 	"QC":"...", // question class
-	"Answer":"...",
+	"CP":"" | "doh", // client connection protocol
-	"OrigAnswer":"...",
+	"Answer":"base64 data",
+	"OrigAnswer":"base64 data",
 	"Result":{
 		"IsFiltered":true,
 		"Reason":3,
@@ -1232,16 +1233,28 @@ Request:
 
 	GET /control/querylog
 	?older_than=2006-01-02T15:04:05.999999999Z07:00
-	&filter_domain=...
+	&search=...
-	&filter_client=...
+	&response_status="..."
-	&filter_question_type=A | AAAA
-	&filter_response_status= | filtered
 
 `older_than` setting is used for paging.  UI uses an empty value for `older_than` on the first request and gets the latest log entries.  To get the older entries, UI sets `older_than` to the `oldest` value from the server's response.
 
-If "filter" settings are set, server returns only entries that match the specified request.
+If search settings are set, server returns only entries that match the specified request.
 
-For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`.  Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
+`search`:
+match by domain name or client IP address.
+The server matches substrings by default: e.g. `adguard.com` matches `www.adguard.com`.
+Strict matching can be enabled by enclosing the value in double quotes: e.g. `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
+
+`response_status`:
+* all
+* filtered             - all kinds of filtering
+* blocked              - blocked or blocked service
+* blocked_safebrowsing - blocked by safebrowsing
+* blocked_parental     - blocked by parental control
+* whitelisted          - whitelisted
+* rewritten            - all kinds of rewrites
+* safe_search          - enforced safe search
+* processed            - not blocked, not white-listed entries
 
 Response:
 
@@ -1264,8 +1277,10 @@ Response:
 			}
 			...
 		],
+		"upstream":"...", // Upstream URL starting with tcp://, tls://, https://, or with an IP address
 		"answer_dnssec": true,
 		"client":"127.0.0.1",
+		"client_proto": "" (plain) | "doh" | "dot",
 		"elapsedMs":"0.098403",
 		"filterId":1,
 		"question":{
@@ -1285,6 +1300,8 @@ Response:
 
 The most recent entries are at the top of list.
 
+If there are no more older entries, `"oldest":""` is returned.
+
 
 ### API: Set querylog parameters
 

dnsforward/dnsforward_http.go

@@ -400,6 +400,11 @@ func checkDNS(input string, bootstrap []string) error {
 	return nil
 }
 
+// Control flow:
+// web
+//  -> dnsforward.handleDOH -> dnsforward.ServeHTTP
+//  -> proxy.ServeHTTP -> proxy.handleDNSRequest
+//  -> dnsforward.handleDNSRequest
 func (s *Server) handleDOH(w http.ResponseWriter, r *http.Request) {
 	if !s.conf.TLSAllowUnencryptedDOH && r.TLS == nil {
 		httpError(r, w, http.StatusNotFound, "Not Found")

dnsforward/stats.go

@@ -39,6 +39,13 @@ func processQueryLogsAndStats(ctx *dnsContext) int {
 			Elapsed:    elapsed,
 			ClientIP:   getIP(d.Addr),
 		}
+
+		if d.Proto == "https" {
+			p.ClientProto = "doh"
+		} else if d.Proto == "tls" {
+			p.ClientProto = "dot"
+		}
+
 		if d.Upstream != nil {
 			p.Upstream = d.Upstream.Address()
 		}

home/whois_test.go

@@ -27,9 +27,9 @@ func TestWhois(t *testing.T) {
 
 	w := Whois{timeoutMsec: 5000}
 	resp, err := w.queryAll("8.8.8.8")
-	assert.True(t, err == nil)
+	assert.Nil(t, err)
 	m := whoisParse(resp)
-	assert.True(t, m["orgname"] == "Google LLC")
+	assert.Equal(t, "Google LLC", m["orgname"])
-	assert.True(t, m["country"] == "US")
+	assert.Equal(t, "US", m["country"])
-	assert.True(t, m["city"] == "Mountain View")
+	assert.Equal(t, "Mountain View", m["city"])
 }

openapi/openapi.yaml

@@ -163,29 +163,26 @@ paths:
                   description: Limit the number of records to be returned
                   schema:
                       type: integer
-                - name: filter_domain
+                - name: search
                   in: query
-                  description: Filter by domain name
+                  description: Filter by domain name or client IP
                   schema:
                       type: string
-                - name: filter_client
+                - name: response_status
-                  in: query
-                  description: Filter by client
-                  schema:
-                      type: string
-                - name: filter_question_type
-                  in: query
-                  description: Filter by question type
-                  schema:
-                      type: string
-                - name: filter_response_status
                   in: query
                   description: Filter by response status
                   schema:
                       type: string
                       enum:
-                          - null
+                          - all
                           - filtered
+                          - blocked
+                          - blocked_safebrowsing
+                          - blocked_parental
+                          - whitelisted
+                          - rewritten
+                          - safe_search
+                          - processed
             responses:
                 "200":
                     description: OK
@@ -1420,11 +1417,19 @@ components:
                     description: Answer from upstream server (optional)
                     items:
                         $ref: "#/components/schemas/DnsAnswer"
+                upstream:
+                    type: string
+                    description: Upstream URL starting with tcp://, tls://, https://, or with an IP address
                 answer_dnssec:
                     type: boolean
                 client:
                     type: string
                     example: 192.168.0.1
+                client_proto:
+                    enum:
+                        - dot
+                        - doh
+                        - ""
                 elapsedMs:
                     type: string
                     example: "54.023928"

querylog/decode.go

@@ -37,6 +37,9 @@ func decodeLogEntry(ent *logEntry, str string) {
 		case "QC":
 			ent.QClass = v
 
+		case "CP":
+			ent.ClientProto = v
+
 		case "Answer":
 			ent.Answer, err = base64.StdEncoding.DecodeString(v)
 		case "OrigAnswer":

querylog/json.go

@@ -63,10 +63,11 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 	}
 
 	jsonEntry := map[string]interface{}{
-		"reason":    entry.Result.Reason.String(),
+		"reason":       entry.Result.Reason.String(),
-		"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
+		"elapsedMs":    strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
-		"time":      entry.Time.Format(time.RFC3339Nano),
+		"time":         entry.Time.Format(time.RFC3339Nano),
-		"client":    l.getClientIP(entry.IP),
+		"client":       l.getClientIP(entry.IP),
+		"client_proto": entry.ClientProto,
 	}
 	jsonEntry["question"] = map[string]interface{}{
 		"host":  entry.QHost,
@@ -112,6 +113,8 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 		}
 	}
 
+	jsonEntry["upstream"] = entry.Upstream
+
 	return jsonEntry
 }
 

querylog/qlog.go

@@ -38,6 +38,8 @@ type logEntry struct {
 	QType  string `json:"QT"`
 	QClass string `json:"QC"`
 
+	ClientProto string `json:"CP"` // "" or "doh"
+
 	Answer     []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net
 	OrigAnswer []byte `json:",omitempty"`
 
@@ -119,9 +121,10 @@ func (l *queryLog) Add(params AddParams) {
 		IP:   l.getClientIP(params.ClientIP.String()),
 		Time: now,
 
-		Result:   *params.Result,
+		Result:      *params.Result,
-		Elapsed:  params.Elapsed,
+		Elapsed:     params.Elapsed,
-		Upstream: params.Upstream,
+		Upstream:    params.Upstream,
+		ClientProto: params.ClientProto,
 	}
 	q := params.Question.Question[0]
 	entry.QHost = strings.ToLower(q.Name[:len(q.Name)-1]) // remove the last dot

querylog/qlog_http.go

@@ -142,10 +142,6 @@ func (l *queryLog) parseSearchCriteria(q url.Values, name string, ct criteriaTyp
 		c.strict = true
 	}
 
-	if ct == ctClient && l.conf.AnonymizeClientIP {
-		c.value = l.getClientIP(c.value)
-	}
-
 	if ct == ctFilteringStatus && !util.ContainsString(filteringStatusValues, c.value) {
 		return false, c, fmt.Errorf("invalid value %s", c.value)
 	}
@@ -180,10 +176,8 @@ func (l *queryLog) parseSearchParams(r *http.Request) (*searchParams, error) {
 	}
 
 	paramNames := map[string]criteriaType{
-		"filter_domain":          ctDomain,
+		"search":          ctDomainOrClient,
-		"filter_client":          ctClient,
+		"response_status": ctFilteringStatus,
-		"filter_question_type":   ctQuestionType,
-		"filter_response_status": ctFilteringStatus,
 	}
 
 	for k, v := range paramNames {

querylog/qlog_test.go

@@ -57,7 +57,7 @@ func TestQueryLog(t *testing.T) {
 	// search by domain (strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctDomain,
+		criteriaType: ctDomainOrClient,
 		strict:       true,
 		value:        "test.example.org",
 	})
@@ -68,7 +68,7 @@ func TestQueryLog(t *testing.T) {
 	// search by domain (not strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctDomain,
+		criteriaType: ctDomainOrClient,
 		strict:       false,
 		value:        "example.org",
 	})
@@ -81,7 +81,7 @@ func TestQueryLog(t *testing.T) {
 	// search by client IP (strict)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctClient,
+		criteriaType: ctDomainOrClient,
 		strict:       true,
 		value:        "2.2.2.2",
 	})
@@ -92,7 +92,7 @@ func TestQueryLog(t *testing.T) {
 	// search by client IP (part of)
 	params = newSearchParams()
 	params.searchCriteria = append(params.searchCriteria, searchCriteria{
-		criteriaType: ctClient,
+		criteriaType: ctDomainOrClient,
 		strict:       false,
 		value:        "2.2.2",
 	})

querylog/querylog.go

@@ -41,13 +41,14 @@ type Config struct {
 
 // AddParams - parameters for Add()
 type AddParams struct {
-	Question   *dns.Msg
+	Question    *dns.Msg
-	Answer     *dns.Msg          // The response we sent to the client (optional)
+	Answer      *dns.Msg          // The response we sent to the client (optional)
-	OrigAnswer *dns.Msg          // The response from an upstream server (optional)
+	OrigAnswer  *dns.Msg          // The response from an upstream server (optional)
-	Result     *dnsfilter.Result // Filtering result (optional)
+	Result      *dnsfilter.Result // Filtering result (optional)
-	Elapsed    time.Duration     // Time spent for processing the request
+	Elapsed     time.Duration     // Time spent for processing the request
-	ClientIP   net.IP
+	ClientIP    net.IP
-	Upstream   string
+	Upstream    string // Upstream server URL
+	ClientProto string // Protocol for the client connection: "" (plain), "doh", "dot"
 }
 
 // New - create a new instance of the query log

querylog/querylog_search.go

@@ -129,7 +129,9 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in
 		}
 	}
 
-	oldest = time.Unix(0, oldestNano)
+	if oldestNano != 0 {
+		oldest = time.Unix(0, oldestNano)
+	}
 	return entries, oldest, total
 }
 

querylog/search_criteria.go

@@ -9,9 +9,7 @@ import (
 type criteriaType int
 
 const (
-	ctDomain          criteriaType = iota // domain name
+	ctDomainOrClient  criteriaType = iota // domain name or client IP address
-	ctClient                              // client IP address
-	ctQuestionType                        // question type
 	ctFilteringStatus                     // filtering status
 )
 
@@ -25,6 +23,7 @@ const (
 	filteringStatusWhitelisted         = "whitelisted"          // whitelisted
 	filteringStatusRewritten           = "rewritten"            // all kinds of rewrites
 	filteringStatusSafeSearch          = "safe_search"          // enforced safe search
+	filteringStatusProcessed           = "processed"            // not blocked, not white-listed entries
 )
 
 // filteringStatusValues -- array with all possible filteringStatus values
@@ -32,6 +31,7 @@ var filteringStatusValues = []string{
 	filteringStatusAll, filteringStatusFiltered, filteringStatusBlocked,
 	filteringStatusBlockedSafebrowsing, filteringStatusBlockedParental,
 	filteringStatusWhitelisted, filteringStatusRewritten, filteringStatusSafeSearch,
+	filteringStatusProcessed,
 }
 
 // searchCriteria - every search request may contain a list of different search criteria
@@ -48,12 +48,9 @@ func (c *searchCriteria) quickMatch(line string) bool {
 	// note that we do this only for a limited set of criteria
 
 	switch c.criteriaType {
-	case ctDomain:
+	case ctDomainOrClient:
-		return c.quickMatchJSONValue(line, "QH")
+		return c.quickMatchJSONValue(line, "QH") ||
-	case ctClient:
+			c.quickMatchJSONValue(line, "IP")
-		return c.quickMatchJSONValue(line, "IP")
-	case ctQuestionType:
-		return c.quickMatchJSONValue(line, "QT")
 	default:
 		return true
 	}
@@ -80,29 +77,23 @@ func (c *searchCriteria) quickMatchJSONValue(line string, propertyName string) b
 // nolint (gocyclo)
 func (c *searchCriteria) match(entry *logEntry) bool {
 	switch c.criteriaType {
-	case ctDomain:
+	case ctDomainOrClient:
 		if c.strict && entry.QHost == c.value {
 			return true
 		}
 		if !c.strict && strings.Contains(entry.QHost, c.value) {
 			return true
 		}
-		return false
+
-	case ctClient:
 		if c.strict && entry.IP == c.value {
 			return true
 		}
 		if !c.strict && strings.Contains(entry.IP, c.value) {
 			return true
 		}
+
 		return false
-	case ctQuestionType:
+
-		if c.strict && entry.QType == c.value {
-			return true
-		}
-		if !c.strict && strings.Contains(entry.QType, c.value) {
-			return true
-		}
 	case ctFilteringStatus:
 		res := entry.Result
 
@@ -110,7 +101,10 @@ func (c *searchCriteria) match(entry *logEntry) bool {
 		case filteringStatusAll:
 			return true
 		case filteringStatusFiltered:
-			return res.IsFiltered
+			return res.IsFiltered ||
+				res.Reason == dnsfilter.NotFilteredWhiteList ||
+				res.Reason == dnsfilter.ReasonRewrite ||
+				res.Reason == dnsfilter.RewriteEtcHosts
 		case filteringStatusBlocked:
 			return res.IsFiltered &&
 				(res.Reason == dnsfilter.FilteredBlackList ||
@@ -120,19 +114,21 @@ func (c *searchCriteria) match(entry *logEntry) bool {
 		case filteringStatusBlockedSafebrowsing:
 			return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeBrowsing
 		case filteringStatusWhitelisted:
-			return res.IsFiltered && res.Reason == dnsfilter.NotFilteredWhiteList
+			return res.Reason == dnsfilter.NotFilteredWhiteList
 		case filteringStatusRewritten:
-			return res.IsFiltered &&
+			return (res.Reason == dnsfilter.ReasonRewrite ||
-				(res.Reason == dnsfilter.ReasonRewrite ||
+				res.Reason == dnsfilter.RewriteEtcHosts)
-					res.Reason == dnsfilter.RewriteEtcHosts)
 		case filteringStatusSafeSearch:
 			return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch
+
+		case filteringStatusProcessed:
+			return !(res.Reason == dnsfilter.FilteredBlackList ||
+				res.Reason == dnsfilter.FilteredBlockedService ||
+				res.Reason == dnsfilter.NotFilteredWhiteList)
+
 		default:
 			return false
 		}
-
-	default:
-		return false
 	}
 
 	return false