a
/
badguardhome
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge: + new query logs API 

* commit '59c4a2886a97143e3a36912ec895dc1a06be88cc':
  openapi
  If there are no more older entries, `"oldest":""` is returned.
  fix search by "whitelisted", "rewritten"
  doc
  fix whois test
  + "dot"
  openapi
  * minor
  + client_proto
  * openapi
  + new query logs API
This commit is contained in:
Andrey Meshkov 2020-06-18 00:23:52 +03:00
parent 4870da7f94 59c4a2886a
commit b1c951fb2c
13 changed files with 116 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
AGHTechDoc.md
View File

@ -1198,8 +1198,9 @@ When a new DNS request is received and processed, we store information about thi
"QH":"...", // target host name without the last dot
"QT":"...", // question type
"QC":"...", // question class
"Answer":"...",
"OrigAnswer":"...",
"CP":"" | "doh", // client connection protocol
"Answer":"base64 data",
"OrigAnswer":"base64 data",
"Result":{
"IsFiltered":true,
"Reason":3,
@ -1232,16 +1233,28 @@ Request:
GET /control/querylog
?older_than=2006-01-02T15:04:05.999999999Z07:00
&filter_domain=...
&filter_client=...
&filter_question_type=A | AAAA
&filter_response_status= | filtered
&search=...
&response_status="..."
`older_than` setting is used for paging. UI uses an empty value for `older_than` on the first request and gets the latest log entries. To get the older entries, UI sets `older_than` to the `oldest` value from the server's response.
If "filter" settings are set, server returns only entries that match the specified request.
If search settings are set, server returns only entries that match the specified request.
For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`. Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
`search`:
match by domain name or client IP address.
The server matches substrings by default: e.g. `adguard.com` matches `www.adguard.com`.
Strict matching can be enabled by enclosing the value in double quotes: e.g. `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
`response_status`:
* all
* filtered - all kinds of filtering
* blocked - blocked or blocked service
* blocked_safebrowsing - blocked by safebrowsing
* blocked_parental - blocked by parental control
* whitelisted - whitelisted
* rewritten - all kinds of rewrites
* safe_search - enforced safe search
* processed - not blocked, not white-listed entries
Response:
@ -1264,8 +1277,10 @@ Response:
}
...
],
"upstream":"...", // Upstream URL starting with tcp://, tls://, https://, or with an IP address
"answer_dnssec": true,
"client":"127.0.0.1",
"client_proto": "" (plain) | "doh" | "dot",
"elapsedMs":"0.098403",
"filterId":1,
"question":{
@ -1285,6 +1300,8 @@ Response:
The most recent entries are at the top of list.
If there are no more older entries, `"oldest":""` is returned.
### API: Set querylog parameters

5
dnsforward/dnsforward_http.go
View File

@ -400,6 +400,11 @@ func checkDNS(input string, bootstrap []string) error {
return nil
}
// Control flow:
// web
// -> dnsforward.handleDOH -> dnsforward.ServeHTTP
// -> proxy.ServeHTTP -> proxy.handleDNSRequest
// -> dnsforward.handleDNSRequest
func (s *Server) handleDOH(w http.ResponseWriter, r *http.Request) {
if !s.conf.TLSAllowUnencryptedDOH && r.TLS == nil {
httpError(r, w, http.StatusNotFound, "Not Found")

7
dnsforward/stats.go
View File

@ -39,6 +39,13 @@ func processQueryLogsAndStats(ctx *dnsContext) int {
Elapsed: elapsed,
ClientIP: getIP(d.Addr),
}
if d.Proto == "https" {
p.ClientProto = "doh"
} else if d.Proto == "tls" {
p.ClientProto = "dot"
}
if d.Upstream != nil {
p.Upstream = d.Upstream.Address()
}

8
home/whois_test.go
View File

@ -27,9 +27,9 @@ func TestWhois(t *testing.T) {
w := Whois{timeoutMsec: 5000}
resp, err := w.queryAll("8.8.8.8")
assert.True(t, err == nil)
assert.Nil(t, err)
m := whoisParse(resp)
assert.True(t, m["orgname"] == "Google LLC")
assert.True(t, m["country"] == "US")
assert.True(t, m["city"] == "Mountain View")
assert.Equal(t, "Google LLC", m["orgname"])
assert.Equal(t, "US", m["country"])
assert.Equal(t, "Mountain View", m["city"])
}

33
openapi/openapi.yaml
View File

@ -163,29 +163,26 @@ paths:
description: Limit the number of records to be returned
schema:
type: integer
- name: filter_domain
- name: search
in: query
description: Filter by domain name
description: Filter by domain name or client IP
schema:
type: string
- name: filter_client
in: query
description: Filter by client
schema:
type: string
- name: filter_question_type
in: query
description: Filter by question type
schema:
type: string
- name: filter_response_status
- name: response_status
in: query
description: Filter by response status
schema:
type: string
enum:
- null
- all
- filtered
- blocked
- blocked_safebrowsing
- blocked_parental
- whitelisted
- rewritten
- safe_search
- processed
responses:
"200":
description: OK
@ -1420,11 +1417,19 @@ components:
description: Answer from upstream server (optional)
items:
$ref: "#/components/schemas/DnsAnswer"
upstream:
type: string
description: Upstream URL starting with tcp://, tls://, https://, or with an IP address
answer_dnssec:
type: boolean
client:
type: string
example: 192.168.0.1
client_proto:
enum:
- dot
- doh
- ""
elapsedMs:
type: string
example: "54.023928"

3
querylog/decode.go
View File

@ -37,6 +37,9 @@ func decodeLogEntry(ent *logEntry, str string) {
case "QC":
ent.QClass = v
case "CP":
ent.ClientProto = v
case "Answer":
ent.Answer, err = base64.StdEncoding.DecodeString(v)
case "OrigAnswer":

11
querylog/json.go
View File

@ -63,10 +63,11 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
jsonEntry := map[string]interface{}{
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"client_proto": entry.ClientProto,
}
jsonEntry["question"] = map[string]interface{}{
"host": entry.QHost,
@ -112,6 +113,8 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
}
jsonEntry["upstream"] = entry.Upstream
return jsonEntry
}

9
querylog/qlog.go
View File

@ -38,6 +38,8 @@ type logEntry struct {
QType string `json:"QT"`
QClass string `json:"QC"`
ClientProto string `json:"CP"` // "" or "doh"
Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net
OrigAnswer []byte `json:",omitempty"`
@ -119,9 +121,10 @@ func (l *queryLog) Add(params AddParams) {
IP: l.getClientIP(params.ClientIP.String()),
Time: now,
Result: *params.Result,
Elapsed: params.Elapsed,
Upstream: params.Upstream,
Result: *params.Result,
Elapsed: params.Elapsed,
Upstream: params.Upstream,
ClientProto: params.ClientProto,
}
q := params.Question.Question[0]
entry.QHost = strings.ToLower(q.Name[:len(q.Name)-1]) // remove the last dot

10
querylog/qlog_http.go
View File

@ -142,10 +142,6 @@ func (l *queryLog) parseSearchCriteria(q url.Values, name string, ct criteriaTyp
c.strict = true
}
if ct == ctClient && l.conf.AnonymizeClientIP {
c.value = l.getClientIP(c.value)
}
if ct == ctFilteringStatus && !util.ContainsString(filteringStatusValues, c.value) {
return false, c, fmt.Errorf("invalid value %s", c.value)
}
@ -180,10 +176,8 @@ func (l *queryLog) parseSearchParams(r *http.Request) (*searchParams, error) {
}
paramNames := map[string]criteriaType{
"filter_domain": ctDomain,
"filter_client": ctClient,
"filter_question_type": ctQuestionType,
"filter_response_status": ctFilteringStatus,
"search": ctDomainOrClient,
"response_status": ctFilteringStatus,
}
for k, v := range paramNames {

8
querylog/qlog_test.go
View File

@ -57,7 +57,7 @@ func TestQueryLog(t *testing.T) {
// search by domain (strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctDomain,
criteriaType: ctDomainOrClient,
strict: true,
value: "test.example.org",
})
@ -68,7 +68,7 @@ func TestQueryLog(t *testing.T) {
// search by domain (not strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctDomain,
criteriaType: ctDomainOrClient,
strict: false,
value: "example.org",
})
@ -81,7 +81,7 @@ func TestQueryLog(t *testing.T) {
// search by client IP (strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctClient,
criteriaType: ctDomainOrClient,
strict: true,
value: "2.2.2.2",
})
@ -92,7 +92,7 @@ func TestQueryLog(t *testing.T) {
// search by client IP (part of)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctClient,
criteriaType: ctDomainOrClient,
strict: false,
value: "2.2.2",
})

15
querylog/querylog.go
View File

@ -41,13 +41,14 @@ type Config struct {
// AddParams - parameters for Add()
type AddParams struct {
Question *dns.Msg
Answer *dns.Msg // The response we sent to the client (optional)
OrigAnswer *dns.Msg // The response from an upstream server (optional)
Result *dnsfilter.Result // Filtering result (optional)
Elapsed time.Duration // Time spent for processing the request
ClientIP net.IP
Upstream string
Question *dns.Msg
Answer *dns.Msg // The response we sent to the client (optional)
OrigAnswer *dns.Msg // The response from an upstream server (optional)
Result *dnsfilter.Result // Filtering result (optional)
Elapsed time.Duration // Time spent for processing the request
ClientIP net.IP
Upstream string // Upstream server URL
ClientProto string // Protocol for the client connection: "" (plain), "doh", "dot"
}
// New - create a new instance of the query log

4
querylog/querylog_search.go
View File

@ -129,7 +129,9 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in
}
}
oldest = time.Unix(0, oldestNano)
if oldestNano != 0 {
oldest = time.Unix(0, oldestNano)
}
return entries, oldest, total
}

50
querylog/search_criteria.go
View File

@ -9,9 +9,7 @@ import (
type criteriaType int
const (
ctDomain criteriaType = iota // domain name
ctClient // client IP address
ctQuestionType // question type
ctDomainOrClient criteriaType = iota // domain name or client IP address
ctFilteringStatus // filtering status
)
@ -25,6 +23,7 @@ const (
filteringStatusWhitelisted = "whitelisted" // whitelisted
filteringStatusRewritten = "rewritten" // all kinds of rewrites
filteringStatusSafeSearch = "safe_search" // enforced safe search
filteringStatusProcessed = "processed" // not blocked, not white-listed entries
)
// filteringStatusValues -- array with all possible filteringStatus values
@ -32,6 +31,7 @@ var filteringStatusValues = []string{
filteringStatusAll, filteringStatusFiltered, filteringStatusBlocked,
filteringStatusBlockedSafebrowsing, filteringStatusBlockedParental,
filteringStatusWhitelisted, filteringStatusRewritten, filteringStatusSafeSearch,
filteringStatusProcessed,
}
// searchCriteria - every search request may contain a list of different search criteria
@ -48,12 +48,9 @@ func (c *searchCriteria) quickMatch(line string) bool {
// note that we do this only for a limited set of criteria
switch c.criteriaType {
case ctDomain:
return c.quickMatchJSONValue(line, "QH")
case ctClient:
return c.quickMatchJSONValue(line, "IP")
case ctQuestionType:
return c.quickMatchJSONValue(line, "QT")
case ctDomainOrClient:
return c.quickMatchJSONValue(line, "QH") ||
c.quickMatchJSONValue(line, "IP")
default:
return true
}
@ -80,29 +77,23 @@ func (c *searchCriteria) quickMatchJSONValue(line string, propertyName string) b
// nolint (gocyclo)
func (c *searchCriteria) match(entry *logEntry) bool {
switch c.criteriaType {
case ctDomain:
case ctDomainOrClient:
if c.strict && entry.QHost == c.value {
return true
}
if !c.strict && strings.Contains(entry.QHost, c.value) {
return true
}
return false
case ctClient:
if c.strict && entry.IP == c.value {
return true
}
if !c.strict && strings.Contains(entry.IP, c.value) {
return true
}
return false
case ctQuestionType:
if c.strict && entry.QType == c.value {
return true
}
if !c.strict && strings.Contains(entry.QType, c.value) {
return true
}
case ctFilteringStatus:
res := entry.Result
@ -110,7 +101,10 @@ func (c *searchCriteria) match(entry *logEntry) bool {
case filteringStatusAll:
return true
case filteringStatusFiltered:
return res.IsFiltered
return res.IsFiltered ||
res.Reason == dnsfilter.NotFilteredWhiteList ||
res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts
case filteringStatusBlocked:
return res.IsFiltered &&
(res.Reason == dnsfilter.FilteredBlackList ||
@ -120,19 +114,21 @@ func (c *searchCriteria) match(entry *logEntry) bool {
case filteringStatusBlockedSafebrowsing:
return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeBrowsing
case filteringStatusWhitelisted:
return res.IsFiltered && res.Reason == dnsfilter.NotFilteredWhiteList
return res.Reason == dnsfilter.NotFilteredWhiteList
case filteringStatusRewritten:
return res.IsFiltered &&
(res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts)
return (res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts)
case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch
case filteringStatusProcessed:
return !(res.Reason == dnsfilter.FilteredBlackList ||
res.Reason == dnsfilter.FilteredBlockedService ||
res.Reason == dnsfilter.NotFilteredWhiteList)
default:
return false
}
default:
return false
}
return false