From 61a24ffc71eda26e10a11e3a9c8506909b6c4b52 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 26 May 2020 16:24:31 +0300 Subject: [PATCH] *(querylog): decode methods to a new file --- querylog/decode.go | 175 ++++++++++++++++++++++++++++++++++++++++ querylog/decode_test.go | 34 ++++++++ querylog/json.go | 166 ------------------------------------- querylog/qlog_test.go | 27 ------- 4 files changed, 209 insertions(+), 193 deletions(-) create mode 100644 querylog/decode.go create mode 100644 querylog/decode_test.go diff --git a/querylog/decode.go b/querylog/decode.go new file mode 100644 index 00000000..3f381140 --- /dev/null +++ b/querylog/decode.go @@ -0,0 +1,175 @@ +package querylog + +import ( + "encoding/base64" + "strconv" + "strings" + "time" + + "github.com/AdguardTeam/AdGuardHome/dnsfilter" + "github.com/AdguardTeam/golibs/log" + "github.com/miekg/dns" +) + +// decodeLogEntry - decodes query log entry from a line +// nolint (gocyclo) +func decodeLogEntry(ent *logEntry, str string) { + var b bool + var i int + var err error + for { + k, v, t := readJSON(&str) + if t == jsonTErr { + break + } + switch k { + case "IP": + if len(ent.IP) == 0 { + ent.IP = v + } + case "T": + ent.Time, err = time.Parse(time.RFC3339, v) + + case "QH": + ent.QHost = v + case "QT": + ent.QType = v + case "QC": + ent.QClass = v + + case "Answer": + ent.Answer, err = base64.StdEncoding.DecodeString(v) + case "OrigAnswer": + ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v) + + case "IsFiltered": + b, err = strconv.ParseBool(v) + ent.Result.IsFiltered = b + case "Rule": + ent.Result.Rule = v + case "FilterID": + i, err = strconv.Atoi(v) + ent.Result.FilterID = int64(i) + case "Reason": + i, err = strconv.Atoi(v) + ent.Result.Reason = dnsfilter.Reason(i) + + case "Upstream": + ent.Upstream = v + case "Elapsed": + i, err = strconv.Atoi(v) + ent.Elapsed = time.Duration(i) + + // pre-v0.99.3 compatibility: + case "Question": + var qstr []byte + qstr, err = base64.StdEncoding.DecodeString(v) + if err != nil { + break + } + q := new(dns.Msg) + err = q.Unpack(qstr) + if err != nil { + break + } + ent.QHost = q.Question[0].Name + if len(ent.QHost) == 0 { + break + } + ent.QHost = ent.QHost[:len(ent.QHost)-1] + ent.QType = dns.TypeToString[q.Question[0].Qtype] + ent.QClass = dns.ClassToString[q.Question[0].Qclass] + case "Time": + ent.Time, err = time.Parse(time.RFC3339, v) + } + + if err != nil { + log.Debug("decodeLogEntry err: %s", err) + break + } + } +} + +// Get value from "key":"value" +func readJSONValue(s, name string) string { + i := strings.Index(s, "\""+name+"\":\"") + if i == -1 { + return "" + } + start := i + 1 + len(name) + 3 + i = strings.IndexByte(s[start:], '"') + if i == -1 { + return "" + } + end := start + i + return s[start:end] +} + +const ( + jsonTErr = iota + jsonTObj + jsonTStr + jsonTNum + jsonTBool +) + +// Parse JSON key-value pair +// e.g.: "key":VALUE where VALUE is "string", true|false (boolean), or 123.456 (number) +// Note the limitations: +// . doesn't support whitespace +// . doesn't support "null" +// . doesn't validate boolean or number +// . no proper handling of {} braces +// . no handling of [] brackets +// Return (key, value, type) +func readJSON(ps *string) (string, string, int32) { + s := *ps + k := "" + v := "" + t := int32(jsonTErr) + + q1 := strings.IndexByte(s, '"') + if q1 == -1 { + return k, v, t + } + q2 := strings.IndexByte(s[q1+1:], '"') + if q2 == -1 { + return k, v, t + } + k = s[q1+1 : q1+1+q2] + s = s[q1+1+q2+1:] + + if len(s) < 2 || s[0] != ':' { + return k, v, t + } + + if s[1] == '"' { + q2 = strings.IndexByte(s[2:], '"') + if q2 == -1 { + return k, v, t + } + v = s[2 : 2+q2] + t = jsonTStr + s = s[2+q2+1:] + + } else if s[1] == '{' { + t = jsonTObj + s = s[1+1:] + + } else { + sep := strings.IndexAny(s[1:], ",}") + if sep == -1 { + return k, v, t + } + v = s[1 : 1+sep] + if s[1] == 't' || s[1] == 'f' { + t = jsonTBool + } else if s[1] == '.' || (s[1] >= '0' && s[1] <= '9') { + t = jsonTNum + } + s = s[1+sep+1:] + } + + *ps = s + return k, v, t +} diff --git a/querylog/decode_test.go b/querylog/decode_test.go new file mode 100644 index 00000000..b5b3b7d9 --- /dev/null +++ b/querylog/decode_test.go @@ -0,0 +1,34 @@ +package querylog + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJSON(t *testing.T) { + s := ` + {"keystr":"val","obj":{"keybool":true,"keyint":123456}} + ` + k, v, jtype := readJSON(&s) + assert.Equal(t, jtype, int32(jsonTStr)) + assert.Equal(t, "keystr", k) + assert.Equal(t, "val", v) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTObj)) + assert.Equal(t, "obj", k) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTBool)) + assert.Equal(t, "keybool", k) + assert.Equal(t, "true", v) + + k, v, jtype = readJSON(&s) + assert.Equal(t, jtype, int32(jsonTNum)) + assert.Equal(t, "keyint", k) + assert.Equal(t, "123456", v) + + k, v, jtype = readJSON(&s) + assert.True(t, jtype == jsonTErr) +} diff --git a/querylog/json.go b/querylog/json.go index 7efed29d..86f35e11 100644 --- a/querylog/json.go +++ b/querylog/json.go @@ -1,181 +1,15 @@ package querylog import ( - "encoding/base64" "fmt" "net" "strconv" - "strings" "time" - "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/golibs/log" "github.com/miekg/dns" ) -// decodeLogEntry - decodes query log entry from a line -// nolint (gocyclo) -func decodeLogEntry(ent *logEntry, str string) { - var b bool - var i int - var err error - for { - k, v, t := readJSON(&str) - if t == jsonTErr { - break - } - switch k { - case "IP": - if len(ent.IP) == 0 { - ent.IP = v - } - case "T": - ent.Time, err = time.Parse(time.RFC3339, v) - - case "QH": - ent.QHost = v - case "QT": - ent.QType = v - case "QC": - ent.QClass = v - - case "Answer": - ent.Answer, err = base64.StdEncoding.DecodeString(v) - case "OrigAnswer": - ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v) - - case "IsFiltered": - b, err = strconv.ParseBool(v) - ent.Result.IsFiltered = b - case "Rule": - ent.Result.Rule = v - case "FilterID": - i, err = strconv.Atoi(v) - ent.Result.FilterID = int64(i) - case "Reason": - i, err = strconv.Atoi(v) - ent.Result.Reason = dnsfilter.Reason(i) - - case "Upstream": - ent.Upstream = v - case "Elapsed": - i, err = strconv.Atoi(v) - ent.Elapsed = time.Duration(i) - - // pre-v0.99.3 compatibility: - case "Question": - var qstr []byte - qstr, err = base64.StdEncoding.DecodeString(v) - if err != nil { - break - } - q := new(dns.Msg) - err = q.Unpack(qstr) - if err != nil { - break - } - ent.QHost = q.Question[0].Name - if len(ent.QHost) == 0 { - break - } - ent.QHost = ent.QHost[:len(ent.QHost)-1] - ent.QType = dns.TypeToString[q.Question[0].Qtype] - ent.QClass = dns.ClassToString[q.Question[0].Qclass] - case "Time": - ent.Time, err = time.Parse(time.RFC3339, v) - } - - if err != nil { - log.Debug("decodeLogEntry err: %s", err) - break - } - } -} - -// Get value from "key":"value" -func readJSONValue(s, name string) string { - i := strings.Index(s, "\""+name+"\":\"") - if i == -1 { - return "" - } - start := i + 1 + len(name) + 3 - i = strings.IndexByte(s[start:], '"') - if i == -1 { - return "" - } - end := start + i - return s[start:end] -} - -const ( - jsonTErr = iota - jsonTObj - jsonTStr - jsonTNum - jsonTBool -) - -// Parse JSON key-value pair -// e.g.: "key":VALUE where VALUE is "string", true|false (boolean), or 123.456 (number) -// Note the limitations: -// . doesn't support whitespace -// . doesn't support "null" -// . doesn't validate boolean or number -// . no proper handling of {} braces -// . no handling of [] brackets -// Return (key, value, type) -func readJSON(ps *string) (string, string, int32) { - s := *ps - k := "" - v := "" - t := int32(jsonTErr) - - q1 := strings.IndexByte(s, '"') - if q1 == -1 { - return k, v, t - } - q2 := strings.IndexByte(s[q1+1:], '"') - if q2 == -1 { - return k, v, t - } - k = s[q1+1 : q1+1+q2] - s = s[q1+1+q2+1:] - - if len(s) < 2 || s[0] != ':' { - return k, v, t - } - - if s[1] == '"' { - q2 = strings.IndexByte(s[2:], '"') - if q2 == -1 { - return k, v, t - } - v = s[2 : 2+q2] - t = jsonTStr - s = s[2+q2+1:] - - } else if s[1] == '{' { - t = jsonTObj - s = s[1+1:] - - } else { - sep := strings.IndexAny(s[1:], ",}") - if sep == -1 { - return k, v, t - } - v = s[1 : 1+sep] - if s[1] == 't' || s[1] == 'f' { - t = jsonTBool - } else if s[1] == '.' || (s[1] >= '0' && s[1] <= '9') { - t = jsonTNum - } - s = s[1+sep+1:] - } - - *ps = s - return k, v, t -} - // Get Client IP address func (l *queryLog) getClientIP(clientIP string) string { if l.conf.AnonymizeClientIP { diff --git a/querylog/qlog_test.go b/querylog/qlog_test.go index 68c346f5..a5392fc0 100644 --- a/querylog/qlog_test.go +++ b/querylog/qlog_test.go @@ -183,33 +183,6 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) { assert.Equal(t, 10, len(entries)) } -func TestJSON(t *testing.T) { - s := ` - {"keystr":"val","obj":{"keybool":true,"keyint":123456}} - ` - k, v, jtype := readJSON(&s) - assert.Equal(t, jtype, int32(jsonTStr)) - assert.Equal(t, "keystr", k) - assert.Equal(t, "val", v) - - k, v, jtype = readJSON(&s) - assert.Equal(t, jtype, int32(jsonTObj)) - assert.Equal(t, "obj", k) - - k, v, jtype = readJSON(&s) - assert.Equal(t, jtype, int32(jsonTBool)) - assert.Equal(t, "keybool", k) - assert.Equal(t, "true", v) - - k, v, jtype = readJSON(&s) - assert.Equal(t, jtype, int32(jsonTNum)) - assert.Equal(t, "keyint", k) - assert.Equal(t, "123456", v) - - k, v, jtype = readJSON(&s) - assert.True(t, jtype == jsonTErr) -} - func addEntry(l *queryLog, host, answerStr, client string) { q := dns.Msg{} q.Question = append(q.Question, dns.Question{