From 2e845e4f4d5f35bfd3d68b4946fb1f576e7362c3 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Tue, 3 Mar 2020 20:21:53 +0300 Subject: [PATCH] + qlog: hide_client_ip setting --- AGHTechDoc.md | 11 +++++++++++ home/config.go | 8 +++++--- home/dns.go | 22 ++++++++++++---------- openapi/openapi.yaml | 3 +++ querylog/qlog.go | 33 +++++++++++++++++++++++++++++---- querylog/qlog_http.go | 9 +++++++-- querylog/querylog.go | 16 +++++++++------- stats/stats.go | 7 ++++--- stats/stats_unit.go | 22 +++++++++++++++++++++- 9 files changed, 101 insertions(+), 30 deletions(-) diff --git a/AGHTechDoc.md b/AGHTechDoc.md index b1303621..8adcc068 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -1287,12 +1287,22 @@ Request: { "enabled": true | false "interval": 1 | 7 | 30 | 90 + "anonymize_client_ip": true | false // anonymize clients' IP addresses } Response: 200 OK +`anonymize_client_ip`: +1. New log entries written to a log file will contain modified client IP addresses. Note that there's no way to obtain the full IP address later for these entries. +2. `GET /control/querylog` response data will contain modified client IP addresses (masked /24 or /112). +3. Searching by client IP won't work for the previously stored entries. + +How `anonymize_client_ip` affects Stats: +1. After AGH restart, new stats entries will contain modified client IP addresses. +2. Existing entries are not affected. + ### API: Get querylog parameters @@ -1307,6 +1317,7 @@ Response: { "enabled": true | false "interval": 1 | 7 | 30 | 90 + "anonymize_client_ip": true | false } diff --git a/home/config.go b/home/config.go index 78c5dfda..2d78b42c 100644 --- a/home/config.go +++ b/home/config.go @@ -77,9 +77,10 @@ type dnsConfig struct { // time interval for statistics (in days) StatsInterval uint32 `yaml:"statistics_interval"` - QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled - QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) - QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk + QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled + QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days) + QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk + AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats dnsforward.FilteringConfig `yaml:",inline"` @@ -242,6 +243,7 @@ func (c *configuration) write() error { config.DNS.QueryLogEnabled = dc.Enabled config.DNS.QueryLogInterval = dc.Interval config.DNS.QueryLogMemSize = dc.MemSize + config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP } if Context.dnsFilter != nil { diff --git a/home/dns.go b/home/dns.go index 29cec636..c9cfb513 100644 --- a/home/dns.go +++ b/home/dns.go @@ -29,22 +29,24 @@ func initDNSServer() error { baseDir := Context.getDataDir() statsConf := stats.Config{ - Filename: filepath.Join(baseDir, "stats.db"), - LimitDays: config.DNS.StatsInterval, - ConfigModified: onConfigModified, - HTTPRegister: httpRegister, + Filename: filepath.Join(baseDir, "stats.db"), + LimitDays: config.DNS.StatsInterval, + AnonymizeClientIP: config.DNS.AnonymizeClientIP, + ConfigModified: onConfigModified, + HTTPRegister: httpRegister, } Context.stats, err = stats.New(statsConf) if err != nil { return fmt.Errorf("Couldn't initialize statistics module") } conf := querylog.Config{ - Enabled: config.DNS.QueryLogEnabled, - BaseDir: baseDir, - Interval: config.DNS.QueryLogInterval, - MemSize: config.DNS.QueryLogMemSize, - ConfigModified: onConfigModified, - HTTPRegister: httpRegister, + Enabled: config.DNS.QueryLogEnabled, + BaseDir: baseDir, + Interval: config.DNS.QueryLogInterval, + MemSize: config.DNS.QueryLogMemSize, + AnonymizeClientIP: config.DNS.AnonymizeClientIP, + ConfigModified: onConfigModified, + HTTPRegister: httpRegister, } Context.queryLog = querylog.New(conf) diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index e597786f..64fb5e5d 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1585,6 +1585,9 @@ definitions: interval: type: "integer" description: "Time period to keep data (1 | 7 | 30 | 90)" + anonymize_client_ip: + type: "boolean" + description: "Anonymize clients' IP addresses" TlsConfig: type: "object" diff --git a/querylog/qlog.go b/querylog/qlog.go index 2cf1c013..9dab0acb 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -2,6 +2,7 @@ package querylog import ( "fmt" + "net" "os" "path/filepath" "strconv" @@ -66,6 +67,7 @@ func (l *queryLog) WriteDiskConfig(dc *DiskConfig) { dc.Enabled = l.conf.Enabled dc.Interval = l.conf.Interval dc.MemSize = l.conf.MemSize + dc.AnonymizeClientIP = l.conf.AnonymizeClientIP } // Clear memory buffer and remove log files @@ -123,7 +125,7 @@ func (l *queryLog) Add(params AddParams) { now := time.Now() entry := logEntry{ - IP: params.ClientIP.String(), + IP: l.getClientIP(params.ClientIP.String()), Time: now, Result: *params.Result, @@ -196,6 +198,10 @@ const ( func (l *queryLog) getData(params getDataParams) map[string]interface{} { now := time.Now() + if len(params.Client) != 0 && l.conf.AnonymizeClientIP { + params.Client = l.getClientIP(params.Client) + } + // add from file fileEntries, oldest, total := l.searchFiles(params) @@ -246,7 +252,7 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} { // the elements order is already reversed (from newer to older) for i := 0; i < len(entries); i++ { entry := entries[i] - jsonEntry := logEntryToJSONEntry(entry) + jsonEntry := l.logEntryToJSONEntry(entry) data = append(data, jsonEntry) } @@ -262,7 +268,26 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} { return result } -func logEntryToJSONEntry(entry *logEntry) map[string]interface{} { +// Get Client IP address +func (l *queryLog) getClientIP(clientIP string) string { + if l.conf.AnonymizeClientIP { + ip := net.ParseIP(clientIP) + if ip != nil { + ip4 := ip.To4() + const AnonymizeClientIP4Mask = 24 + const AnonymizeClientIP6Mask = 112 + if ip4 != nil { + clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String() + } else { + clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String() + } + } + } + + return clientIP +} + +func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} { var msg *dns.Msg if len(entry.Answer) > 0 { @@ -277,7 +302,7 @@ func logEntryToJSONEntry(entry *logEntry) map[string]interface{} { "reason": entry.Result.Reason.String(), "elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64), "time": entry.Time.Format(time.RFC3339Nano), - "client": entry.IP, + "client": l.getClientIP(entry.IP), } jsonEntry["question"] = map[string]interface{}{ "host": entry.QHost, diff --git a/querylog/qlog_http.go b/querylog/qlog_http.go index 07a3aadd..fae8dba6 100644 --- a/querylog/qlog_http.go +++ b/querylog/qlog_http.go @@ -106,8 +106,9 @@ func (l *queryLog) handleQueryLogClear(w http.ResponseWriter, r *http.Request) { } type qlogConfig struct { - Enabled bool `json:"enabled"` - Interval uint32 `json:"interval"` + Enabled bool `json:"enabled"` + Interval uint32 `json:"interval"` + AnonymizeClientIP bool `json:"anonymize_client_ip"` } // Get configuration @@ -115,6 +116,7 @@ func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) { resp := qlogConfig{} resp.Enabled = l.conf.Enabled resp.Interval = l.conf.Interval + resp.AnonymizeClientIP = l.conf.AnonymizeClientIP jsonVal, err := json.Marshal(resp) if err != nil { @@ -151,6 +153,9 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) if req.Exists("interval") { conf.Interval = d.Interval } + if req.Exists("anonymize_client_ip") { + conf.AnonymizeClientIP = d.AnonymizeClientIP + } l.conf = &conf l.lock.Unlock() diff --git a/querylog/querylog.go b/querylog/querylog.go index dcca14dd..0e079ec3 100644 --- a/querylog/querylog.go +++ b/querylog/querylog.go @@ -11,9 +11,10 @@ import ( // DiskConfig - configuration settings that are stored on disk type DiskConfig struct { - Enabled bool - Interval uint32 - MemSize uint32 + Enabled bool + Interval uint32 + MemSize uint32 + AnonymizeClientIP bool } // QueryLog - main interface @@ -32,10 +33,11 @@ type QueryLog interface { // Config - configuration object type Config struct { - Enabled bool - BaseDir string // directory where log file is stored - Interval uint32 // interval to rotate logs (in days) - MemSize uint32 // number of entries kept in memory before they are flushed to disk + Enabled bool + BaseDir string // directory where log file is stored + Interval uint32 // interval to rotate logs (in days) + MemSize uint32 // number of entries kept in memory before they are flushed to disk + AnonymizeClientIP bool // anonymize clients' IP addresses // Called when the configuration is changed by HTTP request ConfigModified func() diff --git a/stats/stats.go b/stats/stats.go index 91b6b25f..8f77425f 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -16,9 +16,10 @@ type DiskConfig struct { // Config - module configuration type Config struct { - Filename string // database file name - LimitDays uint32 // time limit (in days) - UnitID unitIDCallback // user function to get the current unit ID. If nil, the current time hour is used. + Filename string // database file name + LimitDays uint32 // time limit (in days) + UnitID unitIDCallback // user function to get the current unit ID. If nil, the current time hour is used. + AnonymizeClientIP bool // anonymize clients' IP addresses // Called when the configuration is changed by HTTP request ConfigModified func() diff --git a/stats/stats_unit.go b/stats/stats_unit.go index 44aa66d5..5126e69c 100644 --- a/stats/stats_unit.go +++ b/stats/stats_unit.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "encoding/gob" "fmt" + "net" "os" "sort" "sync" @@ -442,6 +443,25 @@ func (s *statsCtx) clear() { log.Debug("Stats: cleared") } +// Get Client IP address +func (s *statsCtx) getClientIP(clientIP string) string { + if s.conf.AnonymizeClientIP { + ip := net.ParseIP(clientIP) + if ip != nil { + ip4 := ip.To4() + const AnonymizeClientIP4Mask = 24 + const AnonymizeClientIP6Mask = 112 + if ip4 != nil { + clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String() + } else { + clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String() + } + } + } + + return clientIP +} + func (s *statsCtx) Update(e Entry) { if e.Result == 0 || e.Result >= rLast || @@ -449,7 +469,7 @@ func (s *statsCtx) Update(e Entry) { !(len(e.Client) == 4 || len(e.Client) == 16) { return } - client := e.Client.String() + client := s.getClientIP(e.Client.String()) s.unitLock.Lock() u := s.unit