From 56c69cdb793df5938f45ea56ac7c9a180063cf52 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Fri, 16 Aug 2019 14:54:11 +0300 Subject: [PATCH 1/4] Revert "fix tests" This reverts commit d9265aa9a8c21d97206b0e43a03b448f2c33a510. --- dnsfilter/dnsfilter_test.go | 10 +++------- dnsforward/dnsforward_test.go | 4 ---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go index 15ecec80..06658afd 100644 --- a/dnsfilter/dnsfilter_test.go +++ b/dnsfilter/dnsfilter_test.go @@ -30,13 +30,13 @@ var setts RequestFilteringSettings func purgeCaches() { if gctx.safebrowsingCache != nil { - gctx.safebrowsingCache.Reset() + gctx.safebrowsingCache.Purge() } if gctx.parentalCache != nil { - gctx.parentalCache.Reset() + gctx.parentalCache.Purge() } if gctx.safeSearchCache != nil { - gctx.safeSearchCache.Reset() + gctx.safeSearchCache.Purge() } } @@ -51,10 +51,6 @@ func NewForTest(c *Config, filters map[int]string) *Dnsfilter { setts = RequestFilteringSettings{} setts.FilteringEnabled = true if c != nil { - c.SafeBrowsingCacheSize = 1024 - c.SafeSearchCacheSize = 1024 - c.ParentalCacheSize = 1024 - setts.SafeSearchEnabled = c.SafeSearchEnabled setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled setts.ParentalEnabled = c.ParentalEnabled diff --git a/dnsforward/dnsforward_test.go b/dnsforward/dnsforward_test.go index d190ea37..6b685510 100644 --- a/dnsforward/dnsforward_test.go +++ b/dnsforward/dnsforward_test.go @@ -458,10 +458,6 @@ func createTestServer(t *testing.T) *Server { s.conf.FilteringConfig.SafeBrowsingEnabled = true s.conf.Filters = make([]dnsfilter.Filter, 0) - s.conf.SafeBrowsingCacheSize = 1024 - s.conf.SafeSearchCacheSize = 1024 - s.conf.ParentalCacheSize = 1024 - rules := "||nxdomain.example.org^\n||null.example.org^\n127.0.0.1 host.example.org\n" filter := dnsfilter.Filter{ID: 0, Data: []byte(rules)} s.conf.Filters = append(s.conf.Filters, filter) From 68d5d595b657ebd1f93e1bd012bc8fc0d5d9f7b0 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Fri, 16 Aug 2019 15:07:24 +0300 Subject: [PATCH 2/4] Revert "* dnsfilter: use fastcache instead of gcache" This reverts commit 6f51df7d2e3eb4672d4762e14e22e831fdadd256. --- dnsfilter/dnsfilter.go | 56 ++++++++++++++++++++++-------------------- go.mod | 5 ++-- go.sum | 24 +++--------------- 3 files changed, 36 insertions(+), 49 deletions(-) diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 4acd4ee7..b1dcf04a 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/gob" "encoding/json" "fmt" "io/ioutil" @@ -16,7 +15,6 @@ import ( "sync/atomic" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/joomcode/errorx" "github.com/AdguardTeam/dnsproxy/upstream" @@ -177,9 +175,9 @@ func (i Reason) String() string { type dnsFilterContext struct { stats Stats dialCache gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers - safebrowsingCache *fastcache.Cache - parentalCache *fastcache.Cache - safeSearchCache *fastcache.Cache + safebrowsingCache gcache.Cache + parentalCache gcache.Cache + safeSearchCache gcache.Cache } var gctx dnsFilterContext // global dnsfilter context @@ -357,33 +355,39 @@ func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result { return res } -func setCacheResult(cache *fastcache.Cache, host string, res Result) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - _ = enc.Encode(res) - cache.Set([]byte(host), buf.Bytes()) - log.Debug("Stored in cache %p: %s => [%d]", cache, host, buf.Len()) +func setCacheResult(cache gcache.Cache, host string, res Result) { + err := cache.Set(host, res) + if err != nil { + log.Debug("cache.Set: %s", err) + return + } + log.Debug("Stored in cache %p: %s", cache, host) } -func getCachedResult(cache *fastcache.Cache, host string) (result Result, isFound bool) { +func getCachedResult(cache gcache.Cache, host string) (result Result, isFound bool) { isFound = false // not found yet // get raw value - rawValue := cache.Get(nil, []byte(host)) - if len(rawValue) == 0 { + rawValue, err := cache.Get(host) + if err == gcache.KeyNotFoundError { + // not a real error, just not found + return Result{}, false + } + if err != nil { + // real error return Result{}, false } - var buf bytes.Buffer - buf.Write(rawValue) - dec := gob.NewDecoder(&buf) - cachedValue := Result{} - err := dec.Decode(&cachedValue) - if err != nil { - log.Debug("gob.Decode(): %s", err) - return Result{}, false + // since it can be something else, validate that it belongs to proper type + cachedValue, ok := rawValue.(Result) + if !ok { + // this is not our type -- error + text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache" + log.Println(text) + return } - return cachedValue, true + isFound = ok + return cachedValue, isFound } // for each dot, hash it and add it to string @@ -853,13 +857,13 @@ func New(c *Config, filters map[int]string) *Dnsfilter { if c != nil { // initialize objects only once if gctx.safebrowsingCache == nil { - gctx.safebrowsingCache = fastcache.New(c.SafeBrowsingCacheSize) + gctx.safebrowsingCache = gcache.New(c.SafeBrowsingCacheSize).LRU().Expiration(defaultCacheTime).Build() } if gctx.safeSearchCache == nil { - gctx.safeSearchCache = fastcache.New(c.SafeSearchCacheSize) + gctx.safeSearchCache = gcache.New(c.SafeSearchCacheSize).LRU().Expiration(defaultCacheTime).Build() } if gctx.parentalCache == nil { - gctx.parentalCache = fastcache.New(c.ParentalCacheSize) + gctx.parentalCache = gcache.New(c.ParentalCacheSize).LRU().Expiration(defaultCacheTime).Build() } if len(c.ResolverAddress) != 0 && gctx.dialCache == nil { gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build() diff --git a/go.mod b/go.mod index 833615bf..c7f29ffa 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,11 @@ module github.com/AdguardTeam/AdGuardHome go 1.12 require ( - github.com/AdguardTeam/dnsproxy v0.18.1 + github.com/AdguardTeam/dnsproxy v0.18.2 github.com/AdguardTeam/golibs v0.1.4 github.com/AdguardTeam/urlfilter v0.5.0 github.com/NYTimes/gziphandler v1.1.1 - github.com/VictoriaMetrics/fastcache v1.5.1 - github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 + github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 github.com/go-test/deep v1.0.1 github.com/gobuffalo/packr v1.19.0 github.com/joomcode/errorx v0.8.0 diff --git a/go.sum b/go.sum index d4d7ada8..6a2f32e8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/AdguardTeam/dnsproxy v0.18.1 h1:KS5FH+Q4i6K6e/QuW3WSsFFT732QZbhGZhEl6kx5YRo= -github.com/AdguardTeam/dnsproxy v0.18.1/go.mod h1:ZRbvuqDXag6IPsJWJp5UK+RGjHE/wUDBHYDNY8/EftI= +github.com/AdguardTeam/dnsproxy v0.18.2 h1:0wgZ/OZ8slD8XXT5/rFjM1FK86Wek7KolQLmhgf2ayQ= +github.com/AdguardTeam/dnsproxy v0.18.2/go.mod h1:R0YfjEUyGf7rsQ31gbw0GahraUjxCmH4dUqPEYVy81k= github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg= github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= github.com/AdguardTeam/golibs v0.1.4 h1:zZirpC23mZkf9Upasn9nbQ+lqqjcXJnkWdXO80vTu7s= @@ -8,19 +8,12 @@ github.com/AdguardTeam/urlfilter v0.5.0 h1:ATzs2Er0BMt7NbZnFJ4UEzt3uIV+rydbQCYqB github.com/AdguardTeam/urlfilter v0.5.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.1 h1:qHgHjyoNFV7jgucU8QZUuU4gcdhfs8QW1kw68OD2Lag= -github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/ameshkov/dnscrypt v1.0.7 h1:7LS9wiC/6c00H3ZdZOlwQSYGTJvs12g5ui9D1VSZ2aQ= github.com/ameshkov/dnscrypt v1.0.7/go.mod h1:rA74ASZ0j4JqPWaiN64hN97QXJ/zu5Kb2xgn295VzWQ= github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug= @@ -30,12 +23,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzs github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= -github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY= -github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 h1:yCfXxYaelOyqnia8F/Yng47qhmfC9nKTRIbYRrRueq4= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -49,8 +38,6 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI= github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= @@ -80,9 +67,6 @@ github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAri github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0/go.mod h1:eMyUVp6f/5jnzM+3zahzl7q6UXLbgSc3MKg/+ow9QW0= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= From d46ebe1c8bccfb398ccf2876047ad44416906ede Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Fri, 16 Aug 2019 15:11:31 +0300 Subject: [PATCH 3/4] Revert "+ config: add cache size settings" This reverts commit 81303b5db74705535b9810b7daddd196891079c5. --- dnsfilter/dnsfilter.go | 11 ++++------- home/config.go | 4 ---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index b1dcf04a..3ca639a3 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -25,6 +25,7 @@ import ( "golang.org/x/net/publicsuffix" ) +const defaultCacheSize = 64 * 1024 // in number of elements const defaultCacheTime = 30 * time.Minute const defaultHTTPTimeout = 5 * time.Minute @@ -67,10 +68,6 @@ type Config struct { SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` ResolverAddress string // DNS server address - SafeBrowsingCacheSize int `yaml:"safebrowsing_cache_size"` - SafeSearchCacheSize int `yaml:"safesearch_cache_size"` - ParentalCacheSize int `yaml:"parental_cache_size"` - Rewrites []RewriteEntry `yaml:"rewrites"` // Filtering callback function @@ -857,13 +854,13 @@ func New(c *Config, filters map[int]string) *Dnsfilter { if c != nil { // initialize objects only once if gctx.safebrowsingCache == nil { - gctx.safebrowsingCache = gcache.New(c.SafeBrowsingCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() } if gctx.safeSearchCache == nil { - gctx.safeSearchCache = gcache.New(c.SafeSearchCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() } if gctx.parentalCache == nil { - gctx.parentalCache = gcache.New(c.ParentalCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() } if len(c.ResolverAddress) != 0 && gctx.dialCache == nil { gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build() diff --git a/home/config.go b/home/config.go index 85d8498a..fd3460ea 100644 --- a/home/config.go +++ b/home/config.go @@ -211,10 +211,6 @@ func initConfig() { // also change the default config config.DNS.UpstreamDNS = defaultDNS } - - config.DNS.SafeBrowsingCacheSize = 64 * 1024 - config.DNS.SafeSearchCacheSize = 64 * 1024 - config.DNS.ParentalCacheSize = 64 * 1024 } // getConfigFilename returns path to the current config file From b37208564bd170f21347db9cfde39614ea4a02c2 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Fri, 16 Aug 2019 15:43:12 +0300 Subject: [PATCH 4/4] - fix build: we're using a new gcache module now --- dnsforward/querylog_top.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dnsforward/querylog_top.go b/dnsforward/querylog_top.go index 43d16699..3a81182d 100644 --- a/dnsforward/querylog_top.go +++ b/dnsforward/querylog_top.go @@ -282,9 +282,9 @@ func (d *dayTop) getStatsTop() *StatsTop { d.hoursReadLock() for hour := 0; hour < 24; hour++ { d.hours[hour].RLock() - do(d.hours[hour].domains.Keys(), d.hours[hour].lockedGetDomains, s.Domains) - do(d.hours[hour].blocked.Keys(), d.hours[hour].lockedGetBlocked, s.Blocked) - do(d.hours[hour].clients.Keys(), d.hours[hour].lockedGetClients, s.Clients) + do(d.hours[hour].domains.Keys(false), d.hours[hour].lockedGetDomains, s.Domains) + do(d.hours[hour].blocked.Keys(false), d.hours[hour].lockedGetBlocked, s.Blocked) + do(d.hours[hour].clients.Keys(false), d.hours[hour].lockedGetClients, s.Clients) d.hours[hour].RUnlock() } d.hoursReadUnlock()