From 6f51df7d2e3eb4672d4762e14e22e831fdadd256 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Tue, 23 Jul 2019 17:14:13 +0300 Subject: [PATCH] * dnsfilter: use fastcache instead of gcache --- dnsfilter/dnsfilter.go | 92 +++++++++++++++++------------------------- go.mod | 1 + go.sum | 16 ++++++++ 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index b9b8c7d2..bc03e448 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -5,8 +5,8 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/gob" "encoding/json" - "errors" "fmt" "io/ioutil" "net" @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/fastcache" "github.com/joomcode/errorx" "github.com/AdguardTeam/dnsproxy/upstream" @@ -135,9 +136,9 @@ const ( type dnsFilterContext struct { stats Stats dialCache gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers - safebrowsingCache gcache.Cache - parentalCache gcache.Cache - safeSearchCache gcache.Cache + safebrowsingCache *fastcache.Cache + parentalCache *fastcache.Cache + safeSearchCache *fastcache.Cache } var gctx dnsFilterContext // global dnsfilter context @@ -233,32 +234,33 @@ func (d *Dnsfilter) CheckHost(host string, qtype uint16, clientAddr string) (Res return Result{}, nil } -func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) { +func setCacheResult(cache *fastcache.Cache, host string, res Result) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + _ = enc.Encode(res) + cache.Set([]byte(host), buf.Bytes()) + log.Debug("Stored in cache %p: %s => [%d]", cache, host, buf.Len()) +} + +func getCachedResult(cache *fastcache.Cache, host string) (result Result, isFound bool) { isFound = false // not found yet // get raw value - rawValue, err := cache.Get(host) - if err == gcache.KeyNotFoundError { - // not a real error, just not found - err = nil - return - } - if err != nil { - // real error - return + rawValue := cache.Get(nil, []byte(host)) + if len(rawValue) == 0 { + return Result{}, false } - // since it can be something else, validate that it belongs to proper type - cachedValue, ok := rawValue.(Result) - if !ok { - // this is not our type -- error - text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache" - log.Println(text) - err = errors.New(text) - return + var buf bytes.Buffer + buf.Write(rawValue) + dec := gob.NewDecoder(&buf) + cachedValue := Result{} + err := dec.Decode(&cachedValue) + if err != nil { + log.Debug("gob.Decode(): %s", err) + return Result{}, false } - isFound = ok - return cachedValue, isFound, err + return cachedValue, true } // for each dot, hash it and add it to string @@ -304,17 +306,13 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { } // Check cache. Return cached result if it was found - cachedValue, isFound, err := getCachedReason(gctx.safeSearchCache, host) + cachedValue, isFound := getCachedResult(gctx.safeSearchCache, host) if isFound { atomic.AddUint64(&gctx.stats.Safesearch.CacheHits, 1) log.Tracef("%s: found in SafeSearch cache", host) return cachedValue, nil } - if err != nil { - return Result{}, err - } - safeHost, ok := d.SafeSearchDomain(host) if !ok { return Result{}, nil @@ -323,11 +321,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { res := Result{IsFiltered: true, Reason: FilteredSafeSearch} if ip := net.ParseIP(safeHost); ip != nil { res.IP = ip - err = gctx.safeSearchCache.Set(host, res) - if err != nil { - return Result{}, nil - } - + setCacheResult(gctx.safeSearchCache, host, res) return res, nil } @@ -350,10 +344,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { } // Cache result - err = gctx.safeSearchCache.Set(host, res) - if err != nil { - return Result{}, nil - } + setCacheResult(gctx.safeSearchCache, host, res) return res, nil } @@ -456,20 +447,17 @@ type formatHandler func(hashparam string) string type bodyHandler func(body []byte, hashes map[string]bool) (Result, error) // real implementation of lookup/check -func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gcache.Cache, hashparamNeedSlash bool, format formatHandler, handleBody bodyHandler) (Result, error) { +func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache *fastcache.Cache, hashparamNeedSlash bool, format formatHandler, handleBody bodyHandler) (Result, error) { // if host ends with a dot, trim it host = strings.ToLower(strings.Trim(host, ".")) // check cache - cachedValue, isFound, err := getCachedReason(cache, host) + cachedValue, isFound := getCachedResult(cache, host) if isFound { atomic.AddUint64(&lookupstats.CacheHits, 1) - log.Tracef("%s: found in the lookup cache", host) + log.Tracef("%s: found in the lookup cache %p", host, cache) return cachedValue, nil } - if err != nil { - return Result{}, err - } // convert hostname to hash parameters hashparam, hashes := hostnameToHashParam(host, hashparamNeedSlash) @@ -502,10 +490,7 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc switch { case resp.StatusCode == 204: // empty result, save cache - err = cache.Set(host, Result{}) - if err != nil { - return Result{}, err - } + setCacheResult(cache, host, Result{}) return Result{}, nil case resp.StatusCode != 200: // error, don't save cache @@ -518,10 +503,7 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc return Result{}, err } - err = cache.Set(host, result) - if err != nil { - return Result{}, err - } + setCacheResult(cache, host, result) return result, nil } @@ -735,13 +717,13 @@ func New(c *Config, filters map[int]string) *Dnsfilter { if c != nil { // initialize objects only once if gctx.safebrowsingCache == nil { - gctx.safebrowsingCache = gcache.New(c.SafeBrowsingCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.safebrowsingCache = fastcache.New(c.SafeBrowsingCacheSize) } if gctx.safeSearchCache == nil { - gctx.safeSearchCache = gcache.New(c.SafeSearchCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.safeSearchCache = fastcache.New(c.SafeSearchCacheSize) } if gctx.parentalCache == nil { - gctx.parentalCache = gcache.New(c.ParentalCacheSize).LRU().Expiration(defaultCacheTime).Build() + gctx.parentalCache = fastcache.New(c.ParentalCacheSize) } if len(c.ResolverAddress) != 0 && gctx.dialCache == nil { gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build() diff --git a/go.mod b/go.mod index 6db14304..13134335 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/AdguardTeam/golibs v0.1.3 github.com/AdguardTeam/urlfilter v0.4.1 github.com/NYTimes/gziphandler v1.1.1 + github.com/VictoriaMetrics/fastcache v1.5.1 github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 github.com/go-test/deep v1.0.1 github.com/gobuffalo/packr v1.19.0 diff --git a/go.sum b/go.sum index 24e53046..21484254 100644 --- a/go.sum +++ b/go.sum @@ -9,12 +9,19 @@ github.com/AdguardTeam/urlfilter v0.4.1 h1:fuQLSEAk8A0zKroLtfvUgWvVnyBmOwK9uAjtI github.com/AdguardTeam/urlfilter v0.4.1/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VictoriaMetrics/fastcache v1.5.1 h1:qHgHjyoNFV7jgucU8QZUuU4gcdhfs8QW1kw68OD2Lag= +github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/ameshkov/dnscrypt v1.0.7 h1:7LS9wiC/6c00H3ZdZOlwQSYGTJvs12g5ui9D1VSZ2aQ= github.com/ameshkov/dnscrypt v1.0.7/go.mod h1:rA74ASZ0j4JqPWaiN64hN97QXJ/zu5Kb2xgn295VzWQ= github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug= @@ -26,6 +33,10 @@ github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttP github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY= github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0= +github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -39,6 +50,8 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI= github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU= @@ -72,6 +85,9 @@ github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAri github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0/go.mod h1:eMyUVp6f/5jnzM+3zahzl7q6UXLbgSc3MKg/+ow9QW0= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=