Merge: * dnsfilter: use fastcache instead of gcache

* commit 'd9265aa9a8c21d97206b0e43a03b448f2c33a510':
  fix tests
  * dnsfilter: use fastcache instead of gcache
  + config: add cache size settings
This commit is contained in:
Andrey Meshkov 2019-07-24 17:04:24 +03:00
commit e09ec5d38e
6 changed files with 76 additions and 69 deletions

View File

@ -5,8 +5,8 @@ import (
"bytes"
"context"
"crypto/sha256"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
@ -16,6 +16,7 @@ import (
"sync/atomic"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/joomcode/errorx"
"github.com/AdguardTeam/dnsproxy/upstream"
@ -26,7 +27,6 @@ import (
"golang.org/x/net/publicsuffix"
)
const defaultCacheSize = 64 * 1024 // in number of elements
const defaultCacheTime = 30 * time.Minute
const defaultHTTPTimeout = 5 * time.Minute
@ -56,6 +56,10 @@ type Config struct {
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
ResolverAddress string // DNS server address
SafeBrowsingCacheSize int `yaml:"safebrowsing_cache_size"`
SafeSearchCacheSize int `yaml:"safesearch_cache_size"`
ParentalCacheSize int `yaml:"parental_cache_size"`
// Filtering callback function
FilterHandler func(clientAddr string, settings *RequestFilteringSettings) `yaml:"-"`
}
@ -132,9 +136,9 @@ const (
type dnsFilterContext struct {
stats Stats
dialCache gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers
safebrowsingCache gcache.Cache
parentalCache gcache.Cache
safeSearchCache gcache.Cache
safebrowsingCache *fastcache.Cache
parentalCache *fastcache.Cache
safeSearchCache *fastcache.Cache
}
var gctx dnsFilterContext // global dnsfilter context
@ -230,32 +234,33 @@ func (d *Dnsfilter) CheckHost(host string, qtype uint16, clientAddr string) (Res
return Result{}, nil
}
func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) {
func setCacheResult(cache *fastcache.Cache, host string, res Result) {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
_ = enc.Encode(res)
cache.Set([]byte(host), buf.Bytes())
log.Debug("Stored in cache %p: %s => [%d]", cache, host, buf.Len())
}
func getCachedResult(cache *fastcache.Cache, host string) (result Result, isFound bool) {
isFound = false // not found yet
// get raw value
rawValue, err := cache.Get(host)
if err == gcache.KeyNotFoundError {
// not a real error, just not found
err = nil
return
}
if err != nil {
// real error
return
rawValue := cache.Get(nil, []byte(host))
if len(rawValue) == 0 {
return Result{}, false
}
// since it can be something else, validate that it belongs to proper type
cachedValue, ok := rawValue.(Result)
if !ok {
// this is not our type -- error
text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache"
log.Println(text)
err = errors.New(text)
return
var buf bytes.Buffer
buf.Write(rawValue)
dec := gob.NewDecoder(&buf)
cachedValue := Result{}
err := dec.Decode(&cachedValue)
if err != nil {
log.Debug("gob.Decode(): %s", err)
return Result{}, false
}
isFound = ok
return cachedValue, isFound, err
return cachedValue, true
}
// for each dot, hash it and add it to string
@ -301,17 +306,13 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
}
// Check cache. Return cached result if it was found
cachedValue, isFound, err := getCachedReason(gctx.safeSearchCache, host)
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, host)
if isFound {
atomic.AddUint64(&gctx.stats.Safesearch.CacheHits, 1)
log.Tracef("%s: found in SafeSearch cache", host)
return cachedValue, nil
}
if err != nil {
return Result{}, err
}
safeHost, ok := d.SafeSearchDomain(host)
if !ok {
return Result{}, nil
@ -320,11 +321,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
res := Result{IsFiltered: true, Reason: FilteredSafeSearch}
if ip := net.ParseIP(safeHost); ip != nil {
res.IP = ip
err = gctx.safeSearchCache.Set(host, res)
if err != nil {
return Result{}, nil
}
setCacheResult(gctx.safeSearchCache, host, res)
return res, nil
}
@ -347,10 +344,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
}
// Cache result
err = gctx.safeSearchCache.Set(host, res)
if err != nil {
return Result{}, nil
}
setCacheResult(gctx.safeSearchCache, host, res)
return res, nil
}
@ -453,20 +447,17 @@ type formatHandler func(hashparam string) string
type bodyHandler func(body []byte, hashes map[string]bool) (Result, error)
// real implementation of lookup/check
func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gcache.Cache, hashparamNeedSlash bool, format formatHandler, handleBody bodyHandler) (Result, error) {
func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache *fastcache.Cache, hashparamNeedSlash bool, format formatHandler, handleBody bodyHandler) (Result, error) {
// if host ends with a dot, trim it
host = strings.ToLower(strings.Trim(host, "."))
// check cache
cachedValue, isFound, err := getCachedReason(cache, host)
cachedValue, isFound := getCachedResult(cache, host)
if isFound {
atomic.AddUint64(&lookupstats.CacheHits, 1)
log.Tracef("%s: found in the lookup cache", host)
log.Tracef("%s: found in the lookup cache %p", host, cache)
return cachedValue, nil
}
if err != nil {
return Result{}, err
}
// convert hostname to hash parameters
hashparam, hashes := hostnameToHashParam(host, hashparamNeedSlash)
@ -499,10 +490,7 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc
switch {
case resp.StatusCode == 204:
// empty result, save cache
err = cache.Set(host, Result{})
if err != nil {
return Result{}, err
}
setCacheResult(cache, host, Result{})
return Result{}, nil
case resp.StatusCode != 200:
// error, don't save cache
@ -515,10 +503,7 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc
return Result{}, err
}
err = cache.Set(host, result)
if err != nil {
return Result{}, err
}
setCacheResult(cache, host, result)
return result, nil
}
@ -732,13 +717,13 @@ func New(c *Config, filters map[int]string) *Dnsfilter {
if c != nil {
// initialize objects only once
if gctx.safebrowsingCache == nil {
gctx.safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
gctx.safebrowsingCache = fastcache.New(c.SafeBrowsingCacheSize)
}
if gctx.safeSearchCache == nil {
gctx.safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
gctx.safeSearchCache = fastcache.New(c.SafeSearchCacheSize)
}
if gctx.parentalCache == nil {
gctx.parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
gctx.parentalCache = fastcache.New(c.ParentalCacheSize)
}
if len(c.ResolverAddress) != 0 && gctx.dialCache == nil {
gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build()

View File

@ -26,13 +26,13 @@ import (
func purgeCaches() {
if gctx.safebrowsingCache != nil {
gctx.safebrowsingCache.Purge()
gctx.safebrowsingCache.Reset()
}
if gctx.parentalCache != nil {
gctx.parentalCache.Purge()
gctx.parentalCache.Reset()
}
if gctx.safeSearchCache != nil {
gctx.safeSearchCache.Purge()
gctx.safeSearchCache.Reset()
}
}
@ -44,6 +44,11 @@ func _Func() string {
}
func NewForTest(c *Config, filters map[int]string) *Dnsfilter {
if c != nil {
c.SafeBrowsingCacheSize = 1024
c.SafeSearchCacheSize = 1024
c.ParentalCacheSize = 1024
}
d := New(c, filters)
purgeCaches()
return d
@ -269,11 +274,7 @@ func TestSafeSearchCacheYandex(t *testing.T) {
}
// Check cache
cachedValue, isFound, err := getCachedReason(gctx.safeSearchCache, domain)
if err != nil {
t.Fatalf("An error occured during cache search for %s: %s", domain, err)
}
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
if !isFound {
t.Fatalf("Safesearch cache doesn't work for %s!", domain)
@ -330,11 +331,7 @@ func TestSafeSearchCacheGoogle(t *testing.T) {
}
// Check cache
cachedValue, isFound, err := getCachedReason(gctx.safeSearchCache, domain)
if err != nil {
t.Fatalf("An error occured during cache search for %s: %s", domain, err)
}
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
if !isFound {
t.Fatalf("Safesearch cache doesn't work for %s!", domain)

View File

@ -458,6 +458,10 @@ func createTestServer(t *testing.T) *Server {
s.conf.FilteringConfig.SafeBrowsingEnabled = true
s.conf.Filters = make([]dnsfilter.Filter, 0)
s.conf.SafeBrowsingCacheSize = 1024
s.conf.SafeSearchCacheSize = 1024
s.conf.ParentalCacheSize = 1024
rules := "||nxdomain.example.org^\n||null.example.org^\n127.0.0.1 host.example.org\n"
filter := dnsfilter.Filter{ID: 0, Data: []byte(rules)}
s.conf.Filters = append(s.conf.Filters, filter)

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/AdguardTeam/golibs v0.1.3
github.com/AdguardTeam/urlfilter v0.4.1
github.com/NYTimes/gziphandler v1.1.1
github.com/VictoriaMetrics/fastcache v1.5.1
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0
github.com/go-test/deep v1.0.1
github.com/gobuffalo/packr v1.19.0

16
go.sum
View File

@ -9,12 +9,19 @@ github.com/AdguardTeam/urlfilter v0.4.1 h1:fuQLSEAk8A0zKroLtfvUgWvVnyBmOwK9uAjtI
github.com/AdguardTeam/urlfilter v0.4.1/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg=
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VictoriaMetrics/fastcache v1.5.1 h1:qHgHjyoNFV7jgucU8QZUuU4gcdhfs8QW1kw68OD2Lag=
github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw=
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/ameshkov/dnscrypt v1.0.7 h1:7LS9wiC/6c00H3ZdZOlwQSYGTJvs12g5ui9D1VSZ2aQ=
github.com/ameshkov/dnscrypt v1.0.7/go.mod h1:rA74ASZ0j4JqPWaiN64hN97QXJ/zu5Kb2xgn295VzWQ=
github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug=
@ -26,6 +33,10 @@ github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttP
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY=
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0=
github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -39,6 +50,8 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq
github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI=
github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI=
github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU=
@ -72,6 +85,9 @@ github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAri
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY=
github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM=
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0/go.mod h1:eMyUVp6f/5jnzM+3zahzl7q6UXLbgSc3MKg/+ow9QW0=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=

View File

@ -205,6 +205,10 @@ func initConfig() {
// also change the default config
config.DNS.UpstreamDNS = defaultDNS
}
config.DNS.SafeBrowsingCacheSize = 64 * 1024
config.DNS.SafeSearchCacheSize = 64 * 1024
config.DNS.ParentalCacheSize = 64 * 1024
}
// getConfigFilename returns path to the current config file