diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 440170a4..6b168a9c 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/binary" + "encoding/gob" "encoding/json" "fmt" "io/ioutil" @@ -18,6 +20,7 @@ import ( "github.com/joomcode/errorx" "github.com/AdguardTeam/dnsproxy/upstream" + "github.com/AdguardTeam/golibs/cache" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/urlfilter" "github.com/bluele/gcache" @@ -25,9 +28,6 @@ import ( "golang.org/x/net/publicsuffix" ) -const defaultCacheSize = 64 * 1024 // in number of elements -const defaultCacheTime = 30 * time.Minute - const defaultHTTPTimeout = 5 * time.Minute const defaultHTTPMaxIdleConnections = 100 @@ -68,6 +68,11 @@ type Config struct { SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` ResolverAddress string // DNS server address + SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes) + SafeSearchCacheSize uint `yaml:"safesearch_cache_size"` // (in bytes) + ParentalCacheSize uint `yaml:"parental_cache_size"` // (in bytes) + CacheTime uint `yaml:"cache_time"` // Element's TTL (in minutes) + Rewrites []RewriteEntry `yaml:"rewrites"` // Filtering callback function @@ -172,9 +177,9 @@ func (r Reason) String() string { type dnsFilterContext struct { stats Stats dialCache gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers - safebrowsingCache gcache.Cache - parentalCache gcache.Cache - safeSearchCache gcache.Cache + safebrowsingCache cache.Cache + parentalCache cache.Cache + safeSearchCache cache.Cache } var gctx dnsFilterContext // global dnsfilter context @@ -352,39 +357,52 @@ func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result { return res } -func setCacheResult(cache gcache.Cache, host string, res Result) { - err := cache.Set(host, res) +/* +expire byte[4] +res Result +*/ +func (d *Dnsfilter) setCacheResult(cache cache.Cache, host string, res Result) { + var buf bytes.Buffer + + expire := uint(time.Now().Unix()) + d.Config.CacheTime*60 + var exp []byte + exp = make([]byte, 4) + binary.BigEndian.PutUint32(exp, uint32(expire)) + _, _ = buf.Write(exp) + + enc := gob.NewEncoder(&buf) + err := enc.Encode(res) if err != nil { - log.Debug("cache.Set: %s", err) + log.Error("gob.Encode(): %s", err) return } + _ = cache.Set([]byte(host), buf.Bytes()) log.Debug("Stored in cache %p: %s", cache, host) } -func getCachedResult(cache gcache.Cache, host string) (result Result, isFound bool) { - isFound = false // not found yet - - // get raw value - rawValue, err := cache.Get(host) - if err == gcache.KeyNotFoundError { - // not a real error, just not found +func getCachedResult(cache cache.Cache, host string) (Result, bool) { + data := cache.Get([]byte(host)) + if data == nil { return Result{}, false } + + exp := int(binary.BigEndian.Uint32(data[:4])) + if exp <= int(time.Now().Unix()) { + cache.Del([]byte(host)) + return Result{}, false + } + + var buf bytes.Buffer + buf.Write(data[4:]) + dec := gob.NewDecoder(&buf) + r := Result{} + err := dec.Decode(&r) if err != nil { - // real error + log.Debug("gob.Decode(): %s", err) return Result{}, false } - // since it can be something else, validate that it belongs to proper type - cachedValue, ok := rawValue.(Result) - if !ok { - // this is not our type -- error - text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache" - log.Println(text) - return - } - isFound = ok - return cachedValue, isFound + return r, true } // for each dot, hash it and add it to string @@ -445,7 +463,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { res := Result{IsFiltered: true, Reason: FilteredSafeSearch} if ip := net.ParseIP(safeHost); ip != nil { res.IP = ip - setCacheResult(gctx.safeSearchCache, host, res) + d.setCacheResult(gctx.safeSearchCache, host, res) return res, nil } @@ -468,7 +486,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) { } // Cache result - setCacheResult(gctx.safeSearchCache, host, res) + d.setCacheResult(gctx.safeSearchCache, host, res) return res, nil } @@ -523,7 +541,7 @@ func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) { result, err := d.lookupCommon(host, &gctx.stats.Safebrowsing, true, format, handleBody) if err == nil { - setCacheResult(gctx.safebrowsingCache, host, result) + d.setCacheResult(gctx.safebrowsingCache, host, result) } return result, err @@ -589,7 +607,7 @@ func (d *Dnsfilter) checkParental(host string) (Result, error) { result, err := d.lookupCommon(host, &gctx.stats.Parental, false, format, handleBody) if err == nil { - setCacheResult(gctx.parentalCache, host, result) + d.setCacheResult(gctx.parentalCache, host, result) } return result, err @@ -852,18 +870,30 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp func New(c *Config, filters map[int]string) *Dnsfilter { if c != nil { + cacheConf := cache.Config{ + EnableLRU: true, + } + // initialize objects only once + if gctx.safebrowsingCache == nil { - gctx.safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() + cacheConf.MaxSize = c.SafeBrowsingCacheSize + gctx.safebrowsingCache = cache.New(cacheConf) } + if gctx.safeSearchCache == nil { - gctx.safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() + cacheConf.MaxSize = c.SafeSearchCacheSize + gctx.safeSearchCache = cache.New(cacheConf) } + if gctx.parentalCache == nil { - gctx.parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() + cacheConf.MaxSize = c.ParentalCacheSize + gctx.parentalCache = cache.New(cacheConf) } + if len(c.ResolverAddress) != 0 && gctx.dialCache == nil { - gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build() + dur := time.Duration(c.CacheTime) * time.Minute + gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(dur).Build() } } diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go index 06658afd..4c574b57 100644 --- a/dnsfilter/dnsfilter_test.go +++ b/dnsfilter/dnsfilter_test.go @@ -30,13 +30,13 @@ var setts RequestFilteringSettings func purgeCaches() { if gctx.safebrowsingCache != nil { - gctx.safebrowsingCache.Purge() + gctx.safebrowsingCache.Clear() } if gctx.parentalCache != nil { - gctx.parentalCache.Purge() + gctx.parentalCache.Clear() } if gctx.safeSearchCache != nil { - gctx.safeSearchCache.Purge() + gctx.safeSearchCache.Clear() } } @@ -51,6 +51,10 @@ func NewForTest(c *Config, filters map[int]string) *Dnsfilter { setts = RequestFilteringSettings{} setts.FilteringEnabled = true if c != nil { + c.SafeBrowsingCacheSize = 1000 + c.SafeSearchCacheSize = 1000 + c.ParentalCacheSize = 1000 + c.CacheTime = 30 setts.SafeSearchEnabled = c.SafeSearchEnabled setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled setts.ParentalEnabled = c.ParentalEnabled diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index 845c8ca6..2eec20a4 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -100,6 +100,7 @@ type FilteringConfig struct { // Per-client settings can override this configuration. BlockedServices []string `yaml:"blocked_services"` + CacheSize uint `yaml:"cache_size"` // DNS cache size (in bytes) dnsfilter.Config `yaml:",inline"` } @@ -203,6 +204,7 @@ func (s *Server) startInternal(config *ServerConfig) error { RatelimitWhitelist: s.conf.RatelimitWhitelist, RefuseAny: s.conf.RefuseAny, CacheEnabled: true, + CacheSizeBytes: int(s.conf.CacheSize), Upstreams: s.conf.Upstreams, DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams, BeforeRequestHandler: s.beforeRequestHandler, diff --git a/dnsforward/dnsforward_test.go b/dnsforward/dnsforward_test.go index cd16f714..6d41c62a 100644 --- a/dnsforward/dnsforward_test.go +++ b/dnsforward/dnsforward_test.go @@ -491,6 +491,11 @@ func createTestServer(t *testing.T) *Server { s.conf.FilteringConfig.SafeBrowsingEnabled = true s.conf.Filters = make([]dnsfilter.Filter, 0) + s.conf.SafeBrowsingCacheSize = 1000 + s.conf.SafeSearchCacheSize = 1000 + s.conf.ParentalCacheSize = 1000 + s.conf.CacheTime = 30 + rules := "||nxdomain.example.org^\n||null.example.org^\n127.0.0.1 host.example.org\n" filter := dnsfilter.Filter{ID: 0, Data: []byte(rules)} s.conf.Filters = append(s.conf.Filters, filter) diff --git a/go.mod b/go.mod index 9a33c497..783486bd 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/AdguardTeam/AdGuardHome go 1.12 require ( - github.com/AdguardTeam/dnsproxy v0.18.4 - github.com/AdguardTeam/golibs v0.1.4 + github.com/AdguardTeam/dnsproxy v0.19.1 + github.com/AdguardTeam/golibs v0.2.1 github.com/AdguardTeam/urlfilter v0.5.0 github.com/NYTimes/gziphandler v1.1.1 github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 @@ -16,9 +16,9 @@ require ( github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 github.com/miekg/dns v1.1.8 github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 - gopkg.in/yaml.v2 v2.2.1 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 3eee3c5c..433dac2d 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -github.com/AdguardTeam/dnsproxy v0.18.4 h1:UjqN83y7uTduPxSJ5Pg7aO92Fgs4MfDDIfjqaK2YujQ= -github.com/AdguardTeam/dnsproxy v0.18.4/go.mod h1:R0YfjEUyGf7rsQ31gbw0GahraUjxCmH4dUqPEYVy81k= +github.com/AdguardTeam/dnsproxy v0.19.1 h1:Xyv5qv9UCj2IVB++2YN/DxdwFBdt3lHjD0Mp/bEJS74= +github.com/AdguardTeam/dnsproxy v0.19.1/go.mod h1:NaulY9i279jZwN8QBbvbZnn5HkrjBgJi4hbFY5nW+Kc= github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg= github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= -github.com/AdguardTeam/golibs v0.1.4 h1:zZirpC23mZkf9Upasn9nbQ+lqqjcXJnkWdXO80vTu7s= -github.com/AdguardTeam/golibs v0.1.4/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= +github.com/AdguardTeam/golibs v0.2.1 h1:jGCnbM5UOUq/GrG+8eLN7Y+OTfEo5F/8L0wq3ur2h4E= +github.com/AdguardTeam/golibs v0.2.1/go.mod h1:caAJ5knSHbR6vV6qfRDgAfXVia4hHgLqeztAY4UX0fw= github.com/AdguardTeam/urlfilter v0.5.0 h1:ATzs2Er0BMt7NbZnFJ4UEzt3uIV+rydbQCYqBXNRbJc= github.com/AdguardTeam/urlfilter v0.5.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= @@ -78,6 +78,8 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -114,5 +116,4 @@ gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+l gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477/go.mod h1:QDV1vrFSrowdoOba0UM8VJPUZONT7dnfdLsM+GG53Z8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/home/config.go b/home/config.go index 1c558e16..b4c943dc 100644 --- a/home/config.go +++ b/home/config.go @@ -211,6 +211,12 @@ func initConfig() { // also change the default config config.DNS.UpstreamDNS = defaultDNS } + + config.DNS.CacheSize = 4 * 1024 * 1024 + config.DNS.SafeBrowsingCacheSize = 1 * 1024 * 1024 + config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024 + config.DNS.ParentalCacheSize = 1 * 1024 * 1024 + config.DNS.CacheTime = 30 } // getConfigFilename returns path to the current config file