* dnsfilter: use golibs/cache
+ config: add cache size settings + config: add cache_time setting
This commit is contained in:
Simon Zolin
parent
7d3fe71597
commit
c616259e8b
|
|
@ -5,6 +5,8 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
|
@ -18,6 +20,7 @@ import (
|
|||
"github.com/joomcode/errorx"
|
||||
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/urlfilter"
|
||||
"github.com/bluele/gcache"
|
||||
|
|
@ -25,9 +28,6 @@ import (
|
|||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
const defaultCacheSize = 64 * 1024 // in number of elements
|
||||
const defaultCacheTime = 30 * time.Minute
|
||||
|
||||
const defaultHTTPTimeout = 5 * time.Minute
|
||||
const defaultHTTPMaxIdleConnections = 100
|
||||
|
||||
|
|
@ -68,6 +68,11 @@ type Config struct {
|
|||
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
|
||||
ResolverAddress string // DNS server address
|
||||
|
||||
SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes)
|
||||
SafeSearchCacheSize uint `yaml:"safesearch_cache_size"` // (in bytes)
|
||||
ParentalCacheSize uint `yaml:"parental_cache_size"` // (in bytes)
|
||||
CacheTime uint `yaml:"cache_time"` // Element's TTL (in minutes)
|
||||
|
||||
Rewrites []RewriteEntry `yaml:"rewrites"`
|
||||
|
||||
// Filtering callback function
|
||||
|
|
@ -172,9 +177,9 @@ func (r Reason) String() string {
|
|||
type dnsFilterContext struct {
|
||||
stats Stats
|
||||
dialCache gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers
|
||||
safebrowsingCache gcache.Cache
|
||||
parentalCache gcache.Cache
|
||||
safeSearchCache gcache.Cache
|
||||
safebrowsingCache cache.Cache
|
||||
parentalCache cache.Cache
|
||||
safeSearchCache cache.Cache
|
||||
}
|
||||
|
||||
var gctx dnsFilterContext // global dnsfilter context
|
||||
|
|
@ -352,39 +357,52 @@ func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result {
|
|||
return res
|
||||
}
|
||||
|
||||
func setCacheResult(cache gcache.Cache, host string, res Result) {
|
||||
err := cache.Set(host, res)
|
||||
/*
|
||||
expire byte[4]
|
||||
res Result
|
||||
*/
|
||||
func (d *Dnsfilter) setCacheResult(cache cache.Cache, host string, res Result) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
expire := uint(time.Now().Unix()) + d.Config.CacheTime*60
|
||||
var exp []byte
|
||||
exp = make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(exp, uint32(expire))
|
||||
_, _ = buf.Write(exp)
|
||||
|
||||
enc := gob.NewEncoder(&buf)
|
||||
err := enc.Encode(res)
|
||||
if err != nil {
|
||||
log.Debug("cache.Set: %s", err)
|
||||
log.Error("gob.Encode(): %s", err)
|
||||
return
|
||||
}
|
||||
_ = cache.Set([]byte(host), buf.Bytes())
|
||||
log.Debug("Stored in cache %p: %s", cache, host)
|
||||
}
|
||||
|
||||
func getCachedResult(cache gcache.Cache, host string) (result Result, isFound bool) {
|
||||
isFound = false // not found yet
|
||||
|
||||
// get raw value
|
||||
rawValue, err := cache.Get(host)
|
||||
if err == gcache.KeyNotFoundError {
|
||||
// not a real error, just not found
|
||||
func getCachedResult(cache cache.Cache, host string) (Result, bool) {
|
||||
data := cache.Get([]byte(host))
|
||||
if data == nil {
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
exp := int(binary.BigEndian.Uint32(data[:4]))
|
||||
if exp <= int(time.Now().Unix()) {
|
||||
cache.Del([]byte(host))
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(data[4:])
|
||||
dec := gob.NewDecoder(&buf)
|
||||
r := Result{}
|
||||
err := dec.Decode(&r)
|
||||
if err != nil {
|
||||
// real error
|
||||
log.Debug("gob.Decode(): %s", err)
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
// since it can be something else, validate that it belongs to proper type
|
||||
cachedValue, ok := rawValue.(Result)
|
||||
if !ok {
|
||||
// this is not our type -- error
|
||||
text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache"
|
||||
log.Println(text)
|
||||
return
|
||||
}
|
||||
isFound = ok
|
||||
return cachedValue, isFound
|
||||
return r, true
|
||||
}
|
||||
|
||||
// for each dot, hash it and add it to string
|
||||
|
|
@ -445,7 +463,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
|
|||
res := Result{IsFiltered: true, Reason: FilteredSafeSearch}
|
||||
if ip := net.ParseIP(safeHost); ip != nil {
|
||||
res.IP = ip
|
||||
setCacheResult(gctx.safeSearchCache, host, res)
|
||||
d.setCacheResult(gctx.safeSearchCache, host, res)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
|
|
@ -468,7 +486,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
|
|||
}
|
||||
|
||||
// Cache result
|
||||
setCacheResult(gctx.safeSearchCache, host, res)
|
||||
d.setCacheResult(gctx.safeSearchCache, host, res)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
|
|
@ -523,7 +541,7 @@ func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) {
|
|||
result, err := d.lookupCommon(host, &gctx.stats.Safebrowsing, true, format, handleBody)
|
||||
|
||||
if err == nil {
|
||||
setCacheResult(gctx.safebrowsingCache, host, result)
|
||||
d.setCacheResult(gctx.safebrowsingCache, host, result)
|
||||
}
|
||||
|
||||
return result, err
|
||||
|
|
@ -589,7 +607,7 @@ func (d *Dnsfilter) checkParental(host string) (Result, error) {
|
|||
result, err := d.lookupCommon(host, &gctx.stats.Parental, false, format, handleBody)
|
||||
|
||||
if err == nil {
|
||||
setCacheResult(gctx.parentalCache, host, result)
|
||||
d.setCacheResult(gctx.parentalCache, host, result)
|
||||
}
|
||||
|
||||
return result, err
|
||||
|
|
@ -852,18 +870,30 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp
|
|||
func New(c *Config, filters map[int]string) *Dnsfilter {
|
||||
|
||||
if c != nil {
|
||||
cacheConf := cache.Config{
|
||||
EnableLRU: true,
|
||||
}
|
||||
|
||||
// initialize objects only once
|
||||
|
||||
if gctx.safebrowsingCache == nil {
|
||||
gctx.safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
|
||||
cacheConf.MaxSize = c.SafeBrowsingCacheSize
|
||||
gctx.safebrowsingCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if gctx.safeSearchCache == nil {
|
||||
gctx.safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
|
||||
cacheConf.MaxSize = c.SafeSearchCacheSize
|
||||
gctx.safeSearchCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if gctx.parentalCache == nil {
|
||||
gctx.parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
|
||||
cacheConf.MaxSize = c.ParentalCacheSize
|
||||
gctx.parentalCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if len(c.ResolverAddress) != 0 && gctx.dialCache == nil {
|
||||
gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build()
|
||||
dur := time.Duration(c.CacheTime) * time.Minute
|
||||
gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(dur).Build()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,13 +30,13 @@ var setts RequestFilteringSettings
|
|||
|
||||
func purgeCaches() {
|
||||
if gctx.safebrowsingCache != nil {
|
||||
gctx.safebrowsingCache.Purge()
|
||||
gctx.safebrowsingCache.Clear()
|
||||
}
|
||||
if gctx.parentalCache != nil {
|
||||
gctx.parentalCache.Purge()
|
||||
gctx.parentalCache.Clear()
|
||||
}
|
||||
if gctx.safeSearchCache != nil {
|
||||
gctx.safeSearchCache.Purge()
|
||||
gctx.safeSearchCache.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -51,6 +51,10 @@ func NewForTest(c *Config, filters map[int]string) *Dnsfilter {
|
|||
setts = RequestFilteringSettings{}
|
||||
setts.FilteringEnabled = true
|
||||
if c != nil {
|
||||
c.SafeBrowsingCacheSize = 1000
|
||||
c.SafeSearchCacheSize = 1000
|
||||
c.ParentalCacheSize = 1000
|
||||
c.CacheTime = 30
|
||||
setts.SafeSearchEnabled = c.SafeSearchEnabled
|
||||
setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled
|
||||
setts.ParentalEnabled = c.ParentalEnabled
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ type FilteringConfig struct {
|
|||
// Per-client settings can override this configuration.
|
||||
BlockedServices []string `yaml:"blocked_services"`
|
||||
|
||||
CacheSize uint `yaml:"cache_size"` // DNS cache size (in bytes)
|
||||
dnsfilter.Config `yaml:",inline"`
|
||||
}
|
||||
|
||||
|
|
@ -203,6 +204,7 @@ func (s *Server) startInternal(config *ServerConfig) error {
|
|||
RatelimitWhitelist: s.conf.RatelimitWhitelist,
|
||||
RefuseAny: s.conf.RefuseAny,
|
||||
CacheEnabled: true,
|
||||
CacheSizeBytes: int(s.conf.CacheSize),
|
||||
Upstreams: s.conf.Upstreams,
|
||||
DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams,
|
||||
BeforeRequestHandler: s.beforeRequestHandler,
|
||||
|
|
|
|||
|
|
@ -491,6 +491,11 @@ func createTestServer(t *testing.T) *Server {
|
|||
s.conf.FilteringConfig.SafeBrowsingEnabled = true
|
||||
s.conf.Filters = make([]dnsfilter.Filter, 0)
|
||||
|
||||
s.conf.SafeBrowsingCacheSize = 1000
|
||||
s.conf.SafeSearchCacheSize = 1000
|
||||
s.conf.ParentalCacheSize = 1000
|
||||
s.conf.CacheTime = 30
|
||||
|
||||
rules := "||nxdomain.example.org^\n||null.example.org^\n127.0.0.1 host.example.org\n"
|
||||
filter := dnsfilter.Filter{ID: 0, Data: []byte(rules)}
|
||||
s.conf.Filters = append(s.conf.Filters, filter)
|
||||
|
|
|
|||
|
|
@ -211,6 +211,12 @@ func initConfig() {
|
|||
// also change the default config
|
||||
config.DNS.UpstreamDNS = defaultDNS
|
||||
}
|
||||
|
||||
config.DNS.CacheSize = 4 * 1024 * 1024
|
||||
config.DNS.SafeBrowsingCacheSize = 1 * 1024 * 1024
|
||||
config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024
|
||||
config.DNS.ParentalCacheSize = 1 * 1024 * 1024
|
||||
config.DNS.CacheTime = 30
|
||||
}
|
||||
|
||||
// getConfigFilename returns path to the current config file
|
||||
|
|
|
|||
Loading…
Reference in New Issue