package dnsfilter import ( "bufio" "bytes" "crypto/sha256" "encoding/json" "errors" "fmt" "io/ioutil" "net" "net/http" "regexp" "strings" "sync" "sync/atomic" "time" "github.com/bluele/gcache" "github.com/hmage/golibs/log" "golang.org/x/net/publicsuffix" ) const defaultCacheSize = 64 * 1024 // in number of elements const defaultCacheTime = 30 * time.Minute const defaultHTTPTimeout = 5 * time.Minute const defaultHTTPMaxIdleConnections = 100 const defaultSafebrowsingServer = "sb.adtidy.org" const defaultSafebrowsingURL = "http://%s/safebrowsing-lookup-hash.html?prefixes=%s" const defaultParentalServer = "pctrl.adguard.com" const defaultParentalURL = "http://%s/check-parental-control-hash?prefixes=%s&sensitivity=%d" // ErrInvalidSyntax is returned by AddRule when the rule is invalid var ErrInvalidSyntax = errors.New("dnsfilter: invalid rule syntax") // ErrAlreadyExists is returned by AddRule when the rule was already added to the filter var ErrAlreadyExists = errors.New("dnsfilter: rule was already added") const shortcutLength = 6 // used for rule search optimization, 6 hits the sweet spot const enableFastLookup = true // flag for debugging, must be true in production for faster performance const enableDelayedCompilation = true // flag for debugging, must be true in production for faster performance // Config allows you to configure DNS filtering with New() or just change variables directly. type Config struct { ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17 ParentalEnabled bool `yaml:"parental_enabled"` SafeSearchEnabled bool `yaml:"safesearch_enabled"` SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` } type privateConfig struct { parentalServer string // access via methods safeBrowsingServer string // access via methods } type rule struct { text string // text without @@ decorators or $ options shortcut string // for speeding up lookup originalText string // original text for reporting back to applications ip net.IP // IP address (for the case when we're matching a hosts file) // options options []string // optional options after $ // parsed options apps []string isWhitelist bool isImportant bool // user-supplied data listID int64 // suffix matching isSuffix bool suffix string // compiled regexp compiled *regexp.Regexp sync.RWMutex } // LookupStats store stats collected during safebrowsing or parental checks type LookupStats struct { Requests uint64 // number of HTTP requests that were sent CacheHits uint64 // number of lookups that didn't need HTTP requests Pending int64 // number of currently pending HTTP requests PendingMax int64 // maximum number of pending HTTP requests } // Stats store LookupStats for both safebrowsing and parental type Stats struct { Safebrowsing LookupStats Parental LookupStats } // Dnsfilter holds added rules and performs hostname matches against the rules type Dnsfilter struct { storage map[string]bool // rule storage, not used for matching, just for filtering out duplicates storageMutex sync.RWMutex // rules are checked against these lists in the order defined here important *rulesTable // more important than whitelist and is checked first whiteList *rulesTable // more important than blacklist blackList *rulesTable // HTTP lookups for safebrowsing and parental client http.Client // handle for http client -- single instance as recommended by docs transport *http.Transport // handle for http transport used by http client Config // for direct access by library users, even a = assignment privateConfig } // Filter represents a filter list type Filter struct { ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase Rules []string `json:"-" yaml:"-"` // not in yaml or json } //go:generate stringer -type=Reason // Reason holds an enum detailing why it was filtered or not filtered type Reason int const ( // reasons for not filtering // NotFilteredNotFound - host was not find in any checks, default value for result NotFilteredNotFound Reason = iota // NotFilteredWhiteList - the host is explicitly whitelisted NotFilteredWhiteList // NotFilteredError - there was a transitive error during check NotFilteredError // reasons for filtering // FilteredBlackList - the host was matched to be advertising host FilteredBlackList // FilteredSafeBrowsing - the host was matched to be malicious/phishing FilteredSafeBrowsing // FilteredParental - the host was matched to be outside of parental control settings FilteredParental // FilteredInvalid - the request was invalid and was not processed FilteredInvalid // FilteredSafeSearch - the host was replaced with safesearch variant FilteredSafeSearch ) // these variables need to survive coredns reload var ( stats Stats safebrowsingCache gcache.Cache parentalCache gcache.Cache ) // Result holds state of hostname check type Result struct { IsFiltered bool `json:",omitempty"` // True if the host name is filtered Reason Reason `json:",omitempty"` // Reason for blocking / unblocking Rule string `json:",omitempty"` // Original rule text IP net.IP `json:",omitempty"` // Not nil only in the case of a hosts file syntax FilterID int64 `json:",omitempty"` // Filter ID the rule belongs to } // Matched can be used to see if any match at all was found, no matter filtered or not func (r Reason) Matched() bool { return r != NotFilteredNotFound } // CheckHost tries to match host against rules, then safebrowsing and parental if they are enabled func (d *Dnsfilter) CheckHost(host string) (Result, error) { // sometimes DNS clients will try to resolve ".", which is a request to get root servers if host == "" { return Result{Reason: NotFilteredNotFound}, nil } host = strings.ToLower(host) // try filter lists first result, err := d.matchHost(host) if err != nil { return result, err } if result.Reason.Matched() { return result, nil } // check safebrowsing if no match if d.SafeBrowsingEnabled { result, err = d.checkSafeBrowsing(host) if err != nil { // failed to do HTTP lookup -- treat it as if we got empty response, but don't save cache log.Printf("Failed to do safebrowsing HTTP lookup, ignoring check: %v", err) return Result{}, nil } if result.Reason.Matched() { return result, nil } } // check parental if no match if d.ParentalEnabled { result, err = d.checkParental(host) if err != nil { // failed to do HTTP lookup -- treat it as if we got empty response, but don't save cache log.Printf("Failed to do parental HTTP lookup, ignoring check: %v", err) return Result{}, nil } if result.Reason.Matched() { return result, nil } } // nothing matched, return nothing return Result{}, nil } // // rules table // type rulesTable struct { rulesByHost map[string]*rule rulesByShortcut map[string][]*rule rulesLeftovers []*rule sync.RWMutex } func newRulesTable() *rulesTable { return &rulesTable{ rulesByHost: make(map[string]*rule), rulesByShortcut: make(map[string][]*rule), rulesLeftovers: make([]*rule, 0), } } func (r *rulesTable) Add(rule *rule) { r.Lock() if rule.ip != nil { // Hosts syntax r.rulesByHost[rule.text] = rule } else if len(rule.shortcut) == shortcutLength && enableFastLookup { // Adblock syntax with a shortcut r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule) } else { // Adblock syntax -- too short to have a shortcut r.rulesLeftovers = append(r.rulesLeftovers, rule) } r.Unlock() } func (r *rulesTable) matchByHost(host string) (Result, error) { // First: examine the hosts-syntax rules res, err := r.searchByHost(host) if err != nil { return res, err } if res.Reason.Matched() { return res, nil } // Second: examine the adblock-syntax rules with shortcuts res, err = r.searchShortcuts(host) if err != nil { return res, err } if res.Reason.Matched() { return res, nil } // Third: examine the others res, err = r.searchLeftovers(host) if err != nil { return res, err } if res.Reason.Matched() { return res, nil } return Result{}, nil } func (r *rulesTable) searchByHost(host string) (Result, error) { rule, ok := r.rulesByHost[host] if ok { return rule.match(host) } return Result{}, nil } func (r *rulesTable) searchShortcuts(host string) (Result, error) { // check in shortcuts first for i := 0; i < len(host); i++ { shortcut := host[i:] if len(shortcut) > shortcutLength { shortcut = shortcut[:shortcutLength] } if len(shortcut) != shortcutLength { continue } rules, ok := r.rulesByShortcut[shortcut] if !ok { continue } for _, rule := range rules { res, err := rule.match(host) // error? stop search if err != nil { return res, err } // matched? stop search if res.Reason.Matched() { return res, err } // continue otherwise } } return Result{}, nil } func (r *rulesTable) searchLeftovers(host string) (Result, error) { for _, rule := range r.rulesLeftovers { res, err := rule.match(host) // error? stop search if err != nil { return res, err } // matched? stop search if res.Reason.Matched() { return res, err } // continue otherwise } return Result{}, nil } func findOptionIndex(text string) int { for i, r := range text { // ignore non-$ if r != '$' { continue } // ignore `\$` if i > 0 && text[i-1] == '\\' { continue } // ignore `$/` if i > len(text) && text[i+1] == '/' { continue } return i + 1 } return -1 } func (rule *rule) extractOptions() error { optIndex := findOptionIndex(rule.text) if optIndex == 0 { // starts with $ return ErrInvalidSyntax } if optIndex == len(rule.text) { // ends with $ return ErrInvalidSyntax } if optIndex < 0 { return nil } optionsStr := rule.text[optIndex:] rule.text = rule.text[:optIndex-1] // remove options from text begin := 0 i := 0 for i = 0; i < len(optionsStr); i++ { switch optionsStr[i] { case ',': if i > 0 { // it might be escaped, if so, ignore if optionsStr[i-1] == '\\' { break // from switch, not for loop } } rule.options = append(rule.options, optionsStr[begin:i]) begin = i + 1 } } if begin != i { // there's still an option remaining rule.options = append(rule.options, optionsStr[begin:]) } return nil } func (rule *rule) parseOptions() error { err := rule.extractOptions() if err != nil { return err } for _, option := range rule.options { switch { case option == "important": rule.isImportant = true case strings.HasPrefix(option, "app="): option = strings.TrimPrefix(option, "app=") rule.apps = strings.Split(option, "|") default: return ErrInvalidSyntax } } return nil } func (rule *rule) extractShortcut() { // regex rules have no shortcuts if rule.text[0] == '/' && rule.text[len(rule.text)-1] == '/' { return } fields := strings.FieldsFunc(rule.text, func(r rune) bool { switch r { case '*', '^', '|': return true } return false }) longestField := "" for _, field := range fields { if len(field) > len(longestField) { longestField = field } } if len(longestField) > shortcutLength { longestField = longestField[:shortcutLength] } rule.shortcut = strings.ToLower(longestField) } func (rule *rule) compile() error { rule.RLock() isCompiled := rule.isSuffix || rule.compiled != nil rule.RUnlock() if isCompiled { return nil } isSuffix, suffix := getSuffix(rule.text) if isSuffix { rule.Lock() rule.isSuffix = isSuffix rule.suffix = suffix rule.Unlock() return nil } expr, err := ruleToRegexp(rule.text) if err != nil { return err } compiled, err := regexp.Compile(expr) if err != nil { return err } rule.Lock() rule.compiled = compiled rule.Unlock() return nil } // Checks if the rule matches the specified host and returns a corresponding Result object func (rule *rule) match(host string) (Result, error) { res := Result{} if rule.ip != nil && rule.text == host { // This is a hosts-syntax rule -- just check that the hostname matches and return the result return Result{ IsFiltered: true, Reason: FilteredBlackList, Rule: rule.originalText, IP: rule.ip, FilterID: rule.listID, }, nil } err := rule.compile() if err != nil { return res, err } rule.RLock() matched := false if rule.isSuffix { if host == rule.suffix { matched = true } else if strings.HasSuffix(host, "."+rule.suffix) { matched = true } } else { matched = rule.compiled.MatchString(host) } rule.RUnlock() if matched { res.Reason = FilteredBlackList res.IsFiltered = true res.FilterID = rule.listID res.Rule = rule.originalText if rule.isWhitelist { res.Reason = NotFilteredWhiteList res.IsFiltered = false } } return res, nil } func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) { isFound = false // not found yet // get raw value rawValue, err := cache.Get(host) if err == gcache.KeyNotFoundError { // not a real error, just not found err = nil return } if err != nil { // real error return } // since it can be something else, validate that it belongs to proper type cachedValue, ok := rawValue.(Result) if !ok { // this is not our type -- error text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache" log.Println(text) err = errors.New(text) return } isFound = ok return cachedValue, isFound, err } // for each dot, hash it and add it to string func hostnameToHashParam(host string, addslash bool) (string, map[string]bool) { var hashparam bytes.Buffer hashes := map[string]bool{} tld, icann := publicsuffix.PublicSuffix(host) if !icann { // private suffixes like cloudfront.net tld = "" } curhost := host for { if curhost == "" { // we've reached end of string break } if tld != "" && curhost == tld { // we've reached the TLD, don't hash it break } tohash := []byte(curhost) if addslash { tohash = append(tohash, '/') } sum := sha256.Sum256(tohash) hexhash := fmt.Sprintf("%X", sum) hashes[hexhash] = true hashparam.WriteString(fmt.Sprintf("%02X%02X%02X%02X/", sum[0], sum[1], sum[2], sum[3])) pos := strings.IndexByte(curhost, byte('.')) if pos < 0 { break } curhost = curhost[pos+1:] } return hashparam.String(), hashes } func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) { // prevent recursion -- checking the host of safebrowsing server makes no sense if host == d.safeBrowsingServer { return Result{}, nil } format := func(hashparam string) string { url := fmt.Sprintf(defaultSafebrowsingURL, d.safeBrowsingServer, hashparam) return url } handleBody := func(body []byte, hashes map[string]bool) (Result, error) { result := Result{} scanner := bufio.NewScanner(strings.NewReader(string(body))) for scanner.Scan() { line := scanner.Text() splitted := strings.Split(line, ":") if len(splitted) < 3 { continue } hash := splitted[2] if _, ok := hashes[hash]; ok { // it's in the hash result.IsFiltered = true result.Reason = FilteredSafeBrowsing result.Rule = splitted[0] break } } if err := scanner.Err(); err != nil { // error, don't save cache return Result{}, err } return result, nil } if safebrowsingCache == nil { safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() } result, err := d.lookupCommon(host, &stats.Safebrowsing, safebrowsingCache, true, format, handleBody) return result, err } func (d *Dnsfilter) checkParental(host string) (Result, error) { // prevent recursion -- checking the host of parental safety server makes no sense if host == d.parentalServer { return Result{}, nil } format := func(hashparam string) string { url := fmt.Sprintf(defaultParentalURL, d.parentalServer, hashparam, d.ParentalSensitivity) return url } handleBody := func(body []byte, hashes map[string]bool) (Result, error) { // parse json var m []struct { Blocked bool `json:"blocked"` ClientTTL int `json:"clientTtl"` Reason string `json:"reason"` Hash string `json:"hash"` } err := json.Unmarshal(body, &m) if err != nil { // error, don't save cache log.Printf("Couldn't parse json '%s': %s", body, err) return Result{}, err } result := Result{} for i := range m { if !hashes[m[i].Hash] { continue } if m[i].Blocked { result.IsFiltered = true result.Reason = FilteredParental result.Rule = fmt.Sprintf("parental %s", m[i].Reason) break } } return result, nil } if parentalCache == nil { parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build() } result, err := d.lookupCommon(host, &stats.Parental, parentalCache, false, format, handleBody) return result, err } type formatHandler func(hashparam string) string type handleBodyHandler func(body []byte, hashes map[string]bool) (Result, error) // real implementation of lookup/check func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gcache.Cache, hashparamNeedSlash bool, format formatHandler, handleBody handleBodyHandler) (Result, error) { // if host ends with a dot, trim it host = strings.ToLower(strings.Trim(host, ".")) // check cache cachedValue, isFound, err := getCachedReason(cache, host) if isFound { atomic.AddUint64(&lookupstats.CacheHits, 1) return cachedValue, nil } if err != nil { return Result{}, err } // convert hostname to hash parameters hashparam, hashes := hostnameToHashParam(host, hashparamNeedSlash) // format URL with our hashes url := format(hashparam) // do HTTP request atomic.AddUint64(&lookupstats.Requests, 1) atomic.AddInt64(&lookupstats.Pending, 1) updateMax(&lookupstats.Pending, &lookupstats.PendingMax) resp, err := d.client.Get(url) atomic.AddInt64(&lookupstats.Pending, -1) if resp != nil && resp.Body != nil { defer resp.Body.Close() } if err != nil { // error, don't save cache return Result{}, err } // get body text body, err := ioutil.ReadAll(resp.Body) if err != nil { // error, don't save cache return Result{}, err } // handle status code switch { case resp.StatusCode == 204: // empty result, save cache err = cache.Set(host, Result{}) if err != nil { return Result{}, err } return Result{}, nil case resp.StatusCode != 200: // error, don't save cache return Result{}, nil } result, err := handleBody(body, hashes) if err != nil { // error, don't save cache return Result{}, err } err = cache.Set(host, result) if err != nil { return Result{}, err } return result, nil } // // Adding rule and matching against the rules // // AddRules is a convinience function to add an array of filters in one call func (d *Dnsfilter) AddRules(filters []Filter) error { for _, f := range filters { for _, rule := range f.Rules { err := d.AddRule(rule, f.ID) if err == ErrAlreadyExists || err == ErrInvalidSyntax { continue } if err != nil { log.Printf("Cannot add rule %s: %s", rule, err) // Just ignore invalid rules continue } } } return nil } // AddRule adds a rule, checking if it is a valid rule first and if it wasn't added already func (d *Dnsfilter) AddRule(input string, filterListID int64) error { input = strings.TrimSpace(input) d.storageMutex.RLock() _, exists := d.storage[input] d.storageMutex.RUnlock() if exists { // already added return ErrAlreadyExists } if !isValidRule(input) { return ErrInvalidSyntax } // First, check if this is a hosts-syntax rule if d.parseEtcHosts(input, filterListID) { // This is a valid hosts-syntax rule, no need for further parsing return nil } // Start parsing the rule rule := rule{ text: input, // will be modified originalText: input, listID: filterListID, } // Mark rule as whitelist if it starts with @@ if strings.HasPrefix(rule.text, "@@") { rule.isWhitelist = true rule.text = rule.text[2:] } err := rule.parseOptions() if err != nil { return err } rule.extractShortcut() if !enableDelayedCompilation { err := rule.compile() if err != nil { return err } } destination := d.blackList if rule.isImportant { destination = d.important } else if rule.isWhitelist { destination = d.whiteList } d.storageMutex.Lock() d.storage[input] = true d.storageMutex.Unlock() destination.Add(&rule) return nil } // Parses the hosts-syntax rules. Returns false if the input string is not of hosts-syntax. func (d *Dnsfilter) parseEtcHosts(input string, filterListID int64) bool { // Strip the trailing comment ruleText := input if pos := strings.IndexByte(ruleText, '#'); pos != -1 { ruleText = ruleText[0:pos] } fields := strings.Fields(ruleText) if len(fields) < 2 { return false } addr := net.ParseIP(fields[0]) if addr == nil { return false } d.storageMutex.Lock() d.storage[input] = true d.storageMutex.Unlock() for _, host := range fields[1:] { rule := rule{ text: host, originalText: input, listID: filterListID, ip: addr, } d.blackList.Add(&rule) } return true } // matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups func (d *Dnsfilter) matchHost(host string) (Result, error) { lists := []*rulesTable{ d.important, d.whiteList, d.blackList, } for _, table := range lists { res, err := table.matchByHost(host) if err != nil { return res, err } if res.Reason.Matched() { return res, nil } } return Result{}, nil } // // lifecycle helper functions // // New creates properly initialized DNS Filter that is ready to be used func New(c *Config) *Dnsfilter { d := new(Dnsfilter) d.storage = make(map[string]bool) d.important = newRulesTable() d.whiteList = newRulesTable() d.blackList = newRulesTable() // Customize the Transport to have larger connection pool defaultRoundTripper := http.DefaultTransport defaultTransportPointer, ok := defaultRoundTripper.(*http.Transport) if !ok { panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport")) } d.transport = defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to d.transport.MaxIdleConns = defaultHTTPMaxIdleConnections // default 100 d.transport.MaxIdleConnsPerHost = defaultHTTPMaxIdleConnections // default 2 d.client = http.Client{ Transport: d.transport, Timeout: defaultHTTPTimeout, } d.safeBrowsingServer = defaultSafebrowsingServer d.parentalServer = defaultParentalServer if c != nil { d.Config = *c } return d } // Destroy is optional if you want to tidy up goroutines without waiting for them to die off // right now it closes idle HTTP connections if there are any func (d *Dnsfilter) Destroy() { if d != nil && d.transport != nil { d.transport.CloseIdleConnections() } } // // config manipulation helpers // // SetSafeBrowsingServer lets you optionally change hostname of safesearch lookup func (d *Dnsfilter) SetSafeBrowsingServer(host string) { if len(host) == 0 { d.safeBrowsingServer = defaultSafebrowsingServer } else { d.safeBrowsingServer = host } } // SetHTTPTimeout lets you optionally change timeout during lookups func (d *Dnsfilter) SetHTTPTimeout(t time.Duration) { d.client.Timeout = t } // ResetHTTPTimeout resets lookup timeouts func (d *Dnsfilter) ResetHTTPTimeout() { d.client.Timeout = defaultHTTPTimeout } // SafeSearchDomain returns replacement address for search engine func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) { if d.SafeSearchEnabled { val, ok := safeSearchDomains[host] return val, ok } return "", false } // // stats // // GetStats return dns filtering stats since startup func (d *Dnsfilter) GetStats() Stats { return stats } // Count returns number of rules added to filter func (d *Dnsfilter) Count() int { return len(d.storage) }