diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 7107418a..d63552f2 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -201,6 +201,7 @@ "form_error_url_or_path_format": "Invalid URL or absolute path of the list", "custom_filter_rules": "Custom filtering rules", "custom_filter_rules_hint": "Enter one rule on a line. You can use either adblock rules or hosts files syntax.", + "system_host_files": "System hosts files", "examples_title": "Examples", "example_meaning_filter_block": "block access to the example.org domain and all its subdomains", "example_meaning_filter_whitelist": "unblock access to the example.org domain and all its subdomains", diff --git a/client/src/helpers/constants.js b/client/src/helpers/constants.js index 195efdb3..0109e030 100644 --- a/client/src/helpers/constants.js +++ b/client/src/helpers/constants.js @@ -529,6 +529,7 @@ export const DETAILED_DATE_FORMAT_OPTIONS = { }; export const CUSTOM_FILTERING_RULES_ID = 0; +export const SYSTEM_HOSTS_FILTER_ID = -1; export const BLOCK_ACTIONS = { BLOCK: 'block', diff --git a/client/src/helpers/helpers.js b/client/src/helpers/helpers.js index d6eaa061..f5271106 100644 --- a/client/src/helpers/helpers.js +++ b/client/src/helpers/helpers.js @@ -26,6 +26,7 @@ import { STANDARD_DNS_PORT, STANDARD_HTTPS_PORT, STANDARD_WEB_PORT, + SYSTEM_HOSTS_FILTER_ID, } from './constants'; /** @@ -791,9 +792,12 @@ export const getFilterName = ( return i18n.t(customFilterTranslationKey); } + if (filterId === SYSTEM_HOSTS_FILTER_ID) { + return i18n.t('system_host_files'); + } + const matchIdPredicate = (filter) => filter.id === filterId; const filter = filters.find(matchIdPredicate) || whitelistFilters.find(matchIdPredicate); - return resolveFilterName(filter); }; diff --git a/internal/aghio/limitedreader.go b/internal/aghio/limitedreader.go index 5b6c57d9..02905e76 100644 --- a/internal/aghio/limitedreader.go +++ b/internal/aghio/limitedreader.go @@ -35,7 +35,7 @@ func (lr *limitedReader) Read(p []byte) (n int, err error) { } if int64(len(p)) > lr.n { - p = p[0:lr.n] + p = p[:lr.n] } n, err = lr.r.Read(p) diff --git a/internal/aghnet/hostscontainer.go b/internal/aghnet/hostscontainer.go index cfa55dd0..d6ca93f4 100644 --- a/internal/aghnet/hostscontainer.go +++ b/internal/aghnet/hostscontainer.go @@ -28,6 +28,69 @@ func DefaultHostsPaths() (paths []string) { return defaultHostsPaths() } +// requestMatcher combines the logic for matching requests and translating the +// appropriate rules. +type requestMatcher struct { + // stateLock protects all the fields of requestMatcher. + stateLock *sync.RWMutex + + // rulesStrg stores the rules obtained from the hosts' file. + rulesStrg *filterlist.RuleStorage + // engine serves rulesStrg. + engine *urlfilter.DNSEngine + + // translator maps generated $dnsrewrite rules into hosts-syntax rules. + // + // TODO(e.burkov): Store the filename from which the rule was parsed. + translator map[string]string +} + +// MatchRequest processes the request rewriting hostnames and addresses read +// from the operating system's hosts files. +// +// res is nil for any request having not an A/AAAA or PTR type. Results +// containing CNAME information may be queried again with the same question type +// and the returned CNAME for Host field of request. Results are guaranteed to +// be direct, i.e. any returned CNAME resolves into actual address like an alias +// in hosts does, see man hosts (5). +// +// It's safe for concurrent use. +func (rm *requestMatcher) MatchRequest( + req urlfilter.DNSRequest, +) (res *urlfilter.DNSResult, ok bool) { + switch req.DNSType { + case dns.TypeA, dns.TypeAAAA, dns.TypePTR: + log.Debug("%s: handling the request", hostsContainerPref) + default: + return nil, false + } + + rm.stateLock.RLock() + defer rm.stateLock.RUnlock() + + return rm.engine.MatchRequest(req) +} + +// Translate returns the source hosts-syntax rule for the generated dnsrewrite +// rule or an empty string if the last doesn't exist. +func (rm *requestMatcher) Translate(rule string) (hostRule string) { + rm.stateLock.RLock() + defer rm.stateLock.RUnlock() + + return rm.translator[rule] +} + +// resetEng updates container's engine and the translation map. +func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) { + rm.stateLock.Lock() + defer rm.stateLock.Unlock() + + rm.rulesStrg = rulesStrg + rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg) + + rm.translator = tr +} + // hostsContainerPref is a prefix for logging and wrapping errors in // HostsContainer's methods. const hostsContainerPref = "hosts container" @@ -35,13 +98,9 @@ const hostsContainerPref = "hosts container" // HostsContainer stores the relevant hosts database provided by the OS and // processes both A/AAAA and PTR DNS requests for those. type HostsContainer struct { - // engLock protects rulesStrg and engine. - engLock *sync.RWMutex - - // rulesStrg stores the rules obtained from the hosts' file. - rulesStrg *filterlist.RuleStorage - // engine serves rulesStrg. - engine *urlfilter.DNSEngine + // requestMatcher matches the requests and translates the rules. It's + // embedded to implement MatchRequest and Translate for *HostsContainer. + requestMatcher // done is the channel to sign closing the container. done chan struct{} @@ -87,7 +146,9 @@ func NewHostsContainer( } hc = &HostsContainer{ - engLock: &sync.RWMutex{}, + requestMatcher: requestMatcher{ + stateLock: &sync.RWMutex{}, + }, done: make(chan struct{}, 1), updates: make(chan *netutil.IPMap, 1), fsys: fsys, @@ -117,25 +178,6 @@ func NewHostsContainer( return hc, nil } -// MatchRequest is the request processing method to resolve hostnames and -// addresses from the operating system's hosts files. res is nil for any -// request having not an A/AAAA or PTR type. It's safe for concurrent use. -func (hc *HostsContainer) MatchRequest( - req urlfilter.DNSRequest, -) (res *urlfilter.DNSResult, ok bool) { - switch req.DNSType { - case dns.TypeA, dns.TypeAAAA, dns.TypePTR: - log.Debug("%s: handling the request", hostsContainerPref) - default: - return nil, false - } - - hc.engLock.RLock() - defer hc.engLock.RUnlock() - - return hc.engine.MatchRequest(req) -} - // Close implements the io.Closer interface for *HostsContainer. Close must // only be called once. The returned err is always nil. func (hc *HostsContainer) Close() (err error) { @@ -203,10 +245,17 @@ func (hc *HostsContainer) handleEvents() { } // hostsParser is a helper type to parse rules from the operating system's hosts -// file. +// file. It exists for only a single refreshing session. type hostsParser struct { - // rules builds the resulting rules list content. - rules *strings.Builder + // rulesBuilder builds the resulting rulesBuilder list content. + rulesBuilder *strings.Builder + + // translations maps generated $dnsrewrite rules to the hosts-translations + // rules. + translations map[string]string + + // cnameSet prevents duplicating cname rules. + cnameSet *stringutil.Set // table stores only the unique IP-hostname pairs. It's also sent to the // updates channel afterwards. @@ -215,8 +264,11 @@ type hostsParser struct { func (hc *HostsContainer) newHostsParser() (hp *hostsParser) { return &hostsParser{ - rules: &strings.Builder{}, - table: netutil.NewIPMap(hc.last.Len()), + rulesBuilder: &strings.Builder{}, + // For A/AAAA and PTRs. + translations: make(map[string]string, hc.last.Len()*2), + cnameSet: stringutil.NewSet(), + table: netutil.NewIPMap(hc.last.Len()), } } @@ -234,9 +286,7 @@ func (hp *hostsParser) parseFile( continue } - for _, host := range hosts { - hp.addPair(ip, host) - } + hp.addPairs(ip, hosts) } return nil, true, s.Err() @@ -244,7 +294,6 @@ func (hp *hostsParser) parseFile( // parseLine parses the line having the hosts syntax ignoring invalid ones. func (hp *hostsParser) parseLine(line string) (ip net.IP, hosts []string) { - line = strings.TrimSpace(line) fields := strings.Fields(line) if len(fields) < 2 { return nil, nil @@ -274,74 +323,142 @@ loop: return ip, hosts } -// add returns true if the pair of ip and host wasn't added to the hp before. -func (hp *hostsParser) add(ip net.IP, host string) (added bool) { +// Simple types of hosts in hosts database. Zero value isn't used to be able +// quizzaciously emulate nil with 0. +const ( + _ = iota + hostAlias + hostMain +) + +// add tries to add the ip-host pair. It returns: +// +// hostAlias if the host is not the first one added for the ip. +// hostMain if the host is the first one added for the ip. +// 0 if the ip-host pair has already been added. +// +func (hp *hostsParser) add(ip net.IP, host string) (hostType int) { v, ok := hp.table.Get(ip) - hosts, _ := v.(*stringutil.Set) - switch { + switch hosts, _ := v.(*stringutil.Set); { case ok && hosts.Has(host): - return false + return 0 case hosts == nil: hosts = stringutil.NewSet(host) hp.table.Set(ip, hosts) + + return hostMain default: hosts.Add(host) - } - return true + return hostAlias + } } -// addPair puts the pair of ip and host to the rules builder if needed. -func (hp *hostsParser) addPair(ip net.IP, host string) { +// addPair puts the pair of ip and host to the rules builder if needed. For +// each ip the first member of hosts will become the main one. +func (hp *hostsParser) addPairs(ip net.IP, hosts []string) { + // Put the rule in a preproccesed format like: + // + // ip host1 host2 ... + // + hostsLine := strings.Join(append([]string{ip.String()}, hosts...), " ") + var mainHost string + for _, host := range hosts { + switch hp.add(ip, host) { + case 0: + continue + case hostMain: + mainHost = host + added, addedPtr := hp.writeMainHostRule(host, ip) + hp.translations[added], hp.translations[addedPtr] = hostsLine, hostsLine + case hostAlias: + pair := fmt.Sprint(host, " ", mainHost) + if hp.cnameSet.Has(pair) { + continue + } + // Since the hostAlias couldn't be returned from add before the + // hostMain the mainHost shouldn't appear empty. + hp.writeAliasHostRule(host, mainHost) + hp.cnameSet.Add(pair) + } + + log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, host) + } +} + +// writeAliasHostRule writes the CNAME rule for the alias-host pair into +// internal builders. +func (hp *hostsParser) writeAliasHostRule(alias, host string) { + const ( + nl = "\n" + sc = ";" + + rwSuccess = rules.MaskSeparator + "$dnsrewrite=NOERROR" + sc + "CNAME" + sc + constLen = len(rules.MaskStartURL) + len(rwSuccess) + len(nl) + ) + + hp.rulesBuilder.Grow(constLen + len(host) + len(alias)) + stringutil.WriteToBuilder(hp.rulesBuilder, rules.MaskStartURL, alias, rwSuccess, host, nl) +} + +// writeMainHostRule writes the actual rule for the qtype and the PTR for the +// host-ip pair into internal builders. +func (hp *hostsParser) writeMainHostRule(host string, ip net.IP) (added, addedPtr string) { arpa, err := netutil.IPToReversedAddr(ip) if err != nil { return } - if !hp.add(ip, host) { - return - } - - qtype := "AAAA" - if ip.To4() != nil { - // Assume the validation of the IP address is performed already. - qtype = "A" - } - const ( nl = "\n" - sc = ";" - rewriteSuccess = "$dnsrewrite=NOERROR" + sc - rewriteSuccessPTR = rewriteSuccess + "PTR" + sc + rwSuccess = "^$dnsrewrite=NOERROR;" + rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;" + + modLen = len("||") + len(rwSuccess) + modLenPTR = len("||") + len(rwSuccessPTR) ) + var qtype string + // The validation of the IP address has been performed earlier so it is + // guaranteed to be either an IPv4 or an IPv6. + if ip.To4() != nil { + qtype = "A" + } else { + qtype = "AAAA" + } + ipStr := ip.String() fqdn := dns.Fqdn(host) - for _, ruleData := range [...][]string{{ - // A/AAAA. - rules.MaskStartURL, + ruleBuilder := &strings.Builder{} + ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr)) + stringutil.WriteToBuilder( + ruleBuilder, + "||", host, - rules.MaskSeparator, - rewriteSuccess, + rwSuccess, qtype, - sc, + ";", ipStr, - nl, - }, { - // PTR. - rules.MaskStartURL, - arpa, - rules.MaskSeparator, - rewriteSuccessPTR, - fqdn, - nl, - }} { - stringutil.WriteToBuilder(hp.rules, ruleData...) - } + ) + added = ruleBuilder.String() - log.Debug("%s: added ip-host pair %q/%q", hostsContainerPref, ip, host) + ruleBuilder.Reset() + ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn)) + stringutil.WriteToBuilder( + ruleBuilder, + "||", + arpa, + rwSuccessPTR, + fqdn, + ) + addedPtr = ruleBuilder.String() + + hp.rulesBuilder.Grow(len(added) + len(addedPtr) + 2*len(nl)) + stringutil.WriteToBuilder(hp.rulesBuilder, added, nl, addedPtr, nl) + + return added, addedPtr } // equalSet returns true if the internal hosts table just parsed equals target. @@ -385,15 +502,16 @@ func (hp *hostsParser) sendUpd(ch chan *netutil.IPMap) { case ch <- upd: // The previous update was just read and the next one pushed. Go on. default: - log.Debug("%s: the channel is broken", hostsContainerPref) + log.Error("%s: the updates channel is broken", hostsContainerPref) } } // newStrg creates a new rules storage from parsed data. func (hp *hostsParser) newStrg() (s *filterlist.RuleStorage, err error) { return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{ + // TODO(e.burkov): Make configurable. ID: -1, - RulesText: hp.rules.String(), + RulesText: hp.rulesBuilder.String(), IgnoreCosmetic: true, }}) } @@ -424,15 +542,7 @@ func (hc *HostsContainer) refresh() (err error) { return fmt.Errorf("initializing rules storage: %w", err) } - hc.resetEng(rulesStrg) + hc.resetEng(rulesStrg, hp.translations) return nil } - -func (hc *HostsContainer) resetEng(rulesStrg *filterlist.RuleStorage) { - hc.engLock.Lock() - defer hc.engLock.Unlock() - - hc.rulesStrg = rulesStrg - hc.engine = urlfilter.NewDNSEngine(hc.rulesStrg) -} diff --git a/internal/aghnet/hostscontainer_test.go b/internal/aghnet/hostscontainer_test.go index 5a08b24f..5686a11b 100644 --- a/internal/aghnet/hostscontainer_test.go +++ b/internal/aghnet/hostscontainer_test.go @@ -3,6 +3,7 @@ package aghnet import ( "io/fs" "net" + "os" "path" "strings" "sync/atomic" @@ -11,9 +12,9 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghtest" "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/urlfilter" + "github.com/AdguardTeam/urlfilter/rules" "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -203,129 +204,6 @@ func TestHostsContainer_Refresh(t *testing.T) { }) } -func TestHostsContainer_MatchRequest(t *testing.T) { - var ( - ip4 = net.IP{127, 0, 0, 1} - ip6 = net.IP{ - 128, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 1, - } - - hostname4 = "localhost" - hostname6 = "localhostv6" - hostname4a = "abcd" - - reversed4, _ = netutil.IPToReversedAddr(ip4) - reversed6, _ = netutil.IPToReversedAddr(ip6) - ) - - const filename = "file1" - - gsfs := fstest.MapFS{ - filename: &fstest.MapFile{Data: []byte( - ip4.String() + " " + hostname4 + " " + hostname4a + nl + - ip6.String() + " " + hostname6 + nl + - `256.256.256.256 fakebroadcast` + nl, - )}, - } - - hc, err := NewHostsContainer(gsfs, &aghtest.FSWatcher{ - OnEvents: func() (e <-chan struct{}) { panic("not implemented") }, - OnAdd: func(name string) (err error) { - assert.Equal(t, filename, name) - - return nil - }, - OnClose: func() (err error) { panic("not implemented") }, - }, filename) - require.NoError(t, err) - - testCase := []struct { - name string - want []interface{} - req urlfilter.DNSRequest - }{{ - name: "a", - want: []interface{}{ip4.To16()}, - req: urlfilter.DNSRequest{ - Hostname: hostname4, - DNSType: dns.TypeA, - }, - }, { - name: "a_for_aaaa", - want: []interface{}{ - ip4.To16(), - }, - req: urlfilter.DNSRequest{ - Hostname: hostname4, - DNSType: dns.TypeAAAA, - }, - }, { - name: "aaaa", - want: []interface{}{ip6}, - req: urlfilter.DNSRequest{ - Hostname: hostname6, - DNSType: dns.TypeAAAA, - }, - }, { - name: "ptr", - want: []interface{}{ - dns.Fqdn(hostname4), - dns.Fqdn(hostname4a), - }, - req: urlfilter.DNSRequest{ - Hostname: reversed4, - DNSType: dns.TypePTR, - }, - }, { - name: "ptr_v6", - want: []interface{}{dns.Fqdn(hostname6)}, - req: urlfilter.DNSRequest{ - Hostname: reversed6, - DNSType: dns.TypePTR, - }, - }, { - name: "a_alias", - want: []interface{}{ip4.To16()}, - req: urlfilter.DNSRequest{ - Hostname: hostname4a, - DNSType: dns.TypeA, - }, - }} - - for _, tc := range testCase { - t.Run(tc.name, func(t *testing.T) { - res, ok := hc.MatchRequest(tc.req) - require.False(t, ok) - require.NotNil(t, res) - - rws := res.DNSRewrites() - require.Len(t, rws, len(tc.want)) - - for i, w := range tc.want { - require.NotNil(t, rws[i]) - - rw := rws[i].DNSRewrite - require.NotNil(t, rw) - - assert.Equal(t, w, rw.Value) - } - }) - } - - t.Run("cname", func(t *testing.T) { - res, ok := hc.MatchRequest(urlfilter.DNSRequest{ - Hostname: hostname4, - DNSType: dns.TypeCNAME, - }) - require.False(t, ok) - - assert.Nil(t, res) - }) -} - func TestHostsContainer_PathsToPatterns(t *testing.T) { const ( dir0 = "dir" @@ -412,53 +290,108 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) { }) } -func TestUniqueRules_AddPair(t *testing.T) { - knownIP := net.IP{1, 2, 3, 4} +func TestHostsContainer(t *testing.T) { + testdata := os.DirFS("./testdata") - const knownHost = "host1" + nRewrites := func(t *testing.T, res *urlfilter.DNSResult, n int) (rws []*rules.DNSRewrite) { + t.Helper() - ipToHost := netutil.NewIPMap(0) - ipToHost.Set(knownIP, *stringutil.NewSet(knownHost)) + rewrites := res.DNSRewrites() + assert.Len(t, rewrites, n) - testCases := []struct { - name string - host string - wantRules string - ip net.IP - }{{ - name: "new_one", - host: "host2", - wantRules: "||host2^$dnsrewrite=NOERROR;A;1.2.3.4\n" + - "||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host2.\n", - ip: knownIP, - }, { - name: "existing_one", - host: knownHost, - wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.4\n" + - "||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host1.\n", - ip: knownIP, - }, { - name: "new_ip", - host: knownHost, - wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.5\n" + - "||5.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;" + knownHost + ".\n", - ip: net.IP{1, 2, 3, 5}, - }, { - name: "bad_ip", - host: knownHost, - wantRules: "", - ip: net.IP{1, 2, 3, 4, 5}, - }} + for _, rewrite := range rewrites { + rw := rewrite.DNSRewrite + require.NotNil(t, rw) - for _, tc := range testCases { - hp := hostsParser{ - rules: &strings.Builder{}, - table: ipToHost.ShallowClone(), + rws = append(rws, rw) } + return rws + } + + testCases := []struct { + testTail func(t *testing.T, res *urlfilter.DNSResult) + name string + req urlfilter.DNSRequest + }{{ + name: "simple", + req: urlfilter.DNSRequest{ + Hostname: "simplehost", + DNSType: dns.TypeA, + }, + testTail: func(t *testing.T, res *urlfilter.DNSResult) { + rws := nRewrites(t, res, 2) + + v, ok := rws[0].Value.(net.IP) + require.True(t, ok) + + assert.True(t, net.IP{1, 0, 0, 1}.Equal(v)) + + v, ok = rws[1].Value.(net.IP) + require.True(t, ok) + + // It's ::1. + assert.True(t, net.IP(append((&[15]byte{})[:], byte(1))).Equal(v)) + }, + }, { + name: "hello_alias", + req: urlfilter.DNSRequest{ + Hostname: "hello.world", + DNSType: dns.TypeA, + }, + testTail: func(t *testing.T, res *urlfilter.DNSResult) { + assert.Equal(t, "hello", nRewrites(t, res, 1)[0].NewCNAME) + }, + }, { + name: "lots_of_aliases", + req: urlfilter.DNSRequest{ + Hostname: "for.testing", + DNSType: dns.TypeA, + }, + testTail: func(t *testing.T, res *urlfilter.DNSResult) { + assert.Equal(t, "a.whole", nRewrites(t, res, 1)[0].NewCNAME) + }, + }, { + name: "reverse", + req: urlfilter.DNSRequest{ + Hostname: "1.0.0.1.in-addr.arpa", + DNSType: dns.TypePTR, + }, + testTail: func(t *testing.T, res *urlfilter.DNSResult) { + rws := nRewrites(t, res, 1) + + assert.Equal(t, dns.TypePTR, rws[0].RRType) + assert.Equal(t, "simplehost.", rws[0].Value) + }, + }, { + name: "non-existing", + req: urlfilter.DNSRequest{ + Hostname: "nonexisting", + DNSType: dns.TypeA, + }, + testTail: func(t *testing.T, res *urlfilter.DNSResult) { + require.NotNil(t, res) + + assert.Nil(t, res.DNSRewrites()) + }, + }} + + stubWatcher := aghtest.FSWatcher{ + OnEvents: func() (e <-chan struct{}) { return nil }, + OnAdd: func(name string) (err error) { return nil }, + OnClose: func() (err error) { panic("not implemented") }, + } + + hc, err := NewHostsContainer(testdata, &stubWatcher, "etc_hosts") + require.NoError(t, err) + + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - hp.addPair(tc.ip, tc.host) - assert.Equal(t, tc.wantRules, hp.rules.String()) + res, ok := hc.MatchRequest(tc.req) + require.False(t, ok) + require.NotNil(t, res) + + tc.testTail(t, res) }) } } diff --git a/internal/aghnet/testdata/etc_hosts b/internal/aghnet/testdata/etc_hosts new file mode 100644 index 00000000..d3cabac2 --- /dev/null +++ b/internal/aghnet/testdata/etc_hosts @@ -0,0 +1,14 @@ +# +# Test /etc/hosts file +# + +1.0.0.1 simplehost +1.0.0.0 hello hello.world + +# See https://github.com/AdguardTeam/AdGuardHome/issues/3846. +1.0.0.2 a.whole lot.of aliases for.testing + +# Same for IPv6. +::1 simplehost +:: hello hello.world +::2 a.whole lot.of aliases for.testing diff --git a/internal/filtering/dnsrewrite.go b/internal/filtering/dnsrewrite.go index a6dda4a6..5263f252 100644 --- a/internal/filtering/dnsrewrite.go +++ b/internal/filtering/dnsrewrite.go @@ -27,7 +27,7 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) { for _, nr := range dnsr { dr := nr.DNSRewrite if dr.NewCNAME != "" { - // NewCNAME rules have a higher priority than the other rules. + // NewCNAME rules have a higher priority than other rules. rules = []*ResultRule{{ FilterListID: int64(nr.GetFilterListID()), Text: nr.RuleText, diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index c4dd4c05..156b08e1 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -376,16 +376,8 @@ type Result struct { // Rules are applied rules. If Rules are not empty, each rule is not nil. Rules []*ResultRule `json:",omitempty"` - // ReverseHosts is the reverse lookup rewrite result. It is empty unless - // Reason is set to RewrittenAutoHosts. - // - // TODO(e.burkov): There is no need for AutoHosts-related fields any more - // since the hosts container now uses $dnsrewrite rules. These fields are - // only used in query log to decode old format. - ReverseHosts []string `json:",omitempty"` - // IPList is the lookup rewrite result. It is empty unless Reason is set to - // RewrittenAutoHosts or Rewritten. + // Rewritten. IPList []net.IP `json:",omitempty"` // CanonName is the CNAME value from the lookup rewrite result. It is empty @@ -464,7 +456,7 @@ func (d *DNSFilter) matchSysHosts( return res, nil } - dnsres, _ := d.EtcHosts.MatchRequest(urlfilter.DNSRequest{ + return d.matchSysHostsIntl(&urlfilter.DNSRequest{ Hostname: host, SortedClientTags: setts.ClientTags, // TODO(e.burkov): Wait for urlfilter update to pass net.IP. @@ -472,18 +464,34 @@ func (d *DNSFilter) matchSysHosts( ClientName: setts.ClientName, DNSType: qtype, }) +} + +// matchSysHostsIntl actually matches the request. It's separated to avoid +// perfoming checks twice. +func (d *DNSFilter) matchSysHostsIntl( + req *urlfilter.DNSRequest, +) (res Result, err error) { + dnsres, _ := d.EtcHosts.MatchRequest(*req) if dnsres == nil { return res, nil } - if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 { - // Check DNS rewrites first, because the API there is a bit awkward. - res = d.processDNSRewrites(dnsr) - res.Reason = RewrittenAutoHosts - // TODO(e.burkov): Put real hosts-syntax rules. - // - // See https://github.com/AdguardTeam/AdGuardHome/issues/3846. - res.Rules = nil + dnsr := dnsres.DNSRewrites() + if len(dnsr) == 0 { + return res, nil + } + + res = d.processDNSRewrites(dnsr) + if cn := res.CanonName; cn != "" { + // Probably an alias. + req.Hostname = cn + + return d.matchSysHostsIntl(req) + } + + res.Reason = RewrittenAutoHosts + for _, r := range res.Rules { + r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text) } return res, nil @@ -799,7 +807,6 @@ func (d *DNSFilter) matchHost( } dnsres, ok := d.filteringEngine.MatchRequest(ureq) - // Check DNS rewrites first, because the API there is a bit awkward. if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 { res = d.processDNSRewrites(dnsr) diff --git a/internal/querylog/decode.go b/internal/querylog/decode.go index 944994a8..d23fc526 100644 --- a/internal/querylog/decode.go +++ b/internal/querylog/decode.go @@ -291,10 +291,13 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) { } if d, ok := keyToken.(json.Delim); ok { - if d == '}' { + switch d { + case '}': i++ - } else if d == ']' { + case ']': return + default: + // Go on. } continue @@ -312,6 +315,11 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) { } } +// decodeResultReverseHosts parses the dec's tokens into ent interpreting it as +// the result of hosts container's $dnsrewrite rule. It assumes there are no +// other occurrences of DNSRewriteResult in the entry since hosts container's +// rewrites currently has the highest priority along the entire filtering +// pipeline. func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) { for { itemToken, err := dec.Token() @@ -335,7 +343,25 @@ func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) { return case string: - ent.Result.ReverseHosts = append(ent.Result.ReverseHosts, v) + v = dns.Fqdn(v) + if res := &ent.Result; res.DNSRewriteResult == nil { + res.DNSRewriteResult = &filtering.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + Response: filtering.DNSRewriteResultResponse{ + dns.TypePTR: []rules.RRValue{v}, + }, + } + + continue + } else { + res.DNSRewriteResult.RCode = dns.RcodeSuccess + } + + if rres := ent.Result.DNSRewriteResult; rres.Response == nil { + rres.Response = filtering.DNSRewriteResultResponse{dns.TypePTR: []rules.RRValue{v}} + } else { + rres.Response[dns.TypePTR] = append(rres.Response[dns.TypePTR], v) + } default: continue } @@ -407,9 +433,9 @@ func decodeResultDNSRewriteResultKey(key string, dec *json.Decoder, ent *logEntr ent.Result.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{} } - // TODO(a.garipov): I give up. This whole file is a mess. - // Luckily, we can assume that this field is relatively rare and - // just use the normal decoding and correct the values. + // TODO(a.garipov): I give up. This whole file is a mess. Luckily, we + // can assume that this field is relatively rare and just use the normal + // decoding and correct the values. err = dec.Decode(&ent.Result.DNSRewriteResult.Response) if err != nil { log.Debug("decodeResultDNSRewriteResultKey response err: %s", err) @@ -463,7 +489,40 @@ func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) { } } +// translateResult converts some fields of the ent.Result to the format +// consistent with current implementation. +func translateResult(ent *logEntry) { + res := &ent.Result + if res.Reason != filtering.RewrittenAutoHosts || len(res.IPList) == 0 { + return + } + + if res.DNSRewriteResult == nil { + res.DNSRewriteResult = &filtering.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + } + } + + if res.DNSRewriteResult.Response == nil { + res.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{} + } + + resp := res.DNSRewriteResult.Response + for _, ip := range res.IPList { + qType := dns.TypeAAAA + if ip.To4() != nil { + qType = dns.TypeA + } + + resp[qType] = append(resp[qType], ip) + } + + res.IPList = nil +} + func decodeResult(dec *json.Decoder, ent *logEntry) { + defer translateResult(ent) + for { keyToken, err := dec.Token() if err != nil { diff --git a/internal/querylog/decode_test.go b/internal/querylog/decode_test.go index 3848d344..245681dd 100644 --- a/internal/querylog/decode_test.go +++ b/internal/querylog/decode_test.go @@ -36,7 +36,6 @@ func TestDecodeLogEntry(t *testing.T) { `"Result":{` + `"IsFiltered":true,` + `"Reason":3,` + - `"ReverseHosts":["example.net"],` + `"IPList":["127.0.0.2"],` + `"Rules":[{"FilterListID":42,"Text":"||an.yandex.ru","IP":"127.0.0.2"},` + `{"FilterListID":43,"Text":"||an2.yandex.ru","IP":"127.0.0.3"}],` + @@ -58,10 +57,9 @@ func TestDecodeLogEntry(t *testing.T) { ClientProto: "", Answer: ans, Result: filtering.Result{ - IsFiltered: true, - Reason: filtering.FilteredBlockList, - ReverseHosts: []string{"example.net"}, - IPList: []net.IP{net.IPv4(127, 0, 0, 2)}, + IsFiltered: true, + Reason: filtering.FilteredBlockList, + IPList: []net.IP{net.IPv4(127, 0, 0, 2)}, Rules: []*filtering.ResultRule{{ FilterListID: 42, Text: "||an.yandex.ru", @@ -170,8 +168,7 @@ func TestDecodeLogEntry(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - l := &logEntry{} - decodeLogEntry(l, tc.log) + decodeLogEntry(new(logEntry), tc.log) s := logOutput.String() if tc.want == "" { @@ -185,3 +182,65 @@ func TestDecodeLogEntry(t *testing.T) { }) } } + +func TestDecodeLogEntry_backwardCompatability(t *testing.T) { + var ( + a1, a2 = net.IP{127, 0, 0, 1}.To16(), net.IP{127, 0, 0, 2}.To16() + aaaa1, aaaa2 = net.ParseIP("::1"), net.ParseIP("::2") + ) + + testCases := []struct { + want *logEntry + entry string + name string + }{{ + entry: `{"Result":{"ReverseHosts":["example.net","example.org"]}`, + want: &logEntry{ + Result: filtering.Result{DNSRewriteResult: &filtering.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + Response: filtering.DNSRewriteResultResponse{ + dns.TypePTR: []rules.RRValue{"example.net.", "example.org."}, + }, + }}, + }, + name: "reverse_hosts", + }, { + entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":10}}`, + want: &logEntry{ + Result: filtering.Result{ + DNSRewriteResult: &filtering.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + Response: filtering.DNSRewriteResultResponse{ + dns.TypeA: []rules.RRValue{a1, a2}, + dns.TypeAAAA: []rules.RRValue{aaaa1, aaaa2}, + }, + }, + Reason: filtering.RewrittenAutoHosts, + }, + }, + name: "iplist_autohosts", + }, { + entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":9}}`, + want: &logEntry{ + Result: filtering.Result{ + IPList: []net.IP{ + a1, + a2, + aaaa1, + aaaa2, + }, + Reason: filtering.Rewritten, + }, + }, + name: "iplist_rewritten", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + e := &logEntry{} + decodeLogEntry(e, tc.entry) + + assert.Equal(t, tc.want, e) + }) + } +}