Pull request: 3846 hosts querylog

Merge in DNS/adguard-home from 3846-hosts-querylog to master

Updates #3846.

Squashed commit of the following:

commit 722e96628b1ccca1a5b5a716b8bcb1da2aefcc3b
Merge: a20ad71e ed868fa4
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Nov 23 17:52:08 2021 +0300

    Merge branch 'master' into 3846-hosts-querylog

commit a20ad71e723dbfa3483c3bdf9e4c8fd15c8b0e3c
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Nov 23 17:28:12 2021 +0300

    client: fix variable name

commit 7013bff05d6cff75c6c25a38d614db8b4b2f0b87
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Nov 23 17:03:26 2021 +0300

    client: fix missing import

commit 8e4a0fb047b4d39ab44a285f59420573d7ba5eec
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Nov 23 16:56:50 2021 +0300

    client: handle system host filter id

commit abbbf662d2f3ea3f5d3569a9c45418e356adbf3c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Nov 22 13:54:52 2021 +0300

    all: imp code

commit c2df63e46e75f84f70a610d18deccbeee672ebda
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Nov 22 12:50:51 2021 +0300

    querylog: rm unused test data

commit 8a1d47d266254fd4aedd4c61c7ea9e48168ea375
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Nov 22 02:52:50 2021 +0300

    aghnet: final imps

commit ade3acb4bebc8bdd755e56f314cdf19bc9375557
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 19 15:48:40 2021 +0300

    all: add hosts container rule list support

client/src/__locales/en.json

@@ -201,6 +201,7 @@
     "form_error_url_or_path_format": "Invalid URL or absolute path of the list",
     "custom_filter_rules": "Custom filtering rules",
     "custom_filter_rules_hint": "Enter one rule on a line. You can use either adblock rules or hosts files syntax.",
+    "system_host_files": "System hosts files",
     "examples_title": "Examples",
     "example_meaning_filter_block": "block access to the example.org domain and all its subdomains",
     "example_meaning_filter_whitelist": "unblock access to the example.org domain and all its subdomains",

client/src/helpers/constants.js

@@ -529,6 +529,7 @@ export const DETAILED_DATE_FORMAT_OPTIONS = {
 };
 
 export const CUSTOM_FILTERING_RULES_ID = 0;
+export const SYSTEM_HOSTS_FILTER_ID = -1;
 
 export const BLOCK_ACTIONS = {
     BLOCK: 'block',

client/src/helpers/helpers.js

@@ -26,6 +26,7 @@ import {
     STANDARD_DNS_PORT,
     STANDARD_HTTPS_PORT,
     STANDARD_WEB_PORT,
+    SYSTEM_HOSTS_FILTER_ID,
 } from './constants';
 
 /**
@@ -791,9 +792,12 @@ export const getFilterName = (
         return i18n.t(customFilterTranslationKey);
     }
 
+    if (filterId === SYSTEM_HOSTS_FILTER_ID) {
+        return i18n.t('system_host_files');
+    }
+
     const matchIdPredicate = (filter) => filter.id === filterId;
     const filter = filters.find(matchIdPredicate) || whitelistFilters.find(matchIdPredicate);
-
     return resolveFilterName(filter);
 };
 

internal/aghio/limitedreader.go

@@ -35,7 +35,7 @@ func (lr *limitedReader) Read(p []byte) (n int, err error) {
 	}
 
 	if int64(len(p)) > lr.n {
-		p = p[0:lr.n]
+		p = p[:lr.n]
 	}
 
 	n, err = lr.r.Read(p)

internal/aghnet/hostscontainer.go

@@ -28,6 +28,69 @@ func DefaultHostsPaths() (paths []string) {
 	return defaultHostsPaths()
 }
 
+// requestMatcher combines the logic for matching requests and translating the
+// appropriate rules.
+type requestMatcher struct {
+	// stateLock protects all the fields of requestMatcher.
+	stateLock *sync.RWMutex
+
+	// rulesStrg stores the rules obtained from the hosts' file.
+	rulesStrg *filterlist.RuleStorage
+	// engine serves rulesStrg.
+	engine *urlfilter.DNSEngine
+
+	// translator maps generated $dnsrewrite rules into hosts-syntax rules.
+	//
+	// TODO(e.burkov):  Store the filename from which the rule was parsed.
+	translator map[string]string
+}
+
+// MatchRequest processes the request rewriting hostnames and addresses read
+// from the operating system's hosts files.
+//
+// res is nil for any request having not an A/AAAA or PTR type.  Results
+// containing CNAME information may be queried again with the same question type
+// and the returned CNAME for Host field of request.  Results are guaranteed to
+// be direct, i.e. any returned CNAME resolves into actual address like an alias
+// in hosts does, see man hosts (5).
+//
+// It's safe for concurrent use.
+func (rm *requestMatcher) MatchRequest(
+	req urlfilter.DNSRequest,
+) (res *urlfilter.DNSResult, ok bool) {
+	switch req.DNSType {
+	case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
+		log.Debug("%s: handling the request", hostsContainerPref)
+	default:
+		return nil, false
+	}
+
+	rm.stateLock.RLock()
+	defer rm.stateLock.RUnlock()
+
+	return rm.engine.MatchRequest(req)
+}
+
+// Translate returns the source hosts-syntax rule for the generated dnsrewrite
+// rule or an empty string if the last doesn't exist.
+func (rm *requestMatcher) Translate(rule string) (hostRule string) {
+	rm.stateLock.RLock()
+	defer rm.stateLock.RUnlock()
+
+	return rm.translator[rule]
+}
+
+// resetEng updates container's engine and the translation map.
+func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) {
+	rm.stateLock.Lock()
+	defer rm.stateLock.Unlock()
+
+	rm.rulesStrg = rulesStrg
+	rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg)
+
+	rm.translator = tr
+}
+
 // hostsContainerPref is a prefix for logging and wrapping errors in
 // HostsContainer's methods.
 const hostsContainerPref = "hosts container"
@@ -35,13 +98,9 @@ const hostsContainerPref = "hosts container"
 // HostsContainer stores the relevant hosts database provided by the OS and
 // processes both A/AAAA and PTR DNS requests for those.
 type HostsContainer struct {
-	// engLock protects rulesStrg and engine.
+	// requestMatcher matches the requests and translates the rules.  It's
-	engLock *sync.RWMutex
+	// embedded to implement MatchRequest and Translate for *HostsContainer.
-
+	requestMatcher
-	// rulesStrg stores the rules obtained from the hosts' file.
-	rulesStrg *filterlist.RuleStorage
-	// engine serves rulesStrg.
-	engine *urlfilter.DNSEngine
 
 	// done is the channel to sign closing the container.
 	done chan struct{}
@@ -87,7 +146,9 @@ func NewHostsContainer(
 	}
 
 	hc = &HostsContainer{
-		engLock:  &sync.RWMutex{},
+		requestMatcher: requestMatcher{
+			stateLock: &sync.RWMutex{},
+		},
 		done:     make(chan struct{}, 1),
 		updates:  make(chan *netutil.IPMap, 1),
 		fsys:     fsys,
@@ -117,25 +178,6 @@ func NewHostsContainer(
 	return hc, nil
 }
 
-// MatchRequest is the request processing method to resolve hostnames and
-// addresses from the operating system's hosts files.  res is nil for any
-// request having not an A/AAAA or PTR type.  It's safe for concurrent use.
-func (hc *HostsContainer) MatchRequest(
-	req urlfilter.DNSRequest,
-) (res *urlfilter.DNSResult, ok bool) {
-	switch req.DNSType {
-	case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
-		log.Debug("%s: handling the request", hostsContainerPref)
-	default:
-		return nil, false
-	}
-
-	hc.engLock.RLock()
-	defer hc.engLock.RUnlock()
-
-	return hc.engine.MatchRequest(req)
-}
-
 // Close implements the io.Closer interface for *HostsContainer.  Close must
 // only be called once.  The returned err is always nil.
 func (hc *HostsContainer) Close() (err error) {
@@ -203,10 +245,17 @@ func (hc *HostsContainer) handleEvents() {
 }
 
 // hostsParser is a helper type to parse rules from the operating system's hosts
-// file.
+// file.  It exists for only a single refreshing session.
 type hostsParser struct {
-	// rules builds the resulting rules list content.
+	// rulesBuilder builds the resulting rulesBuilder list content.
-	rules *strings.Builder
+	rulesBuilder *strings.Builder
+
+	// translations maps generated $dnsrewrite rules to the hosts-translations
+	// rules.
+	translations map[string]string
+
+	// cnameSet prevents duplicating cname rules.
+	cnameSet *stringutil.Set
 
 	// table stores only the unique IP-hostname pairs.  It's also sent to the
 	// updates channel afterwards.
@@ -215,7 +264,10 @@ type hostsParser struct {
 
 func (hc *HostsContainer) newHostsParser() (hp *hostsParser) {
 	return &hostsParser{
-		rules: &strings.Builder{},
+		rulesBuilder: &strings.Builder{},
+		// For A/AAAA and PTRs.
+		translations: make(map[string]string, hc.last.Len()*2),
+		cnameSet:     stringutil.NewSet(),
 		table:        netutil.NewIPMap(hc.last.Len()),
 	}
 }
@@ -234,9 +286,7 @@ func (hp *hostsParser) parseFile(
 			continue
 		}
 
-		for _, host := range hosts {
+		hp.addPairs(ip, hosts)
-			hp.addPair(ip, host)
-		}
 	}
 
 	return nil, true, s.Err()
@@ -244,7 +294,6 @@ func (hp *hostsParser) parseFile(
 
 // parseLine parses the line having the hosts syntax ignoring invalid ones.
 func (hp *hostsParser) parseLine(line string) (ip net.IP, hosts []string) {
-	line = strings.TrimSpace(line)
 	fields := strings.Fields(line)
 	if len(fields) < 2 {
 		return nil, nil
@@ -274,74 +323,142 @@ loop:
 	return ip, hosts
 }
 
-// add returns true if the pair of ip and host wasn't added to the hp before.
+// Simple types of hosts in hosts database.  Zero value isn't used to be able
-func (hp *hostsParser) add(ip net.IP, host string) (added bool) {
+// quizzaciously emulate nil with 0.
+const (
+	_ = iota
+	hostAlias
+	hostMain
+)
+
+// add tries to add the ip-host pair.  It returns:
+//
+//   hostAlias if the host is not the first one added for the ip.
+//   hostMain  if the host is the first one added for the ip.
+//   0         if the ip-host pair has already been added.
+//
+func (hp *hostsParser) add(ip net.IP, host string) (hostType int) {
 	v, ok := hp.table.Get(ip)
-	hosts, _ := v.(*stringutil.Set)
+	switch hosts, _ := v.(*stringutil.Set); {
-	switch {
 	case ok && hosts.Has(host):
-		return false
+		return 0
 	case hosts == nil:
 		hosts = stringutil.NewSet(host)
 		hp.table.Set(ip, hosts)
+
+		return hostMain
 	default:
 		hosts.Add(host)
+
+		return hostAlias
+	}
 }
 
-	return true
+// addPair puts the pair of ip and host to the rules builder if needed.  For
+// each ip the first member of hosts will become the main one.
+func (hp *hostsParser) addPairs(ip net.IP, hosts []string) {
+	// Put the rule in a preproccesed format like:
+	//
+	//   ip host1 host2 ...
+	//
+	hostsLine := strings.Join(append([]string{ip.String()}, hosts...), " ")
+	var mainHost string
+	for _, host := range hosts {
+		switch hp.add(ip, host) {
+		case 0:
+			continue
+		case hostMain:
+			mainHost = host
+			added, addedPtr := hp.writeMainHostRule(host, ip)
+			hp.translations[added], hp.translations[addedPtr] = hostsLine, hostsLine
+		case hostAlias:
+			pair := fmt.Sprint(host, " ", mainHost)
+			if hp.cnameSet.Has(pair) {
+				continue
+			}
+			// Since the hostAlias couldn't be returned from add before the
+			// hostMain the mainHost shouldn't appear empty.
+			hp.writeAliasHostRule(host, mainHost)
+			hp.cnameSet.Add(pair)
 		}
 
-// addPair puts the pair of ip and host to the rules builder if needed.
+		log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, host)
-func (hp *hostsParser) addPair(ip net.IP, host string) {
+	}
+}
+
+// writeAliasHostRule writes the CNAME rule for the alias-host pair into
+// internal builders.
+func (hp *hostsParser) writeAliasHostRule(alias, host string) {
+	const (
+		nl = "\n"
+		sc = ";"
+
+		rwSuccess = rules.MaskSeparator + "$dnsrewrite=NOERROR" + sc + "CNAME" + sc
+		constLen  = len(rules.MaskStartURL) + len(rwSuccess) + len(nl)
+	)
+
+	hp.rulesBuilder.Grow(constLen + len(host) + len(alias))
+	stringutil.WriteToBuilder(hp.rulesBuilder, rules.MaskStartURL, alias, rwSuccess, host, nl)
+}
+
+// writeMainHostRule writes the actual rule for the qtype and the PTR for the
+// host-ip pair into internal builders.
+func (hp *hostsParser) writeMainHostRule(host string, ip net.IP) (added, addedPtr string) {
 	arpa, err := netutil.IPToReversedAddr(ip)
 	if err != nil {
 		return
 	}
 
-	if !hp.add(ip, host) {
-		return
-	}
-
-	qtype := "AAAA"
-	if ip.To4() != nil {
-		// Assume the validation of the IP address is performed already.
-		qtype = "A"
-	}
-
 	const (
 		nl = "\n"
-		sc = ";"
 
-		rewriteSuccess    = "$dnsrewrite=NOERROR" + sc
+		rwSuccess    = "^$dnsrewrite=NOERROR;"
-		rewriteSuccessPTR = rewriteSuccess + "PTR" + sc
+		rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;"
+
+		modLen    = len("||") + len(rwSuccess)
+		modLenPTR = len("||") + len(rwSuccessPTR)
 	)
 
+	var qtype string
+	// The validation of the IP address has been performed earlier so it is
+	// guaranteed to be either an IPv4 or an IPv6.
+	if ip.To4() != nil {
+		qtype = "A"
+	} else {
+		qtype = "AAAA"
+	}
+
 	ipStr := ip.String()
 	fqdn := dns.Fqdn(host)
 
-	for _, ruleData := range [...][]string{{
+	ruleBuilder := &strings.Builder{}
-		// A/AAAA.
+	ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr))
-		rules.MaskStartURL,
+	stringutil.WriteToBuilder(
+		ruleBuilder,
+		"||",
 		host,
-		rules.MaskSeparator,
+		rwSuccess,
-		rewriteSuccess,
 		qtype,
-		sc,
+		";",
 		ipStr,
-		nl,
+	)
-	}, {
+	added = ruleBuilder.String()
-		// PTR.
-		rules.MaskStartURL,
-		arpa,
-		rules.MaskSeparator,
-		rewriteSuccessPTR,
-		fqdn,
-		nl,
-	}} {
-		stringutil.WriteToBuilder(hp.rules, ruleData...)
-	}
 
-	log.Debug("%s: added ip-host pair %q/%q", hostsContainerPref, ip, host)
+	ruleBuilder.Reset()
+	ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn))
+	stringutil.WriteToBuilder(
+		ruleBuilder,
+		"||",
+		arpa,
+		rwSuccessPTR,
+		fqdn,
+	)
+	addedPtr = ruleBuilder.String()
+
+	hp.rulesBuilder.Grow(len(added) + len(addedPtr) + 2*len(nl))
+	stringutil.WriteToBuilder(hp.rulesBuilder, added, nl, addedPtr, nl)
+
+	return added, addedPtr
 }
 
 // equalSet returns true if the internal hosts table just parsed equals target.
@@ -385,15 +502,16 @@ func (hp *hostsParser) sendUpd(ch chan *netutil.IPMap) {
 	case ch <- upd:
 		// The previous update was just read and the next one pushed.  Go on.
 	default:
-		log.Debug("%s: the channel is broken", hostsContainerPref)
+		log.Error("%s: the updates channel is broken", hostsContainerPref)
 	}
 }
 
 // newStrg creates a new rules storage from parsed data.
 func (hp *hostsParser) newStrg() (s *filterlist.RuleStorage, err error) {
 	return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{
+		// TODO(e.burkov):  Make configurable.
 		ID:             -1,
-		RulesText:      hp.rules.String(),
+		RulesText:      hp.rulesBuilder.String(),
 		IgnoreCosmetic: true,
 	}})
 }
@@ -424,15 +542,7 @@ func (hc *HostsContainer) refresh() (err error) {
 		return fmt.Errorf("initializing rules storage: %w", err)
 	}
 
-	hc.resetEng(rulesStrg)
+	hc.resetEng(rulesStrg, hp.translations)
 
 	return nil
 }
-
-func (hc *HostsContainer) resetEng(rulesStrg *filterlist.RuleStorage) {
-	hc.engLock.Lock()
-	defer hc.engLock.Unlock()
-
-	hc.rulesStrg = rulesStrg
-	hc.engine = urlfilter.NewDNSEngine(hc.rulesStrg)
-}

internal/aghnet/hostscontainer_test.go

@@ -3,6 +3,7 @@ package aghnet
 import (
 	"io/fs"
 	"net"
+	"os"
 	"path"
 	"strings"
 	"sync/atomic"
@@ -11,9 +12,9 @@ import (
 
 	"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/stringutil"
 	"github.com/AdguardTeam/urlfilter"
+	"github.com/AdguardTeam/urlfilter/rules"
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -203,129 +204,6 @@ func TestHostsContainer_Refresh(t *testing.T) {
 	})
 }
 
-func TestHostsContainer_MatchRequest(t *testing.T) {
-	var (
-		ip4 = net.IP{127, 0, 0, 1}
-		ip6 = net.IP{
-			128, 0, 0, 0,
-			0, 0, 0, 0,
-			0, 0, 0, 0,
-			0, 0, 0, 1,
-		}
-
-		hostname4  = "localhost"
-		hostname6  = "localhostv6"
-		hostname4a = "abcd"
-
-		reversed4, _ = netutil.IPToReversedAddr(ip4)
-		reversed6, _ = netutil.IPToReversedAddr(ip6)
-	)
-
-	const filename = "file1"
-
-	gsfs := fstest.MapFS{
-		filename: &fstest.MapFile{Data: []byte(
-			ip4.String() + " " + hostname4 + " " + hostname4a + nl +
-				ip6.String() + " " + hostname6 + nl +
-				`256.256.256.256 fakebroadcast` + nl,
-		)},
-	}
-
-	hc, err := NewHostsContainer(gsfs, &aghtest.FSWatcher{
-		OnEvents: func() (e <-chan struct{}) { panic("not implemented") },
-		OnAdd: func(name string) (err error) {
-			assert.Equal(t, filename, name)
-
-			return nil
-		},
-		OnClose: func() (err error) { panic("not implemented") },
-	}, filename)
-	require.NoError(t, err)
-
-	testCase := []struct {
-		name string
-		want []interface{}
-		req  urlfilter.DNSRequest
-	}{{
-		name: "a",
-		want: []interface{}{ip4.To16()},
-		req: urlfilter.DNSRequest{
-			Hostname: hostname4,
-			DNSType:  dns.TypeA,
-		},
-	}, {
-		name: "a_for_aaaa",
-		want: []interface{}{
-			ip4.To16(),
-		},
-		req: urlfilter.DNSRequest{
-			Hostname: hostname4,
-			DNSType:  dns.TypeAAAA,
-		},
-	}, {
-		name: "aaaa",
-		want: []interface{}{ip6},
-		req: urlfilter.DNSRequest{
-			Hostname: hostname6,
-			DNSType:  dns.TypeAAAA,
-		},
-	}, {
-		name: "ptr",
-		want: []interface{}{
-			dns.Fqdn(hostname4),
-			dns.Fqdn(hostname4a),
-		},
-		req: urlfilter.DNSRequest{
-			Hostname: reversed4,
-			DNSType:  dns.TypePTR,
-		},
-	}, {
-		name: "ptr_v6",
-		want: []interface{}{dns.Fqdn(hostname6)},
-		req: urlfilter.DNSRequest{
-			Hostname: reversed6,
-			DNSType:  dns.TypePTR,
-		},
-	}, {
-		name: "a_alias",
-		want: []interface{}{ip4.To16()},
-		req: urlfilter.DNSRequest{
-			Hostname: hostname4a,
-			DNSType:  dns.TypeA,
-		},
-	}}
-
-	for _, tc := range testCase {
-		t.Run(tc.name, func(t *testing.T) {
-			res, ok := hc.MatchRequest(tc.req)
-			require.False(t, ok)
-			require.NotNil(t, res)
-
-			rws := res.DNSRewrites()
-			require.Len(t, rws, len(tc.want))
-
-			for i, w := range tc.want {
-				require.NotNil(t, rws[i])
-
-				rw := rws[i].DNSRewrite
-				require.NotNil(t, rw)
-
-				assert.Equal(t, w, rw.Value)
-			}
-		})
-	}
-
-	t.Run("cname", func(t *testing.T) {
-		res, ok := hc.MatchRequest(urlfilter.DNSRequest{
-			Hostname: hostname4,
-			DNSType:  dns.TypeCNAME,
-		})
-		require.False(t, ok)
-
-		assert.Nil(t, res)
-	})
-}
-
 func TestHostsContainer_PathsToPatterns(t *testing.T) {
 	const (
 		dir0 = "dir"
@@ -412,53 +290,108 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) {
 	})
 }
 
-func TestUniqueRules_AddPair(t *testing.T) {
+func TestHostsContainer(t *testing.T) {
-	knownIP := net.IP{1, 2, 3, 4}
+	testdata := os.DirFS("./testdata")
 
-	const knownHost = "host1"
+	nRewrites := func(t *testing.T, res *urlfilter.DNSResult, n int) (rws []*rules.DNSRewrite) {
+		t.Helper()
 
-	ipToHost := netutil.NewIPMap(0)
+		rewrites := res.DNSRewrites()
-	ipToHost.Set(knownIP, *stringutil.NewSet(knownHost))
+		assert.Len(t, rewrites, n)
 
-	testCases := []struct {
+		for _, rewrite := range rewrites {
-		name      string
+			rw := rewrite.DNSRewrite
-		host      string
+			require.NotNil(t, rw)
-		wantRules string
-		ip        net.IP
-	}{{
-		name: "new_one",
-		host: "host2",
-		wantRules: "||host2^$dnsrewrite=NOERROR;A;1.2.3.4\n" +
-			"||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host2.\n",
-		ip: knownIP,
-	}, {
-		name: "existing_one",
-		host: knownHost,
-		wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.4\n" +
-			"||4.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;host1.\n",
-		ip: knownIP,
-	}, {
-		name: "new_ip",
-		host: knownHost,
-		wantRules: "||" + knownHost + "^$dnsrewrite=NOERROR;A;1.2.3.5\n" +
-			"||5.3.2.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;" + knownHost + ".\n",
-		ip: net.IP{1, 2, 3, 5},
-	}, {
-		name:      "bad_ip",
-		host:      knownHost,
-		wantRules: "",
-		ip:        net.IP{1, 2, 3, 4, 5},
-	}}
 
-	for _, tc := range testCases {
+			rws = append(rws, rw)
-		hp := hostsParser{
-			rules: &strings.Builder{},
-			table: ipToHost.ShallowClone(),
 		}
 
+		return rws
+	}
+
+	testCases := []struct {
+		testTail func(t *testing.T, res *urlfilter.DNSResult)
+		name     string
+		req      urlfilter.DNSRequest
+	}{{
+		name: "simple",
+		req: urlfilter.DNSRequest{
+			Hostname: "simplehost",
+			DNSType:  dns.TypeA,
+		},
+		testTail: func(t *testing.T, res *urlfilter.DNSResult) {
+			rws := nRewrites(t, res, 2)
+
+			v, ok := rws[0].Value.(net.IP)
+			require.True(t, ok)
+
+			assert.True(t, net.IP{1, 0, 0, 1}.Equal(v))
+
+			v, ok = rws[1].Value.(net.IP)
+			require.True(t, ok)
+
+			// It's ::1.
+			assert.True(t, net.IP(append((&[15]byte{})[:], byte(1))).Equal(v))
+		},
+	}, {
+		name: "hello_alias",
+		req: urlfilter.DNSRequest{
+			Hostname: "hello.world",
+			DNSType:  dns.TypeA,
+		},
+		testTail: func(t *testing.T, res *urlfilter.DNSResult) {
+			assert.Equal(t, "hello", nRewrites(t, res, 1)[0].NewCNAME)
+		},
+	}, {
+		name: "lots_of_aliases",
+		req: urlfilter.DNSRequest{
+			Hostname: "for.testing",
+			DNSType:  dns.TypeA,
+		},
+		testTail: func(t *testing.T, res *urlfilter.DNSResult) {
+			assert.Equal(t, "a.whole", nRewrites(t, res, 1)[0].NewCNAME)
+		},
+	}, {
+		name: "reverse",
+		req: urlfilter.DNSRequest{
+			Hostname: "1.0.0.1.in-addr.arpa",
+			DNSType:  dns.TypePTR,
+		},
+		testTail: func(t *testing.T, res *urlfilter.DNSResult) {
+			rws := nRewrites(t, res, 1)
+
+			assert.Equal(t, dns.TypePTR, rws[0].RRType)
+			assert.Equal(t, "simplehost.", rws[0].Value)
+		},
+	}, {
+		name: "non-existing",
+		req: urlfilter.DNSRequest{
+			Hostname: "nonexisting",
+			DNSType:  dns.TypeA,
+		},
+		testTail: func(t *testing.T, res *urlfilter.DNSResult) {
+			require.NotNil(t, res)
+
+			assert.Nil(t, res.DNSRewrites())
+		},
+	}}
+
+	stubWatcher := aghtest.FSWatcher{
+		OnEvents: func() (e <-chan struct{}) { return nil },
+		OnAdd:    func(name string) (err error) { return nil },
+		OnClose:  func() (err error) { panic("not implemented") },
+	}
+
+	hc, err := NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
+	require.NoError(t, err)
+
+	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			hp.addPair(tc.ip, tc.host)
+			res, ok := hc.MatchRequest(tc.req)
-			assert.Equal(t, tc.wantRules, hp.rules.String())
+			require.False(t, ok)
+			require.NotNil(t, res)
+
+			tc.testTail(t, res)
 		})
 	}
 }

internal/aghnet/testdata/etc_hosts

@@ -0,0 +1,14 @@
+#
+# Test /etc/hosts file
+#
+
+1.0.0.1 simplehost
+1.0.0.0 hello hello.world
+
+# See https://github.com/AdguardTeam/AdGuardHome/issues/3846.
+1.0.0.2 a.whole lot.of aliases for.testing
+
+# Same for IPv6.
+::1 simplehost
+::  hello hello.world
+::2 a.whole lot.of aliases for.testing

internal/filtering/dnsrewrite.go

@@ -27,7 +27,7 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
 	for _, nr := range dnsr {
 		dr := nr.DNSRewrite
 		if dr.NewCNAME != "" {
-			// NewCNAME rules have a higher priority than the other rules.
+			// NewCNAME rules have a higher priority than other rules.
 			rules = []*ResultRule{{
 				FilterListID: int64(nr.GetFilterListID()),
 				Text:         nr.RuleText,

internal/filtering/filtering.go

@@ -376,16 +376,8 @@ type Result struct {
 	// Rules are applied rules.  If Rules are not empty, each rule is not nil.
 	Rules []*ResultRule `json:",omitempty"`
 
-	// ReverseHosts is the reverse lookup rewrite result.  It is empty unless
-	// Reason is set to RewrittenAutoHosts.
-	//
-	// TODO(e.burkov):  There is no need for AutoHosts-related fields any more
-	// since the hosts container now uses $dnsrewrite rules.  These fields are
-	// only used in query log to decode old format.
-	ReverseHosts []string `json:",omitempty"`
-
 	// IPList is the lookup rewrite result.  It is empty unless Reason is set to
-	// RewrittenAutoHosts or Rewritten.
+	// Rewritten.
 	IPList []net.IP `json:",omitempty"`
 
 	// CanonName is the CNAME value from the lookup rewrite result.  It is empty
@@ -464,7 +456,7 @@ func (d *DNSFilter) matchSysHosts(
 		return res, nil
 	}
 
-	dnsres, _ := d.EtcHosts.MatchRequest(urlfilter.DNSRequest{
+	return d.matchSysHostsIntl(&urlfilter.DNSRequest{
 		Hostname:         host,
 		SortedClientTags: setts.ClientTags,
 		// TODO(e.burkov):  Wait for urlfilter update to pass net.IP.
@@ -472,18 +464,34 @@ func (d *DNSFilter) matchSysHosts(
 		ClientName: setts.ClientName,
 		DNSType:    qtype,
 	})
+}
+
+// matchSysHostsIntl actually matches the request.  It's separated to avoid
+// perfoming checks twice.
+func (d *DNSFilter) matchSysHostsIntl(
+	req *urlfilter.DNSRequest,
+) (res Result, err error) {
+	dnsres, _ := d.EtcHosts.MatchRequest(*req)
 	if dnsres == nil {
 		return res, nil
 	}
 
-	if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 {
+	dnsr := dnsres.DNSRewrites()
-		// Check DNS rewrites first, because the API there is a bit awkward.
+	if len(dnsr) == 0 {
+		return res, nil
+	}
+
 	res = d.processDNSRewrites(dnsr)
+	if cn := res.CanonName; cn != "" {
+		// Probably an alias.
+		req.Hostname = cn
+
+		return d.matchSysHostsIntl(req)
+	}
+
 	res.Reason = RewrittenAutoHosts
-		// TODO(e.burkov):  Put real hosts-syntax rules.
+	for _, r := range res.Rules {
-		//
+		r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text)
-		// See https://github.com/AdguardTeam/AdGuardHome/issues/3846.
-		res.Rules = nil
 	}
 
 	return res, nil
@@ -799,7 +807,6 @@ func (d *DNSFilter) matchHost(
 	}
 
 	dnsres, ok := d.filteringEngine.MatchRequest(ureq)
-
 	// Check DNS rewrites first, because the API there is a bit awkward.
 	if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 {
 		res = d.processDNSRewrites(dnsr)

internal/querylog/decode.go

@@ -291,10 +291,13 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) {
 			}
 
 			if d, ok := keyToken.(json.Delim); ok {
-				if d == '}' {
+				switch d {
+				case '}':
 					i++
-				} else if d == ']' {
+				case ']':
 					return
+				default:
+					// Go on.
 				}
 
 				continue
@@ -312,6 +315,11 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) {
 	}
 }
 
+// decodeResultReverseHosts parses the dec's tokens into ent interpreting it as
+// the result of hosts container's $dnsrewrite rule.  It assumes there are no
+// other occurrences of DNSRewriteResult in the entry since hosts container's
+// rewrites currently has the highest priority along the entire filtering
+// pipeline.
 func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) {
 	for {
 		itemToken, err := dec.Token()
@@ -335,7 +343,25 @@ func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) {
 
 			return
 		case string:
-			ent.Result.ReverseHosts = append(ent.Result.ReverseHosts, v)
+			v = dns.Fqdn(v)
+			if res := &ent.Result; res.DNSRewriteResult == nil {
+				res.DNSRewriteResult = &filtering.DNSRewriteResult{
+					RCode: dns.RcodeSuccess,
+					Response: filtering.DNSRewriteResultResponse{
+						dns.TypePTR: []rules.RRValue{v},
+					},
+				}
+
+				continue
+			} else {
+				res.DNSRewriteResult.RCode = dns.RcodeSuccess
+			}
+
+			if rres := ent.Result.DNSRewriteResult; rres.Response == nil {
+				rres.Response = filtering.DNSRewriteResultResponse{dns.TypePTR: []rules.RRValue{v}}
+			} else {
+				rres.Response[dns.TypePTR] = append(rres.Response[dns.TypePTR], v)
+			}
 		default:
 			continue
 		}
@@ -407,9 +433,9 @@ func decodeResultDNSRewriteResultKey(key string, dec *json.Decoder, ent *logEntr
 			ent.Result.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{}
 		}
 
-		// TODO(a.garipov): I give up.  This whole file is a mess.
+		// TODO(a.garipov): I give up.  This whole file is a mess.  Luckily, we
-		// Luckily, we can assume that this field is relatively rare and
+		// can assume that this field is relatively rare and just use the normal
-		// just use the normal decoding and correct the values.
+		// decoding and correct the values.
 		err = dec.Decode(&ent.Result.DNSRewriteResult.Response)
 		if err != nil {
 			log.Debug("decodeResultDNSRewriteResultKey response err: %s", err)
@@ -463,7 +489,40 @@ func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) {
 	}
 }
 
+// translateResult converts some fields of the ent.Result to the format
+// consistent with current implementation.
+func translateResult(ent *logEntry) {
+	res := &ent.Result
+	if res.Reason != filtering.RewrittenAutoHosts || len(res.IPList) == 0 {
+		return
+	}
+
+	if res.DNSRewriteResult == nil {
+		res.DNSRewriteResult = &filtering.DNSRewriteResult{
+			RCode: dns.RcodeSuccess,
+		}
+	}
+
+	if res.DNSRewriteResult.Response == nil {
+		res.DNSRewriteResult.Response = filtering.DNSRewriteResultResponse{}
+	}
+
+	resp := res.DNSRewriteResult.Response
+	for _, ip := range res.IPList {
+		qType := dns.TypeAAAA
+		if ip.To4() != nil {
+			qType = dns.TypeA
+		}
+
+		resp[qType] = append(resp[qType], ip)
+	}
+
+	res.IPList = nil
+}
+
 func decodeResult(dec *json.Decoder, ent *logEntry) {
+	defer translateResult(ent)
+
 	for {
 		keyToken, err := dec.Token()
 		if err != nil {

internal/querylog/decode_test.go

@@ -36,7 +36,6 @@ func TestDecodeLogEntry(t *testing.T) {
 			`"Result":{` +
 			`"IsFiltered":true,` +
 			`"Reason":3,` +
-			`"ReverseHosts":["example.net"],` +
 			`"IPList":["127.0.0.2"],` +
 			`"Rules":[{"FilterListID":42,"Text":"||an.yandex.ru","IP":"127.0.0.2"},` +
 			`{"FilterListID":43,"Text":"||an2.yandex.ru","IP":"127.0.0.3"}],` +
@@ -60,7 +59,6 @@ func TestDecodeLogEntry(t *testing.T) {
 			Result: filtering.Result{
 				IsFiltered: true,
 				Reason:     filtering.FilteredBlockList,
-				ReverseHosts: []string{"example.net"},
 				IPList:     []net.IP{net.IPv4(127, 0, 0, 2)},
 				Rules: []*filtering.ResultRule{{
 					FilterListID: 42,
@@ -170,8 +168,7 @@ func TestDecodeLogEntry(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			l := &logEntry{}
+			decodeLogEntry(new(logEntry), tc.log)
-			decodeLogEntry(l, tc.log)
 
 			s := logOutput.String()
 			if tc.want == "" {
@@ -185,3 +182,65 @@ func TestDecodeLogEntry(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeLogEntry_backwardCompatability(t *testing.T) {
+	var (
+		a1, a2       = net.IP{127, 0, 0, 1}.To16(), net.IP{127, 0, 0, 2}.To16()
+		aaaa1, aaaa2 = net.ParseIP("::1"), net.ParseIP("::2")
+	)
+
+	testCases := []struct {
+		want  *logEntry
+		entry string
+		name  string
+	}{{
+		entry: `{"Result":{"ReverseHosts":["example.net","example.org"]}`,
+		want: &logEntry{
+			Result: filtering.Result{DNSRewriteResult: &filtering.DNSRewriteResult{
+				RCode: dns.RcodeSuccess,
+				Response: filtering.DNSRewriteResultResponse{
+					dns.TypePTR: []rules.RRValue{"example.net.", "example.org."},
+				},
+			}},
+		},
+		name: "reverse_hosts",
+	}, {
+		entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":10}}`,
+		want: &logEntry{
+			Result: filtering.Result{
+				DNSRewriteResult: &filtering.DNSRewriteResult{
+					RCode: dns.RcodeSuccess,
+					Response: filtering.DNSRewriteResultResponse{
+						dns.TypeA:    []rules.RRValue{a1, a2},
+						dns.TypeAAAA: []rules.RRValue{aaaa1, aaaa2},
+					},
+				},
+				Reason: filtering.RewrittenAutoHosts,
+			},
+		},
+		name: "iplist_autohosts",
+	}, {
+		entry: `{"Result":{"IPList":["127.0.0.1","127.0.0.2","::1","::2"],"Reason":9}}`,
+		want: &logEntry{
+			Result: filtering.Result{
+				IPList: []net.IP{
+					a1,
+					a2,
+					aaaa1,
+					aaaa2,
+				},
+				Reason: filtering.Rewritten,
+			},
+		},
+		name: "iplist_rewritten",
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			e := &logEntry{}
+			decodeLogEntry(e, tc.entry)
+
+			assert.Equal(t, tc.want, e)
+		})
+	}
+}