* use urlfilter v0.4.0
Now we pass filtering rules to urlfilter as filer file names, rather than the list of rule strings. (Note: user rules are still passed as the list of rule strings). As a result, we don't store the contents of filter files in memory.
This commit is contained in:
Simon Zolin
parent
ceac4cbdd5
commit
134d9275bb
|
|
@ -48,7 +48,6 @@ type RequestFilteringSettings struct {
|
|||
|
||||
// Config allows you to configure DNS filtering with New() or just change variables directly.
|
||||
type Config struct {
|
||||
FilteringTempFilename string `yaml:"filtering_temp_filename"` // temporary file for storing unused filtering rules
|
||||
ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17
|
||||
ParentalEnabled bool `yaml:"parental_enabled"`
|
||||
UsePlainHTTP bool `yaml:"-"` // use plain HTTP for requests to parental and safe browsing servers
|
||||
|
|
@ -82,7 +81,7 @@ type Stats struct {
|
|||
|
||||
// Dnsfilter holds added rules and performs hostname matches against the rules
|
||||
type Dnsfilter struct {
|
||||
rulesStorage *urlfilter.RulesStorage
|
||||
rulesStorage *urlfilter.RuleStorage
|
||||
filteringEngine *urlfilter.DNSEngine
|
||||
|
||||
// HTTP lookups for safebrowsing and parental
|
||||
|
|
@ -97,6 +96,7 @@ type Dnsfilter struct {
|
|||
type Filter struct {
|
||||
ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
|
||||
Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n'
|
||||
FilePath string `json:"-" yaml:"-"` // Path to a filtering rules file
|
||||
}
|
||||
|
||||
//go:generate stringer -type=Reason
|
||||
|
|
@ -527,13 +527,31 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc
|
|||
|
||||
// Initialize urlfilter objects
|
||||
func (d *Dnsfilter) initFiltering(filters map[int]string) error {
|
||||
listArray := []urlfilter.RuleList{}
|
||||
for id, dataOrFilePath := range filters {
|
||||
var list urlfilter.RuleList
|
||||
if id == 0 {
|
||||
list = &urlfilter.StringRuleList{
|
||||
ID: 0,
|
||||
RulesText: dataOrFilePath,
|
||||
IgnoreCosmetic: false,
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
d.rulesStorage, err = urlfilter.NewRuleStorage(d.FilteringTempFilename)
|
||||
list, err = urlfilter.NewFileRuleList(id, dataOrFilePath, false)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("urlfilter.NewFileRuleList(): %s: %s", dataOrFilePath, err)
|
||||
}
|
||||
}
|
||||
listArray = append(listArray, list)
|
||||
}
|
||||
|
||||
d.filteringEngine = urlfilter.NewDNSEngine(filters, d.rulesStorage)
|
||||
var err error
|
||||
d.rulesStorage, err = urlfilter.NewRuleStorage(listArray)
|
||||
if err != nil {
|
||||
return fmt.Errorf("urlfilter.NewRuleStorage(): %s", err)
|
||||
}
|
||||
d.filteringEngine = urlfilter.NewDNSEngine(d.rulesStorage)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -260,7 +260,11 @@ func (s *Server) initDNSFilter() error {
|
|||
if s.conf.FilteringEnabled {
|
||||
filters = make(map[int]string)
|
||||
for _, f := range s.conf.Filters {
|
||||
if f.ID == 0 {
|
||||
filters[int(f.ID)] = string(f.Data)
|
||||
} else {
|
||||
filters[int(f.ID)] = f.FilePath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -5,12 +5,12 @@ go 1.12
|
|||
require (
|
||||
github.com/AdguardTeam/dnsproxy v0.15.0
|
||||
github.com/AdguardTeam/golibs v0.1.3
|
||||
github.com/AdguardTeam/urlfilter v0.3.0
|
||||
github.com/AdguardTeam/urlfilter v0.4.0
|
||||
github.com/NYTimes/gziphandler v1.1.1
|
||||
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0
|
||||
github.com/go-test/deep v1.0.1
|
||||
github.com/gobuffalo/packr v1.19.0
|
||||
github.com/joomcode/errorx v0.1.0
|
||||
github.com/joomcode/errorx v0.8.0
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
|
||||
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b
|
||||
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -5,6 +5,8 @@ github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z1
|
|||
github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
|
||||
github.com/AdguardTeam/urlfilter v0.3.0 h1:WNd3uZEYWwxylUuA8QS6V5DqHNsVFw3ZD/E2rd5HGpo=
|
||||
github.com/AdguardTeam/urlfilter v0.3.0/go.mod h1:9xfZ6R2vB8LlT8G9LxtbNhDsbr/xybUOSwmJvpXhl/c=
|
||||
github.com/AdguardTeam/urlfilter v0.4.0 h1:s4EFwI4+gzBdnATNKbuOY53wS2PCHgFZBv1Ixxva6tg=
|
||||
github.com/AdguardTeam/urlfilter v0.4.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg=
|
||||
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
|
||||
|
|
@ -47,6 +49,8 @@ github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
|
|||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||
github.com/joomcode/errorx v0.1.0 h1:QmJMiI1DE1UFje2aI1ZWO/VMT5a32qBoXUclGOt8vsc=
|
||||
github.com/joomcode/errorx v0.1.0/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ=
|
||||
github.com/joomcode/errorx v0.8.0 h1:GhAqPtcYuo1O7TOIbtzEIDzPGQ3SrKJ3tdjXNmUtDNo=
|
||||
github.com/joomcode/errorx v0.8.0/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b h1:vfiqKno48aUndBMjTeWFpCExNnTf2Xnd6d228L4EfTQ=
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ func generateServerConfig() dnsforward.ServerConfig {
|
|||
for _, filter := range config.Filters {
|
||||
filters = append(filters, dnsfilter.Filter{
|
||||
ID: filter.ID,
|
||||
Data: filter.Data,
|
||||
FilePath: filter.Path(),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ func refreshFiltersIfNecessary(force bool) int {
|
|||
log.Info("Updated filter #%d. Rules: %d -> %d",
|
||||
f.ID, f.RulesCount, uf.RulesCount)
|
||||
f.Name = uf.Name
|
||||
f.Data = uf.Data
|
||||
f.Data = nil
|
||||
f.RulesCount = uf.RulesCount
|
||||
f.checksum = uf.checksum
|
||||
updateCount++
|
||||
|
|
@ -339,6 +339,9 @@ func (filter *filter) update() (bool, error) {
|
|||
}
|
||||
|
||||
// saves filter contents to the file in dataDir
|
||||
// This method is safe to call during filters update,
|
||||
// because it creates a new file and then renames it,
|
||||
// so the currently opened file descriptors to the old filter file remain valid.
|
||||
func (filter *filter) save() error {
|
||||
filterFilePath := filter.Path()
|
||||
log.Printf("Saving filter %d contents to: %s", filter.ID, filterFilePath)
|
||||
|
|
@ -369,7 +372,7 @@ func (filter *filter) load() error {
|
|||
rulesCount, _ := parseFilterContents(filterFileContents)
|
||||
|
||||
filter.RulesCount = rulesCount
|
||||
filter.Data = filterFileContents
|
||||
filter.Data = nil
|
||||
filter.checksum = crc32.ChecksumIEEE(filterFileContents)
|
||||
filter.LastUpdated = filter.LastTimeUpdated()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue