From adb422fedf64a2fc87bd296c9dc3d12c5e3a4d72 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 4 Sep 2019 14:12:00 +0300 Subject: [PATCH] filtering: refactor; change API; add "filters_update_interval" setting + config: "filters_update_interval" * add /control/filtering_info * remove /control/filtering/enable * remove /control/filtering/disable * add /control/filtering_config * remove /control/filtering/status * add /control/filtering/set_url * remove /control/filtering/enable_url * remove /control/filtering/disable_url --- AGHTechDoc.md | 78 ++++++++++++++ dnsfilter/dnsfilter.go | 6 +- dnsforward/dnsforward.go | 6 +- go.mod | 1 - home/config.go | 38 +++---- home/control.go | 13 +-- home/control_filtering.go | 208 +++++++++++++++++++++++--------------- home/control_install.go | 6 +- home/dns.go | 6 ++ home/filter.go | 52 +++++++--- home/filter_test.go | 37 +++++++ home/home.go | 18 +--- querylog/qlog.go | 4 +- querylog/querylog.go | 8 +- 14 files changed, 324 insertions(+), 157 deletions(-) create mode 100644 home/filter_test.go diff --git a/AGHTechDoc.md b/AGHTechDoc.md index cfeddd40..c2206449 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -45,6 +45,11 @@ Contents: * Query logs * API: Set querylog parameters * API: Get querylog parameters +* Filtering + * Filters update mechanism + * API: Get filtering parameters + * API: Set filtering parameters + * API: Set URL parameters ## Relations between subsystems @@ -1019,3 +1024,76 @@ Response: "enabled": true | false "interval": 1 | 7 | 30 | 90 } + + +## Filtering + +### Filters update mechanism + +Filters can be updated either manually by request from UI or automatically. +Auto-update interval can be configured in UI. If it is 0, auto-update is disabled. +When the last modification date of filter files is older than auto-update interval, auto-update procedure is started. +If an enabled filter file doesn't exist, it's downloaded on application startup. This includes the case when installation wizard is completed and there are no filter files yet. +When auto-update time comes, server starts the update procedure by downloading filter files. After new filter files are in place, it restarts DNS filtering module with new rules. +Only filters that are enabled by configuration can be updated. +As a result of the update procedure, all enabled filter files are written to disk, refreshed (their last modification date is equal to the current time) and loaded. + + +### API: Get filtering parameters + +Request: + + GET /control/filtering_info + +Response: + + 200 OK + + { + "enabled": true | false + "interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24 + "filters":[ + { + "id":1 + "enabled":true, + "url":"https://...", + "name":"...", + "rules_count":1234, + "last_updated":"2019-09-04T18:29:30+00:00", + } + ... + ], + "user_rules":["...", ...] + } + + +### API: Set filtering parameters + +Request: + + POST /control/filtering_config + + { + "enabled": true | false + "interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24 + } + +Response: + + 200 OK + + +### API: Set URL parameters + +Request: + + POST /control/filtering/set_url + + { + "url": "..." + "enabled": true | false + } + +Response: + + 200 OK diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 6b168a9c..14c12844 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -114,9 +114,9 @@ type Dnsfilter struct { // Filter represents a filter list type Filter struct { - ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase - Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n' - FilePath string `json:"-" yaml:"-"` // Path to a filtering rules file + ID int64 // auto-assigned when filter is added (see nextFilterID) + Data []byte `yaml:"-"` // List of rules divided by '\n' + FilePath string `yaml:"-"` // Path to a filtering rules file } //go:generate stringer -type=Reason diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index ee1c1e0d..deb5892c 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -66,8 +66,10 @@ func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server { // FilteringConfig represents the DNS filtering configuration of AdGuard Home // The zero FilteringConfig is empty and ready for use. type FilteringConfig struct { - ProtectionEnabled bool `yaml:"protection_enabled"` // whether or not use any of dnsfilter features - FilteringEnabled bool `yaml:"filtering_enabled"` // whether or not use filter lists + ProtectionEnabled bool `yaml:"protection_enabled"` // whether or not use any of dnsfilter features + FilteringEnabled bool `yaml:"filtering_enabled"` // whether or not use filter lists + FiltersUpdateIntervalHours uint32 `yaml:"filters_update_interval"` // time period to update filters (in hours) + BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600) QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled diff --git a/go.mod b/go.mod index 8949d9b8..66751c7f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/AdguardTeam/golibs v0.2.1 github.com/AdguardTeam/urlfilter v0.5.0 github.com/NYTimes/gziphandler v1.1.1 - github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 github.com/etcd-io/bbolt v1.3.3 github.com/go-test/deep v1.0.1 diff --git a/home/config.go b/home/config.go index 9323b1ce..9062d148 100644 --- a/home/config.go +++ b/home/config.go @@ -10,7 +10,6 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/dhcpd" - "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/dnsforward" "github.com/AdguardTeam/AdGuardHome/querylog" "github.com/AdguardTeam/AdGuardHome/stats" @@ -72,6 +71,7 @@ type configuration struct { client *http.Client stats stats.Stats queryLog querylog.QueryLog + filteringStarted bool // cached version.json to avoid hammering github.io for each page reload versionCheckJSON []byte @@ -172,16 +172,17 @@ var config = configuration{ Port: 53, StatsInterval: 1, FilteringConfig: dnsforward.FilteringConfig{ - ProtectionEnabled: true, // whether or not use any of dnsfilter features - FilteringEnabled: true, // whether or not use filter lists - BlockingMode: "nxdomain", // mode how to answer filtered requests - BlockedResponseTTL: 10, // in seconds - QueryLogEnabled: true, - QueryLogInterval: 1, - Ratelimit: 20, - RefuseAny: true, - BootstrapDNS: defaultBootstrap, - AllServers: false, + ProtectionEnabled: true, // whether or not use any of dnsfilter features + FilteringEnabled: true, // whether or not use filter lists + FiltersUpdateIntervalHours: 24, + BlockingMode: "nxdomain", // mode how to answer filtered requests + BlockedResponseTTL: 10, // in seconds + QueryLogEnabled: true, + QueryLogInterval: 1, + Ratelimit: 20, + RefuseAny: true, + BootstrapDNS: defaultBootstrap, + AllServers: false, }, UpstreamDNS: defaultDNS, }, @@ -191,12 +192,6 @@ var config = configuration{ PortDNSOverTLS: 853, // needs to be passed through to dnsproxy }, }, - Filters: []filter{ - {Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"}, - {Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"}, - {Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"}, - {Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"}, - }, DHCP: dhcpd.ServerConfig{ LeaseDuration: 86400, ICMPTimeout: 1000, @@ -226,6 +221,7 @@ func initConfig() { config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024 config.DNS.ParentalCacheSize = 1 * 1024 * 1024 config.DNS.CacheTime = 30 + config.Filters = defaultFilters() } // getConfigFilename returns path to the current config file @@ -276,6 +272,9 @@ func parseConfig() error { if !checkStatsInterval(config.DNS.StatsInterval) { config.DNS.StatsInterval = 1 } + if !checkFiltersUpdateIntervalHours(config.DNS.FiltersUpdateIntervalHours) { + config.DNS.FiltersUpdateIntervalHours = 24 + } if !checkQueryLogInterval(config.DNS.QueryLogInterval) { config.DNS.QueryLogInterval = 1 @@ -308,11 +307,6 @@ func parseConfig() error { return err } - // Deduplicate filters - deduplicateFilters() - - updateUniqueFilterID(config.Filters) - return nil } diff --git a/home/control.go b/home/control.go index 27187626..8fb2d15f 100644 --- a/home/control.go +++ b/home/control.go @@ -7,7 +7,6 @@ import ( "net/http" "strconv" "strings" - "time" "github.com/AdguardTeam/AdGuardHome/dnsforward" "github.com/AdguardTeam/dnsproxy/upstream" @@ -17,8 +16,6 @@ import ( "github.com/miekg/dns" ) -const updatePeriod = time.Hour * 24 - var protocols = []string{"tls://", "https://", "tcp://", "sdns://"} // ---------------- @@ -547,15 +544,6 @@ func registerControlHandlers() { httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage) http.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON))) httpRegister(http.MethodPost, "/control/update", handleUpdate) - httpRegister(http.MethodPost, "/control/filtering/enable", handleFilteringEnable) - httpRegister(http.MethodPost, "/control/filtering/disable", handleFilteringDisable) - httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL) - httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL) - httpRegister(http.MethodPost, "/control/filtering/enable_url", handleFilteringEnableURL) - httpRegister(http.MethodPost, "/control/filtering/disable_url", handleFilteringDisableURL) - httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh) - httpRegister(http.MethodGet, "/control/filtering/status", handleFilteringStatus) - httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules) httpRegister(http.MethodPost, "/control/safebrowsing/enable", handleSafeBrowsingEnable) httpRegister(http.MethodPost, "/control/safebrowsing/disable", handleSafeBrowsingDisable) httpRegister(http.MethodGet, "/control/safebrowsing/status", handleSafeBrowsingStatus) @@ -575,6 +563,7 @@ func registerControlHandlers() { httpRegister(http.MethodGet, "/control/access/list", handleAccessList) httpRegister(http.MethodPost, "/control/access/set", handleAccessSet) + RegisterFilteringHandlers() RegisterTLSHandlers() RegisterClientsHandlers() registerRewritesHandlers() diff --git a/home/control_filtering.go b/home/control_filtering.go index 8eded166..03699953 100644 --- a/home/control_filtering.go +++ b/home/control_filtering.go @@ -5,74 +5,57 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "strings" + "time" "github.com/AdguardTeam/golibs/log" - "github.com/asaskevich/govalidator" ) -func handleFilteringEnable(w http.ResponseWriter, r *http.Request) { - config.DNS.FilteringEnabled = true - httpUpdateConfigReloadDNSReturnOK(w, r) +// IsValidURL - return TRUE if URL is valid +func IsValidURL(rawurl string) bool { + url, err := url.ParseRequestURI(rawurl) + if err != nil { + return false //Couldn't even parse the rawurl + } + if len(url.Scheme) == 0 { + return false //No Scheme found + } + return true } -func handleFilteringDisable(w http.ResponseWriter, r *http.Request) { - config.DNS.FilteringEnabled = false - httpUpdateConfigReloadDNSReturnOK(w, r) -} - -func handleFilteringStatus(w http.ResponseWriter, r *http.Request) { - data := map[string]interface{}{ - "enabled": config.DNS.FilteringEnabled, - } - - config.RLock() - data["filters"] = config.Filters - data["user_rules"] = config.UserRules - jsonVal, err := json.Marshal(data) - config.RUnlock() - - if err != nil { - httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err) - return - } - - w.Header().Set("Content-Type", "application/json") - _, err = w.Write(jsonVal) - if err != nil { - httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err) - return - } +type filterAddJSON struct { + Name string `json:"name"` + URL string `json:"url"` } func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) { - f := filter{} - err := json.NewDecoder(r.Body).Decode(&f) + fj := filterAddJSON{} + err := json.NewDecoder(r.Body).Decode(&fj) if err != nil { httpError(w, http.StatusBadRequest, "Failed to parse request body json: %s", err) return } - if len(f.URL) == 0 { - http.Error(w, "URL parameter was not specified", http.StatusBadRequest) - return - } - - if valid := govalidator.IsRequestURL(f.URL); !valid { - http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest) + if !IsValidURL(fj.URL) { + http.Error(w, "Invalid URL", http.StatusBadRequest) return } // Check for duplicates - if filterExists(f.URL) { - httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", f.URL) + if filterExists(fj.URL) { + httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", fj.URL) return } // Set necessary properties + f := filter{ + Enabled: true, + URL: fj.URL, + Name: fj.Name, + } f.ID = assignUniqueFilterID() - f.Enabled = true // Download the filter contents ok, err := f.update() @@ -133,7 +116,7 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) { return } - if valid := govalidator.IsRequestURL(req.URL); !valid { + if !IsValidURL(req.URL) { http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest) return } @@ -166,54 +149,27 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) { httpUpdateConfigReloadDNSReturnOK(w, r) } -func handleFilteringEnableURL(w http.ResponseWriter, r *http.Request) { - parameters, err := parseParametersFromBody(r.Body) - if err != nil { - httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err) - return - } - - url, ok := parameters["url"] - if !ok { - http.Error(w, "URL parameter was not specified", http.StatusBadRequest) - return - } - - if valid := govalidator.IsRequestURL(url); !valid { - http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest) - return - } - - found := filterEnable(url, true) - if !found { - http.Error(w, "URL parameter was not previously added", http.StatusBadRequest) - return - } - - httpUpdateConfigReloadDNSReturnOK(w, r) +type filterURLJSON struct { + URL string `json:"url"` + Enabled bool `json:"enabled"` } -func handleFilteringDisableURL(w http.ResponseWriter, r *http.Request) { - parameters, err := parseParametersFromBody(r.Body) +func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) { + fj := filterURLJSON{} + err := json.NewDecoder(r.Body).Decode(&fj) if err != nil { - httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err) + httpError(w, http.StatusBadRequest, "json decode: %s", err) return } - url, ok := parameters["url"] - if !ok { - http.Error(w, "URL parameter was not specified", http.StatusBadRequest) + if !IsValidURL(fj.URL) { + http.Error(w, "invalid URL", http.StatusBadRequest) return } - if valid := govalidator.IsRequestURL(url); !valid { - http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest) - return - } - - found := filterEnable(url, false) + found := filterEnable(fj.URL, fj.Enabled) if !found { - http.Error(w, "URL parameter was not previously added", http.StatusBadRequest) + http.Error(w, "URL doesn't exist", http.StatusBadRequest) return } @@ -235,3 +191,91 @@ func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) { updated := refreshFiltersIfNecessary(true) fmt.Fprintf(w, "OK %d filters updated\n", updated) } + +type filterJSON struct { + ID int64 `json:"id"` + Enabled bool `json:"enabled"` + URL string `json:"url"` + Name string `json:"name"` + RulesCount uint32 `json:"rules_count"` + LastUpdated string `json:"last_updated"` +} + +type filteringConfig struct { + Enabled bool `json:"enabled"` + Interval uint32 `json:"interval"` // in hours + Filters []filterJSON `json:"filters"` + UserRules []string `json:"user_rules"` +} + +// Get filtering configuration +func handleFilteringInfo(w http.ResponseWriter, r *http.Request) { + resp := filteringConfig{} + config.RLock() + resp.Enabled = config.DNS.FilteringEnabled + resp.Interval = config.DNS.FiltersUpdateIntervalHours + for _, f := range config.Filters { + fj := filterJSON{ + ID: f.ID, + Enabled: f.Enabled, + URL: f.URL, + Name: f.Name, + RulesCount: uint32(f.RulesCount), + } + + if f.LastUpdated.Second() != 0 { + fj.LastUpdated = f.LastUpdated.Format(time.RFC3339) + } + + resp.Filters = append(resp.Filters, fj) + } + resp.UserRules = config.UserRules + config.RUnlock() + + jsonVal, err := json.Marshal(resp) + if err != nil { + httpError(w, http.StatusInternalServerError, "json encode: %s", err) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(jsonVal) + if err != nil { + httpError(w, http.StatusInternalServerError, "http write: %s", err) + } +} + +// Set filtering configuration +func handleFilteringConfig(w http.ResponseWriter, r *http.Request) { + req := filteringConfig{} + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + httpError(w, http.StatusBadRequest, "json decode: %s", err) + return + } + + if !checkFiltersUpdateIntervalHours(req.Interval) { + httpError(w, http.StatusBadRequest, "Unsupported interval") + return + } + + config.DNS.FilteringEnabled = req.Enabled + config.DNS.FiltersUpdateIntervalHours = req.Interval + httpUpdateConfigReloadDNSReturnOK(w, r) + + returnOK(w) +} + +// RegisterFilteringHandlers - register handlers +func RegisterFilteringHandlers() { + httpRegister(http.MethodGet, "/control/filtering_info", handleFilteringInfo) + httpRegister(http.MethodPost, "/control/filtering_config", handleFilteringConfig) + httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL) + httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL) + httpRegister(http.MethodPost, "/control/filtering/set_url", handleFilteringSetURL) + httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh) + httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules) +} + +func checkFiltersUpdateIntervalHours(i uint32) bool { + return i == 0 || i == 1 || i == 12 || i == 1*24 || i == 3*24 || i == 7*24 +} diff --git a/home/control_install.go b/home/control_install.go index e1193f6b..9eb47bfb 100644 --- a/home/control_install.go +++ b/home/control_install.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "os/exec" + "path/filepath" "strconv" "github.com/AdguardTeam/golibs/log" @@ -239,6 +240,9 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) { config.AuthName = newSettings.Username config.AuthPass = newSettings.Password + dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir) + initDNSServer(dnsBaseDir) + err = startDNSServer() if err != nil { config.firstRun = true @@ -255,8 +259,6 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) { return } - go refreshFiltersIfNecessary(false) - // this needs to be done in a goroutine because Shutdown() is a blocking call, and it will block // until all requests are finished, and _we_ are inside a request right now, so it will block indefinitely if restartHTTP { diff --git a/home/dns.go b/home/dns.go index 53958bd5..a016d302 100644 --- a/home/dns.go +++ b/home/dns.go @@ -49,6 +49,7 @@ func initDNSServer(baseDir string) { config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog) initRDNS() + initFiltering() } func isRunning() bool { @@ -165,6 +166,11 @@ func startDNSServer() error { return errorx.Decorate(err, "Couldn't start forwarding DNS server") } + if !config.filteringStarted { + config.filteringStarted = true + startRefreshFilters() + } + return nil } diff --git a/home/filter.go b/home/filter.go index a3b6003b..73c4d9a0 100644 --- a/home/filter.go +++ b/home/filter.go @@ -21,13 +21,35 @@ var ( filterTitleRegexp = regexp.MustCompile(`^! Title: +(.*)$`) ) +func initFiltering() { + loadFilters() + deduplicateFilters() + updateUniqueFilterID(config.Filters) +} + +func startRefreshFilters() { + go func() { + _ = refreshFiltersIfNecessary(false) + }() + go periodicallyRefreshFilters() +} + +func defaultFilters() []filter { + return []filter{ + {Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"}, + {Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"}, + {Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"}, + {Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"}, + } +} + // field ordering is important -- yaml fields will mirror ordering from here type filter struct { - Enabled bool `json:"enabled"` - URL string `json:"url"` - Name string `json:"name" yaml:"name"` - RulesCount int `json:"rulesCount" yaml:"-"` - LastUpdated time.Time `json:"lastUpdated,omitempty" yaml:"-"` + Enabled bool + URL string + Name string `yaml:"name"` + RulesCount int `yaml:"-"` + LastUpdated time.Time `yaml:"-"` checksum uint32 // checksum of the file data dnsfilter.Filter `yaml:",inline"` @@ -119,8 +141,7 @@ func loadFilters() { err := filter.load() if err != nil { - // This is okay for the first start, the filter will be loaded later - log.Debug("Couldn't load filter %d contents due to %s", filter.ID, err) + log.Error("Couldn't load filter %d contents due to %s", filter.ID, err) } } } @@ -159,7 +180,12 @@ func assignUniqueFilterID() int64 { // Sets up a timer that will be checking for filters updates periodically func periodicallyRefreshFilters() { - for range time.Tick(time.Minute) { + for { + time.Sleep(1 * time.Hour) + if config.DNS.FiltersUpdateIntervalHours == 0 { + continue + } + refreshFiltersIfNecessary(false) } } @@ -180,10 +206,7 @@ func refreshFiltersIfNecessary(force bool) int { var updateFilters []filter var updateFlags []bool // 'true' if filter data has changed - if config.firstRun { - return 0 - } - + now := time.Now() config.RLock() for i := range config.Filters { f := &config.Filters[i] // otherwise we will be operating on a copy @@ -192,7 +215,8 @@ func refreshFiltersIfNecessary(force bool) int { continue } - if !force && time.Since(f.LastUpdated) <= updatePeriod { + expireTime := f.LastUpdated.Unix() + int64(config.DNS.FiltersUpdateIntervalHours)*60*60 + if !force && expireTime > now.Unix() { continue } @@ -214,7 +238,7 @@ func refreshFiltersIfNecessary(force bool) int { log.Printf("Failed to update filter %s: %s\n", uf.URL, err) continue } - uf.LastUpdated = time.Now() + uf.LastUpdated = now if updated { updateCount++ } diff --git a/home/filter_test.go b/home/filter_test.go new file mode 100644 index 00000000..63736c38 --- /dev/null +++ b/home/filter_test.go @@ -0,0 +1,37 @@ +package home + +import ( + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFilters(t *testing.T) { + config.client = &http.Client{ + Timeout: time.Minute * 5, + } + + f := filter{ + URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", + } + + // download + ok, err := f.update() + assert.True(t, ok && err == nil) + + // refresh + ok, err = f.update() + assert.True(t, !ok && err == nil) + + err = f.save() + assert.True(t, err == nil) + + err = f.load() + assert.True(t, err == nil) + + f.unload() + os.Remove(f.Path()) +} diff --git a/home/home.go b/home/home.go index 0ae64459..5f2d4fc0 100644 --- a/home/home.go +++ b/home/home.go @@ -135,22 +135,17 @@ func run(args options) { config.BindPort = args.bindPort } - loadFilters() - if !config.firstRun { // Save the updated config err := config.write() if err != nil { log.Fatal(err) } - } - // Init the DNS server instance before registering HTTP handlers - dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir) - initDNSServer(dnsBaseDir) + dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir) + initDNSServer(dnsBaseDir) - if !config.firstRun { - err := startDNSServer() + err = startDNSServer() if err != nil { log.Fatal(err) } @@ -165,13 +160,6 @@ func run(args options) { config.pidFileName = args.pidFile } - // Update filters we've just loaded right away, don't wait for periodic update timer - go func() { - refreshFiltersIfNecessary(false) - }() - // Schedule automatic filters updates - go periodicallyRefreshFilters() - // Initialize and run the admin Web interface box := packr.NewBox("../build/static") diff --git a/querylog/qlog.go b/querylog/qlog.go index cfc0604c..690f4e21 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -35,7 +35,7 @@ type queryLog struct { lock sync.RWMutex } -// newQueryLog creates a new instance of the query log +// create a new instance of the query log func newQueryLog(conf Config) *queryLog { l := queryLog{} l.logFile = filepath.Join(conf.BaseDir, queryLogFileName) @@ -53,7 +53,6 @@ func (l *queryLog) Configure(conf Config) { l.conf = conf } -// Clear memory buffer and remove the file func (l *queryLog) Clear() { l.fileFlushLock.Lock() defer l.fileFlushLock.Unlock() @@ -164,7 +163,6 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res } } -// getQueryLogJson returns a map with the current query log ready to be converted to a JSON func (l *queryLog) GetData() []map[string]interface{} { l.lock.RLock() values := make([]*logEntry, len(l.cache)) diff --git a/querylog/querylog.go b/querylog/querylog.go index c995183b..3a18eb53 100644 --- a/querylog/querylog.go +++ b/querylog/querylog.go @@ -10,14 +10,20 @@ import ( // QueryLog - main interface type QueryLog interface { + // Close query log object Close() // Set new configuration at runtime // Currently only 'Interval' field is supported. Configure(conf Config) + // Add a log entry Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) + + // Get log entries GetData() []map[string]interface{} + + // Clear memory buffer and remove log files Clear() } @@ -27,7 +33,7 @@ type Config struct { Interval uint32 // interval to rotate logs (in hours) } -// New - create instance +// New - create a new instance of the query log func New(conf Config) QueryLog { return newQueryLog(conf) }