filtering: refactor; change API; add "filters_update_interval" setting

+ config: "filters_update_interval"
* add /control/filtering_info
* remove /control/filtering/enable
* remove /control/filtering/disable

* add /control/filtering_config
* remove /control/filtering/status

* add /control/filtering/set_url
* remove /control/filtering/enable_url
* remove /control/filtering/disable_url

AGHTechDoc.md

@@ -45,6 +45,11 @@ Contents:
 * Query logs
 	* API: Set querylog parameters
 	* API: Get querylog parameters
+* Filtering
+	* Filters update mechanism
+	* API: Get filtering parameters
+	* API: Set filtering parameters
+	* API: Set URL parameters
 
 
 ## Relations between subsystems
@@ -1019,3 +1024,76 @@ Response:
 		"enabled": true | false
 		"interval": 1 | 7 | 30 | 90
 	}
+
+
+## Filtering
+
+### Filters update mechanism
+
+Filters can be updated either manually by request from UI or automatically.
+Auto-update interval can be configured in UI.  If it is 0, auto-update is disabled.
+When the last modification date of filter files is older than auto-update interval, auto-update procedure is started.
+If an enabled filter file doesn't exist, it's downloaded on application startup.  This includes the case when installation wizard is completed and there are no filter files yet.
+When auto-update time comes, server starts the update procedure by downloading filter files.  After new filter files are in place, it restarts DNS filtering module with new rules.
+Only filters that are enabled by configuration can be updated.
+As a result of the update procedure, all enabled filter files are written to disk, refreshed (their last modification date is equal to the current time) and loaded.
+
+
+### API: Get filtering parameters
+
+Request:
+
+	GET /control/filtering_info
+
+Response:
+
+	200 OK
+
+	{
+		"enabled": true | false
+		"interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24
+		"filters":[
+			{
+			"id":1
+			"enabled":true,
+			"url":"https://...",
+			"name":"...",
+			"rules_count":1234,
+			"last_updated":"2019-09-04T18:29:30+00:00",
+			}
+			...
+		],
+		"user_rules":["...", ...]
+	}
+
+
+### API: Set filtering parameters
+
+Request:
+
+	POST /control/filtering_config
+
+	{
+		"enabled": true | false
+		"interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24
+	}
+
+Response:
+
+	200 OK
+
+
+### API: Set URL parameters
+
+Request:
+
+	POST /control/filtering/set_url
+
+	{
+		"url": "..."
+		"enabled": true | false
+	}
+
+Response:
+
+	200 OK

dnsfilter/dnsfilter.go

@@ -114,9 +114,9 @@ type Dnsfilter struct {
 
 // Filter represents a filter list
 type Filter struct {
-	ID       int64  `json:"id"`         // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
+	ID       int64  // auto-assigned when filter is added (see nextFilterID)
-	Data     []byte `json:"-" yaml:"-"` // List of rules divided by '\n'
+	Data     []byte `yaml:"-"` // List of rules divided by '\n'
-	FilePath string `json:"-" yaml:"-"` // Path to a filtering rules file
+	FilePath string `yaml:"-"` // Path to a filtering rules file
 }
 
 //go:generate stringer -type=Reason

dnsforward/dnsforward.go

@@ -68,6 +68,8 @@ func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
 type FilteringConfig struct {
 	ProtectionEnabled          bool   `yaml:"protection_enabled"`      // whether or not use any of dnsfilter features
 	FilteringEnabled           bool   `yaml:"filtering_enabled"`       // whether or not use filter lists
+	FiltersUpdateIntervalHours uint32 `yaml:"filters_update_interval"` // time period to update filters (in hours)
+
 	BlockingMode       string   `yaml:"blocking_mode"`        // mode how to answer filtered requests
 	BlockedResponseTTL uint32   `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
 	QueryLogEnabled    bool     `yaml:"querylog_enabled"`     // if true, query log is enabled

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/AdguardTeam/golibs v0.2.1
 	github.com/AdguardTeam/urlfilter v0.5.0
 	github.com/NYTimes/gziphandler v1.1.1
-	github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
 	github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833
 	github.com/etcd-io/bbolt v1.3.3
 	github.com/go-test/deep v1.0.1

home/config.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/dhcpd"
-	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
 	"github.com/AdguardTeam/AdGuardHome/querylog"
 	"github.com/AdguardTeam/AdGuardHome/stats"
@@ -72,6 +71,7 @@ type configuration struct {
 	client           *http.Client
 	stats            stats.Stats
 	queryLog         querylog.QueryLog
+	filteringStarted bool
 
 	// cached version.json to avoid hammering github.io for each page reload
 	versionCheckJSON     []byte
@@ -174,6 +174,7 @@ var config = configuration{
 		FilteringConfig: dnsforward.FilteringConfig{
 			ProtectionEnabled:          true, // whether or not use any of dnsfilter features
 			FilteringEnabled:           true, // whether or not use filter lists
+			FiltersUpdateIntervalHours: 24,
 			BlockingMode:               "nxdomain", // mode how to answer filtered requests
 			BlockedResponseTTL:         10,         // in seconds
 			QueryLogEnabled:            true,
@@ -191,12 +192,6 @@ var config = configuration{
 			PortDNSOverTLS: 853, // needs to be passed through to dnsproxy
 		},
 	},
-	Filters: []filter{
-		{Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"},
-		{Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"},
-		{Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"},
-		{Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"},
-	},
 	DHCP: dhcpd.ServerConfig{
 		LeaseDuration: 86400,
 		ICMPTimeout:   1000,
@@ -226,6 +221,7 @@ func initConfig() {
 	config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024
 	config.DNS.ParentalCacheSize = 1 * 1024 * 1024
 	config.DNS.CacheTime = 30
+	config.Filters = defaultFilters()
 }
 
 // getConfigFilename returns path to the current config file
@@ -276,6 +272,9 @@ func parseConfig() error {
 	if !checkStatsInterval(config.DNS.StatsInterval) {
 		config.DNS.StatsInterval = 1
 	}
+	if !checkFiltersUpdateIntervalHours(config.DNS.FiltersUpdateIntervalHours) {
+		config.DNS.FiltersUpdateIntervalHours = 24
+	}
 
 	if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
 		config.DNS.QueryLogInterval = 1
@@ -308,11 +307,6 @@ func parseConfig() error {
 		return err
 	}
 
-	// Deduplicate filters
-	deduplicateFilters()
-
-	updateUniqueFilterID(config.Filters)
-
 	return nil
 }
 

home/control.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
 	"github.com/AdguardTeam/dnsproxy/upstream"
@@ -17,8 +16,6 @@ import (
 	"github.com/miekg/dns"
 )
 
-const updatePeriod = time.Hour * 24
-
 var protocols = []string{"tls://", "https://", "tcp://", "sdns://"}
 
 // ----------------
@@ -547,15 +544,6 @@ func registerControlHandlers() {
 	httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage)
 	http.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON)))
 	httpRegister(http.MethodPost, "/control/update", handleUpdate)
-	httpRegister(http.MethodPost, "/control/filtering/enable", handleFilteringEnable)
-	httpRegister(http.MethodPost, "/control/filtering/disable", handleFilteringDisable)
-	httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL)
-	httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL)
-	httpRegister(http.MethodPost, "/control/filtering/enable_url", handleFilteringEnableURL)
-	httpRegister(http.MethodPost, "/control/filtering/disable_url", handleFilteringDisableURL)
-	httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh)
-	httpRegister(http.MethodGet, "/control/filtering/status", handleFilteringStatus)
-	httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules)
 	httpRegister(http.MethodPost, "/control/safebrowsing/enable", handleSafeBrowsingEnable)
 	httpRegister(http.MethodPost, "/control/safebrowsing/disable", handleSafeBrowsingDisable)
 	httpRegister(http.MethodGet, "/control/safebrowsing/status", handleSafeBrowsingStatus)
@@ -575,6 +563,7 @@ func registerControlHandlers() {
 	httpRegister(http.MethodGet, "/control/access/list", handleAccessList)
 	httpRegister(http.MethodPost, "/control/access/set", handleAccessSet)
 
+	RegisterFilteringHandlers()
 	RegisterTLSHandlers()
 	RegisterClientsHandlers()
 	registerRewritesHandlers()

home/control_filtering.go

@@ -5,74 +5,57 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/AdguardTeam/golibs/log"
-	"github.com/asaskevich/govalidator"
 )
 
-func handleFilteringEnable(w http.ResponseWriter, r *http.Request) {
+// IsValidURL - return TRUE if URL is valid
-	config.DNS.FilteringEnabled = true
+func IsValidURL(rawurl string) bool {
-	httpUpdateConfigReloadDNSReturnOK(w, r)
+	url, err := url.ParseRequestURI(rawurl)
+	if err != nil {
+		return false //Couldn't even parse the rawurl
+	}
+	if len(url.Scheme) == 0 {
+		return false //No Scheme found
+	}
+	return true
 }
 
-func handleFilteringDisable(w http.ResponseWriter, r *http.Request) {
+type filterAddJSON struct {
-	config.DNS.FilteringEnabled = false
+	Name string `json:"name"`
-	httpUpdateConfigReloadDNSReturnOK(w, r)
+	URL  string `json:"url"`
-}
-
-func handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
-	data := map[string]interface{}{
-		"enabled": config.DNS.FilteringEnabled,
-	}
-
-	config.RLock()
-	data["filters"] = config.Filters
-	data["user_rules"] = config.UserRules
-	jsonVal, err := json.Marshal(data)
-	config.RUnlock()
-
-	if err != nil {
-		httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	_, err = w.Write(jsonVal)
-	if err != nil {
-		httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
-		return
-	}
 }
 
 func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
-	f := filter{}
+	fj := filterAddJSON{}
-	err := json.NewDecoder(r.Body).Decode(&f)
+	err := json.NewDecoder(r.Body).Decode(&fj)
 	if err != nil {
 		httpError(w, http.StatusBadRequest, "Failed to parse request body json: %s", err)
 		return
 	}
 
-	if len(f.URL) == 0 {
+	if !IsValidURL(fj.URL) {
-		http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
+		http.Error(w, "Invalid URL", http.StatusBadRequest)
-		return
-	}
-
-	if valid := govalidator.IsRequestURL(f.URL); !valid {
-		http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
 		return
 	}
 
 	// Check for duplicates
-	if filterExists(f.URL) {
+	if filterExists(fj.URL) {
-		httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", f.URL)
+		httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", fj.URL)
 		return
 	}
 
 	// Set necessary properties
+	f := filter{
+		Enabled: true,
+		URL:     fj.URL,
+		Name:    fj.Name,
+	}
 	f.ID = assignUniqueFilterID()
-	f.Enabled = true
 
 	// Download the filter contents
 	ok, err := f.update()
@@ -133,7 +116,7 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if valid := govalidator.IsRequestURL(req.URL); !valid {
+	if !IsValidURL(req.URL) {
 		http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
 		return
 	}
@@ -166,54 +149,27 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
 	httpUpdateConfigReloadDNSReturnOK(w, r)
 }
 
-func handleFilteringEnableURL(w http.ResponseWriter, r *http.Request) {
+type filterURLJSON struct {
-	parameters, err := parseParametersFromBody(r.Body)
+	URL     string `json:"url"`
-	if err != nil {
+	Enabled bool   `json:"enabled"`
-		httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err)
-		return
-	}
-
-	url, ok := parameters["url"]
-	if !ok {
-		http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
-		return
-	}
-
-	if valid := govalidator.IsRequestURL(url); !valid {
-		http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
-		return
-	}
-
-	found := filterEnable(url, true)
-	if !found {
-		http.Error(w, "URL parameter was not previously added", http.StatusBadRequest)
-		return
-	}
-
-	httpUpdateConfigReloadDNSReturnOK(w, r)
 }
 
-func handleFilteringDisableURL(w http.ResponseWriter, r *http.Request) {
+func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
-	parameters, err := parseParametersFromBody(r.Body)
+	fj := filterURLJSON{}
+	err := json.NewDecoder(r.Body).Decode(&fj)
 	if err != nil {
-		httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err)
+		httpError(w, http.StatusBadRequest, "json decode: %s", err)
 		return
 	}
 
-	url, ok := parameters["url"]
+	if !IsValidURL(fj.URL) {
-	if !ok {
+		http.Error(w, "invalid URL", http.StatusBadRequest)
-		http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
 		return
 	}
 
-	if valid := govalidator.IsRequestURL(url); !valid {
+	found := filterEnable(fj.URL, fj.Enabled)
-		http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
-		return
-	}
-
-	found := filterEnable(url, false)
 	if !found {
-		http.Error(w, "URL parameter was not previously added", http.StatusBadRequest)
+		http.Error(w, "URL doesn't exist", http.StatusBadRequest)
 		return
 	}
 
@@ -235,3 +191,91 @@ func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
 	updated := refreshFiltersIfNecessary(true)
 	fmt.Fprintf(w, "OK %d filters updated\n", updated)
 }
+
+type filterJSON struct {
+	ID          int64  `json:"id"`
+	Enabled     bool   `json:"enabled"`
+	URL         string `json:"url"`
+	Name        string `json:"name"`
+	RulesCount  uint32 `json:"rules_count"`
+	LastUpdated string `json:"last_updated"`
+}
+
+type filteringConfig struct {
+	Enabled   bool         `json:"enabled"`
+	Interval  uint32       `json:"interval"` // in hours
+	Filters   []filterJSON `json:"filters"`
+	UserRules []string     `json:"user_rules"`
+}
+
+// Get filtering configuration
+func handleFilteringInfo(w http.ResponseWriter, r *http.Request) {
+	resp := filteringConfig{}
+	config.RLock()
+	resp.Enabled = config.DNS.FilteringEnabled
+	resp.Interval = config.DNS.FiltersUpdateIntervalHours
+	for _, f := range config.Filters {
+		fj := filterJSON{
+			ID:         f.ID,
+			Enabled:    f.Enabled,
+			URL:        f.URL,
+			Name:       f.Name,
+			RulesCount: uint32(f.RulesCount),
+		}
+
+		if f.LastUpdated.Second() != 0 {
+			fj.LastUpdated = f.LastUpdated.Format(time.RFC3339)
+		}
+
+		resp.Filters = append(resp.Filters, fj)
+	}
+	resp.UserRules = config.UserRules
+	config.RUnlock()
+
+	jsonVal, err := json.Marshal(resp)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "json encode: %s", err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(w, http.StatusInternalServerError, "http write: %s", err)
+	}
+}
+
+// Set filtering configuration
+func handleFilteringConfig(w http.ResponseWriter, r *http.Request) {
+	req := filteringConfig{}
+	err := json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		httpError(w, http.StatusBadRequest, "json decode: %s", err)
+		return
+	}
+
+	if !checkFiltersUpdateIntervalHours(req.Interval) {
+		httpError(w, http.StatusBadRequest, "Unsupported interval")
+		return
+	}
+
+	config.DNS.FilteringEnabled = req.Enabled
+	config.DNS.FiltersUpdateIntervalHours = req.Interval
+	httpUpdateConfigReloadDNSReturnOK(w, r)
+
+	returnOK(w)
+}
+
+// RegisterFilteringHandlers - register handlers
+func RegisterFilteringHandlers() {
+	httpRegister(http.MethodGet, "/control/filtering_info", handleFilteringInfo)
+	httpRegister(http.MethodPost, "/control/filtering_config", handleFilteringConfig)
+	httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL)
+	httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL)
+	httpRegister(http.MethodPost, "/control/filtering/set_url", handleFilteringSetURL)
+	httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh)
+	httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules)
+}
+
+func checkFiltersUpdateIntervalHours(i uint32) bool {
+	return i == 0 || i == 1 || i == 12 || i == 1*24 || i == 3*24 || i == 7*24
+}

home/control_install.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 
 	"github.com/AdguardTeam/golibs/log"
@@ -239,6 +240,9 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) {
 	config.AuthName = newSettings.Username
 	config.AuthPass = newSettings.Password
 
+	dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir)
+	initDNSServer(dnsBaseDir)
+
 	err = startDNSServer()
 	if err != nil {
 		config.firstRun = true
@@ -255,8 +259,6 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	go refreshFiltersIfNecessary(false)
-
 	// this needs to be done in a goroutine because Shutdown() is a blocking call, and it will block
 	// until all requests are finished, and _we_ are inside a request right now, so it will block indefinitely
 	if restartHTTP {

home/dns.go

@@ -49,6 +49,7 @@ func initDNSServer(baseDir string) {
 	config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
 
 	initRDNS()
+	initFiltering()
 }
 
 func isRunning() bool {
@@ -165,6 +166,11 @@ func startDNSServer() error {
 		return errorx.Decorate(err, "Couldn't start forwarding DNS server")
 	}
 
+	if !config.filteringStarted {
+		config.filteringStarted = true
+		startRefreshFilters()
+	}
+
 	return nil
 }
 

home/filter.go

@@ -21,13 +21,35 @@ var (
 	filterTitleRegexp = regexp.MustCompile(`^! Title: +(.*)$`)
 )
 
+func initFiltering() {
+	loadFilters()
+	deduplicateFilters()
+	updateUniqueFilterID(config.Filters)
+}
+
+func startRefreshFilters() {
+	go func() {
+		_ = refreshFiltersIfNecessary(false)
+	}()
+	go periodicallyRefreshFilters()
+}
+
+func defaultFilters() []filter {
+	return []filter{
+		{Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"},
+		{Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"},
+		{Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"},
+		{Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"},
+	}
+}
+
 // field ordering is important -- yaml fields will mirror ordering from here
 type filter struct {
-	Enabled     bool      `json:"enabled"`
+	Enabled     bool
-	URL         string    `json:"url"`
+	URL         string
-	Name        string    `json:"name" yaml:"name"`
+	Name        string    `yaml:"name"`
-	RulesCount  int       `json:"rulesCount" yaml:"-"`
+	RulesCount  int       `yaml:"-"`
-	LastUpdated time.Time `json:"lastUpdated,omitempty" yaml:"-"`
+	LastUpdated time.Time `yaml:"-"`
 	checksum    uint32    // checksum of the file data
 
 	dnsfilter.Filter `yaml:",inline"`
@@ -119,8 +141,7 @@ func loadFilters() {
 
 		err := filter.load()
 		if err != nil {
-			// This is okay for the first start, the filter will be loaded later
+			log.Error("Couldn't load filter %d contents due to %s", filter.ID, err)
-			log.Debug("Couldn't load filter %d contents due to %s", filter.ID, err)
 		}
 	}
 }
@@ -159,7 +180,12 @@ func assignUniqueFilterID() int64 {
 
 // Sets up a timer that will be checking for filters updates periodically
 func periodicallyRefreshFilters() {
-	for range time.Tick(time.Minute) {
+	for {
+		time.Sleep(1 * time.Hour)
+		if config.DNS.FiltersUpdateIntervalHours == 0 {
+			continue
+		}
+
 		refreshFiltersIfNecessary(false)
 	}
 }
@@ -180,10 +206,7 @@ func refreshFiltersIfNecessary(force bool) int {
 	var updateFilters []filter
 	var updateFlags []bool // 'true' if filter data has changed
 
-	if config.firstRun {
+	now := time.Now()
-		return 0
-	}
-
 	config.RLock()
 	for i := range config.Filters {
 		f := &config.Filters[i] // otherwise we will be operating on a copy
@@ -192,7 +215,8 @@ func refreshFiltersIfNecessary(force bool) int {
 			continue
 		}
 
-		if !force && time.Since(f.LastUpdated) <= updatePeriod {
+		expireTime := f.LastUpdated.Unix() + int64(config.DNS.FiltersUpdateIntervalHours)*60*60
+		if !force && expireTime > now.Unix() {
 			continue
 		}
 
@@ -214,7 +238,7 @@ func refreshFiltersIfNecessary(force bool) int {
 			log.Printf("Failed to update filter %s: %s\n", uf.URL, err)
 			continue
 		}
-		uf.LastUpdated = time.Now()
+		uf.LastUpdated = now
 		if updated {
 			updateCount++
 		}

home/filter_test.go

@@ -0,0 +1,37 @@
+package home
+
+import (
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilters(t *testing.T) {
+	config.client = &http.Client{
+		Timeout: time.Minute * 5,
+	}
+
+	f := filter{
+		URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt",
+	}
+
+	// download
+	ok, err := f.update()
+	assert.True(t, ok && err == nil)
+
+	// refresh
+	ok, err = f.update()
+	assert.True(t, !ok && err == nil)
+
+	err = f.save()
+	assert.True(t, err == nil)
+
+	err = f.load()
+	assert.True(t, err == nil)
+
+	f.unload()
+	os.Remove(f.Path())
+}

home/home.go

@@ -135,22 +135,17 @@ func run(args options) {
 		config.BindPort = args.bindPort
 	}
 
-	loadFilters()
-
 	if !config.firstRun {
 		// Save the updated config
 		err := config.write()
 		if err != nil {
 			log.Fatal(err)
 		}
-	}
 
-	// Init the DNS server instance before registering HTTP handlers
 		dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir)
 		initDNSServer(dnsBaseDir)
 
-	if !config.firstRun {
+		err = startDNSServer()
-		err := startDNSServer()
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -165,13 +160,6 @@ func run(args options) {
 		config.pidFileName = args.pidFile
 	}
 
-	// Update filters we've just loaded right away, don't wait for periodic update timer
-	go func() {
-		refreshFiltersIfNecessary(false)
-	}()
-	// Schedule automatic filters updates
-	go periodicallyRefreshFilters()
-
 	// Initialize and run the admin Web interface
 	box := packr.NewBox("../build/static")
 

querylog/qlog.go

@@ -35,7 +35,7 @@ type queryLog struct {
 	lock  sync.RWMutex
 }
 
-// newQueryLog creates a new instance of the query log
+// create a new instance of the query log
 func newQueryLog(conf Config) *queryLog {
 	l := queryLog{}
 	l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
@@ -53,7 +53,6 @@ func (l *queryLog) Configure(conf Config) {
 	l.conf = conf
 }
 
-// Clear memory buffer and remove the file
 func (l *queryLog) Clear() {
 	l.fileFlushLock.Lock()
 	defer l.fileFlushLock.Unlock()
@@ -164,7 +163,6 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res
 	}
 }
 
-// getQueryLogJson returns a map with the current query log ready to be converted to a JSON
 func (l *queryLog) GetData() []map[string]interface{} {
 	l.lock.RLock()
 	values := make([]*logEntry, len(l.cache))

querylog/querylog.go

@@ -10,14 +10,20 @@ import (
 
 // QueryLog - main interface
 type QueryLog interface {
+	// Close query log object
 	Close()
 
 	// Set new configuration at runtime
 	// Currently only 'Interval' field is supported.
 	Configure(conf Config)
 
+	// Add a log entry
 	Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
+
+	// Get log entries
 	GetData() []map[string]interface{}
+
+	// Clear memory buffer and remove log files
 	Clear()
 }
 
@@ -27,7 +33,7 @@ type Config struct {
 	Interval uint32 // interval to rotate logs (in hours)
 }
 
-// New - create instance
+// New - create a new instance of the query log
 func New(conf Config) QueryLog {
 	return newQueryLog(conf)
 }