filtering: refactor; change API; add "filters_update_interval" setting
+ config: "filters_update_interval" * add /control/filtering_info * remove /control/filtering/enable * remove /control/filtering/disable * add /control/filtering_config * remove /control/filtering/status * add /control/filtering/set_url * remove /control/filtering/enable_url * remove /control/filtering/disable_url
This commit is contained in:
Simon Zolin
parent
8c89973365
commit
adb422fedf
|
|
@ -45,6 +45,11 @@ Contents:
|
|||
* Query logs
|
||||
* API: Set querylog parameters
|
||||
* API: Get querylog parameters
|
||||
* Filtering
|
||||
* Filters update mechanism
|
||||
* API: Get filtering parameters
|
||||
* API: Set filtering parameters
|
||||
* API: Set URL parameters
|
||||
|
||||
|
||||
## Relations between subsystems
|
||||
|
|
@ -1019,3 +1024,76 @@ Response:
|
|||
"enabled": true | false
|
||||
"interval": 1 | 7 | 30 | 90
|
||||
}
|
||||
|
||||
|
||||
## Filtering
|
||||
|
||||
### Filters update mechanism
|
||||
|
||||
Filters can be updated either manually by request from UI or automatically.
|
||||
Auto-update interval can be configured in UI. If it is 0, auto-update is disabled.
|
||||
When the last modification date of filter files is older than auto-update interval, auto-update procedure is started.
|
||||
If an enabled filter file doesn't exist, it's downloaded on application startup. This includes the case when installation wizard is completed and there are no filter files yet.
|
||||
When auto-update time comes, server starts the update procedure by downloading filter files. After new filter files are in place, it restarts DNS filtering module with new rules.
|
||||
Only filters that are enabled by configuration can be updated.
|
||||
As a result of the update procedure, all enabled filter files are written to disk, refreshed (their last modification date is equal to the current time) and loaded.
|
||||
|
||||
|
||||
### API: Get filtering parameters
|
||||
|
||||
Request:
|
||||
|
||||
GET /control/filtering_info
|
||||
|
||||
Response:
|
||||
|
||||
200 OK
|
||||
|
||||
{
|
||||
"enabled": true | false
|
||||
"interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24
|
||||
"filters":[
|
||||
{
|
||||
"id":1
|
||||
"enabled":true,
|
||||
"url":"https://...",
|
||||
"name":"...",
|
||||
"rules_count":1234,
|
||||
"last_updated":"2019-09-04T18:29:30+00:00",
|
||||
}
|
||||
...
|
||||
],
|
||||
"user_rules":["...", ...]
|
||||
}
|
||||
|
||||
|
||||
### API: Set filtering parameters
|
||||
|
||||
Request:
|
||||
|
||||
POST /control/filtering_config
|
||||
|
||||
{
|
||||
"enabled": true | false
|
||||
"interval": 0 | 1 | 12 | 1*24 || 3*24 || 7*24
|
||||
}
|
||||
|
||||
Response:
|
||||
|
||||
200 OK
|
||||
|
||||
|
||||
### API: Set URL parameters
|
||||
|
||||
Request:
|
||||
|
||||
POST /control/filtering/set_url
|
||||
|
||||
{
|
||||
"url": "..."
|
||||
"enabled": true | false
|
||||
}
|
||||
|
||||
Response:
|
||||
|
||||
200 OK
|
||||
|
|
|
|||
|
|
@ -114,9 +114,9 @@ type Dnsfilter struct {
|
|||
|
||||
// Filter represents a filter list
|
||||
type Filter struct {
|
||||
ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
|
||||
Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n'
|
||||
FilePath string `json:"-" yaml:"-"` // Path to a filtering rules file
|
||||
ID int64 // auto-assigned when filter is added (see nextFilterID)
|
||||
Data []byte `yaml:"-"` // List of rules divided by '\n'
|
||||
FilePath string `yaml:"-"` // Path to a filtering rules file
|
||||
}
|
||||
|
||||
//go:generate stringer -type=Reason
|
||||
|
|
|
|||
|
|
@ -68,6 +68,8 @@ func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
|
|||
type FilteringConfig struct {
|
||||
ProtectionEnabled bool `yaml:"protection_enabled"` // whether or not use any of dnsfilter features
|
||||
FilteringEnabled bool `yaml:"filtering_enabled"` // whether or not use filter lists
|
||||
FiltersUpdateIntervalHours uint32 `yaml:"filters_update_interval"` // time period to update filters (in hours)
|
||||
|
||||
BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests
|
||||
BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
|
||||
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -7,7 +7,6 @@ require (
|
|||
github.com/AdguardTeam/golibs v0.2.1
|
||||
github.com/AdguardTeam/urlfilter v0.5.0
|
||||
github.com/NYTimes/gziphandler v1.1.1
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
|
||||
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833
|
||||
github.com/etcd-io/bbolt v1.3.3
|
||||
github.com/go-test/deep v1.0.1
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/dhcpd"
|
||||
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
|
||||
"github.com/AdguardTeam/AdGuardHome/dnsforward"
|
||||
"github.com/AdguardTeam/AdGuardHome/querylog"
|
||||
"github.com/AdguardTeam/AdGuardHome/stats"
|
||||
|
|
@ -72,6 +71,7 @@ type configuration struct {
|
|||
client *http.Client
|
||||
stats stats.Stats
|
||||
queryLog querylog.QueryLog
|
||||
filteringStarted bool
|
||||
|
||||
// cached version.json to avoid hammering github.io for each page reload
|
||||
versionCheckJSON []byte
|
||||
|
|
@ -174,6 +174,7 @@ var config = configuration{
|
|||
FilteringConfig: dnsforward.FilteringConfig{
|
||||
ProtectionEnabled: true, // whether or not use any of dnsfilter features
|
||||
FilteringEnabled: true, // whether or not use filter lists
|
||||
FiltersUpdateIntervalHours: 24,
|
||||
BlockingMode: "nxdomain", // mode how to answer filtered requests
|
||||
BlockedResponseTTL: 10, // in seconds
|
||||
QueryLogEnabled: true,
|
||||
|
|
@ -191,12 +192,6 @@ var config = configuration{
|
|||
PortDNSOverTLS: 853, // needs to be passed through to dnsproxy
|
||||
},
|
||||
},
|
||||
Filters: []filter{
|
||||
{Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"},
|
||||
{Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"},
|
||||
{Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"},
|
||||
{Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"},
|
||||
},
|
||||
DHCP: dhcpd.ServerConfig{
|
||||
LeaseDuration: 86400,
|
||||
ICMPTimeout: 1000,
|
||||
|
|
@ -226,6 +221,7 @@ func initConfig() {
|
|||
config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024
|
||||
config.DNS.ParentalCacheSize = 1 * 1024 * 1024
|
||||
config.DNS.CacheTime = 30
|
||||
config.Filters = defaultFilters()
|
||||
}
|
||||
|
||||
// getConfigFilename returns path to the current config file
|
||||
|
|
@ -276,6 +272,9 @@ func parseConfig() error {
|
|||
if !checkStatsInterval(config.DNS.StatsInterval) {
|
||||
config.DNS.StatsInterval = 1
|
||||
}
|
||||
if !checkFiltersUpdateIntervalHours(config.DNS.FiltersUpdateIntervalHours) {
|
||||
config.DNS.FiltersUpdateIntervalHours = 24
|
||||
}
|
||||
|
||||
if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
|
||||
config.DNS.QueryLogInterval = 1
|
||||
|
|
@ -308,11 +307,6 @@ func parseConfig() error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Deduplicate filters
|
||||
deduplicateFilters()
|
||||
|
||||
updateUniqueFilterID(config.Filters)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/dnsforward"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
|
|
@ -17,8 +16,6 @@ import (
|
|||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
const updatePeriod = time.Hour * 24
|
||||
|
||||
var protocols = []string{"tls://", "https://", "tcp://", "sdns://"}
|
||||
|
||||
// ----------------
|
||||
|
|
@ -547,15 +544,6 @@ func registerControlHandlers() {
|
|||
httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage)
|
||||
http.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON)))
|
||||
httpRegister(http.MethodPost, "/control/update", handleUpdate)
|
||||
httpRegister(http.MethodPost, "/control/filtering/enable", handleFilteringEnable)
|
||||
httpRegister(http.MethodPost, "/control/filtering/disable", handleFilteringDisable)
|
||||
httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/enable_url", handleFilteringEnableURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/disable_url", handleFilteringDisableURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh)
|
||||
httpRegister(http.MethodGet, "/control/filtering/status", handleFilteringStatus)
|
||||
httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules)
|
||||
httpRegister(http.MethodPost, "/control/safebrowsing/enable", handleSafeBrowsingEnable)
|
||||
httpRegister(http.MethodPost, "/control/safebrowsing/disable", handleSafeBrowsingDisable)
|
||||
httpRegister(http.MethodGet, "/control/safebrowsing/status", handleSafeBrowsingStatus)
|
||||
|
|
@ -575,6 +563,7 @@ func registerControlHandlers() {
|
|||
httpRegister(http.MethodGet, "/control/access/list", handleAccessList)
|
||||
httpRegister(http.MethodPost, "/control/access/set", handleAccessSet)
|
||||
|
||||
RegisterFilteringHandlers()
|
||||
RegisterTLSHandlers()
|
||||
RegisterClientsHandlers()
|
||||
registerRewritesHandlers()
|
||||
|
|
|
|||
|
|
@ -5,74 +5,57 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/asaskevich/govalidator"
|
||||
)
|
||||
|
||||
func handleFilteringEnable(w http.ResponseWriter, r *http.Request) {
|
||||
config.DNS.FilteringEnabled = true
|
||||
httpUpdateConfigReloadDNSReturnOK(w, r)
|
||||
// IsValidURL - return TRUE if URL is valid
|
||||
func IsValidURL(rawurl string) bool {
|
||||
url, err := url.ParseRequestURI(rawurl)
|
||||
if err != nil {
|
||||
return false //Couldn't even parse the rawurl
|
||||
}
|
||||
if len(url.Scheme) == 0 {
|
||||
return false //No Scheme found
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func handleFilteringDisable(w http.ResponseWriter, r *http.Request) {
|
||||
config.DNS.FilteringEnabled = false
|
||||
httpUpdateConfigReloadDNSReturnOK(w, r)
|
||||
}
|
||||
|
||||
func handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
|
||||
data := map[string]interface{}{
|
||||
"enabled": config.DNS.FilteringEnabled,
|
||||
}
|
||||
|
||||
config.RLock()
|
||||
data["filters"] = config.Filters
|
||||
data["user_rules"] = config.UserRules
|
||||
jsonVal, err := json.Marshal(data)
|
||||
config.RUnlock()
|
||||
|
||||
if err != nil {
|
||||
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err = w.Write(jsonVal)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
|
||||
return
|
||||
}
|
||||
type filterAddJSON struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
|
||||
f := filter{}
|
||||
err := json.NewDecoder(r.Body).Decode(&f)
|
||||
fj := filterAddJSON{}
|
||||
err := json.NewDecoder(r.Body).Decode(&fj)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusBadRequest, "Failed to parse request body json: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(f.URL) == 0 {
|
||||
http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if valid := govalidator.IsRequestURL(f.URL); !valid {
|
||||
http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
|
||||
if !IsValidURL(fj.URL) {
|
||||
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
if filterExists(f.URL) {
|
||||
httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", f.URL)
|
||||
if filterExists(fj.URL) {
|
||||
httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", fj.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Set necessary properties
|
||||
f := filter{
|
||||
Enabled: true,
|
||||
URL: fj.URL,
|
||||
Name: fj.Name,
|
||||
}
|
||||
f.ID = assignUniqueFilterID()
|
||||
f.Enabled = true
|
||||
|
||||
// Download the filter contents
|
||||
ok, err := f.update()
|
||||
|
|
@ -133,7 +116,7 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if valid := govalidator.IsRequestURL(req.URL); !valid {
|
||||
if !IsValidURL(req.URL) {
|
||||
http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
|
@ -166,54 +149,27 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
|
|||
httpUpdateConfigReloadDNSReturnOK(w, r)
|
||||
}
|
||||
|
||||
func handleFilteringEnableURL(w http.ResponseWriter, r *http.Request) {
|
||||
parameters, err := parseParametersFromBody(r.Body)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
url, ok := parameters["url"]
|
||||
if !ok {
|
||||
http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if valid := govalidator.IsRequestURL(url); !valid {
|
||||
http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
found := filterEnable(url, true)
|
||||
if !found {
|
||||
http.Error(w, "URL parameter was not previously added", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
httpUpdateConfigReloadDNSReturnOK(w, r)
|
||||
type filterURLJSON struct {
|
||||
URL string `json:"url"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func handleFilteringDisableURL(w http.ResponseWriter, r *http.Request) {
|
||||
parameters, err := parseParametersFromBody(r.Body)
|
||||
func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
|
||||
fj := filterURLJSON{}
|
||||
err := json.NewDecoder(r.Body).Decode(&fj)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err)
|
||||
httpError(w, http.StatusBadRequest, "json decode: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
url, ok := parameters["url"]
|
||||
if !ok {
|
||||
http.Error(w, "URL parameter was not specified", http.StatusBadRequest)
|
||||
if !IsValidURL(fj.URL) {
|
||||
http.Error(w, "invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if valid := govalidator.IsRequestURL(url); !valid {
|
||||
http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
found := filterEnable(url, false)
|
||||
found := filterEnable(fj.URL, fj.Enabled)
|
||||
if !found {
|
||||
http.Error(w, "URL parameter was not previously added", http.StatusBadRequest)
|
||||
http.Error(w, "URL doesn't exist", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -235,3 +191,91 @@ func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
|
|||
updated := refreshFiltersIfNecessary(true)
|
||||
fmt.Fprintf(w, "OK %d filters updated\n", updated)
|
||||
}
|
||||
|
||||
type filterJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
URL string `json:"url"`
|
||||
Name string `json:"name"`
|
||||
RulesCount uint32 `json:"rules_count"`
|
||||
LastUpdated string `json:"last_updated"`
|
||||
}
|
||||
|
||||
type filteringConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Interval uint32 `json:"interval"` // in hours
|
||||
Filters []filterJSON `json:"filters"`
|
||||
UserRules []string `json:"user_rules"`
|
||||
}
|
||||
|
||||
// Get filtering configuration
|
||||
func handleFilteringInfo(w http.ResponseWriter, r *http.Request) {
|
||||
resp := filteringConfig{}
|
||||
config.RLock()
|
||||
resp.Enabled = config.DNS.FilteringEnabled
|
||||
resp.Interval = config.DNS.FiltersUpdateIntervalHours
|
||||
for _, f := range config.Filters {
|
||||
fj := filterJSON{
|
||||
ID: f.ID,
|
||||
Enabled: f.Enabled,
|
||||
URL: f.URL,
|
||||
Name: f.Name,
|
||||
RulesCount: uint32(f.RulesCount),
|
||||
}
|
||||
|
||||
if f.LastUpdated.Second() != 0 {
|
||||
fj.LastUpdated = f.LastUpdated.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
resp.Filters = append(resp.Filters, fj)
|
||||
}
|
||||
resp.UserRules = config.UserRules
|
||||
config.RUnlock()
|
||||
|
||||
jsonVal, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err = w.Write(jsonVal)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusInternalServerError, "http write: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set filtering configuration
|
||||
func handleFilteringConfig(w http.ResponseWriter, r *http.Request) {
|
||||
req := filteringConfig{}
|
||||
err := json.NewDecoder(r.Body).Decode(&req)
|
||||
if err != nil {
|
||||
httpError(w, http.StatusBadRequest, "json decode: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !checkFiltersUpdateIntervalHours(req.Interval) {
|
||||
httpError(w, http.StatusBadRequest, "Unsupported interval")
|
||||
return
|
||||
}
|
||||
|
||||
config.DNS.FilteringEnabled = req.Enabled
|
||||
config.DNS.FiltersUpdateIntervalHours = req.Interval
|
||||
httpUpdateConfigReloadDNSReturnOK(w, r)
|
||||
|
||||
returnOK(w)
|
||||
}
|
||||
|
||||
// RegisterFilteringHandlers - register handlers
|
||||
func RegisterFilteringHandlers() {
|
||||
httpRegister(http.MethodGet, "/control/filtering_info", handleFilteringInfo)
|
||||
httpRegister(http.MethodPost, "/control/filtering_config", handleFilteringConfig)
|
||||
httpRegister(http.MethodPost, "/control/filtering/add_url", handleFilteringAddURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/remove_url", handleFilteringRemoveURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/set_url", handleFilteringSetURL)
|
||||
httpRegister(http.MethodPost, "/control/filtering/refresh", handleFilteringRefresh)
|
||||
httpRegister(http.MethodPost, "/control/filtering/set_rules", handleFilteringSetRules)
|
||||
}
|
||||
|
||||
func checkFiltersUpdateIntervalHours(i uint32) bool {
|
||||
return i == 0 || i == 1 || i == 12 || i == 1*24 || i == 3*24 || i == 7*24
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
|
|
@ -239,6 +240,9 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) {
|
|||
config.AuthName = newSettings.Username
|
||||
config.AuthPass = newSettings.Password
|
||||
|
||||
dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir)
|
||||
initDNSServer(dnsBaseDir)
|
||||
|
||||
err = startDNSServer()
|
||||
if err != nil {
|
||||
config.firstRun = true
|
||||
|
|
@ -255,8 +259,6 @@ func handleInstallConfigure(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
go refreshFiltersIfNecessary(false)
|
||||
|
||||
// this needs to be done in a goroutine because Shutdown() is a blocking call, and it will block
|
||||
// until all requests are finished, and _we_ are inside a request right now, so it will block indefinitely
|
||||
if restartHTTP {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ func initDNSServer(baseDir string) {
|
|||
config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
|
||||
|
||||
initRDNS()
|
||||
initFiltering()
|
||||
}
|
||||
|
||||
func isRunning() bool {
|
||||
|
|
@ -165,6 +166,11 @@ func startDNSServer() error {
|
|||
return errorx.Decorate(err, "Couldn't start forwarding DNS server")
|
||||
}
|
||||
|
||||
if !config.filteringStarted {
|
||||
config.filteringStarted = true
|
||||
startRefreshFilters()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,13 +21,35 @@ var (
|
|||
filterTitleRegexp = regexp.MustCompile(`^! Title: +(.*)$`)
|
||||
)
|
||||
|
||||
func initFiltering() {
|
||||
loadFilters()
|
||||
deduplicateFilters()
|
||||
updateUniqueFilterID(config.Filters)
|
||||
}
|
||||
|
||||
func startRefreshFilters() {
|
||||
go func() {
|
||||
_ = refreshFiltersIfNecessary(false)
|
||||
}()
|
||||
go periodicallyRefreshFilters()
|
||||
}
|
||||
|
||||
func defaultFilters() []filter {
|
||||
return []filter{
|
||||
{Filter: dnsfilter.Filter{ID: 1}, Enabled: true, URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", Name: "AdGuard Simplified Domain Names filter"},
|
||||
{Filter: dnsfilter.Filter{ID: 2}, Enabled: false, URL: "https://adaway.org/hosts.txt", Name: "AdAway"},
|
||||
{Filter: dnsfilter.Filter{ID: 3}, Enabled: false, URL: "https://hosts-file.net/ad_servers.txt", Name: "hpHosts - Ad and Tracking servers only"},
|
||||
{Filter: dnsfilter.Filter{ID: 4}, Enabled: false, URL: "https://www.malwaredomainlist.com/hostslist/hosts.txt", Name: "MalwareDomainList.com Hosts List"},
|
||||
}
|
||||
}
|
||||
|
||||
// field ordering is important -- yaml fields will mirror ordering from here
|
||||
type filter struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
URL string `json:"url"`
|
||||
Name string `json:"name" yaml:"name"`
|
||||
RulesCount int `json:"rulesCount" yaml:"-"`
|
||||
LastUpdated time.Time `json:"lastUpdated,omitempty" yaml:"-"`
|
||||
Enabled bool
|
||||
URL string
|
||||
Name string `yaml:"name"`
|
||||
RulesCount int `yaml:"-"`
|
||||
LastUpdated time.Time `yaml:"-"`
|
||||
checksum uint32 // checksum of the file data
|
||||
|
||||
dnsfilter.Filter `yaml:",inline"`
|
||||
|
|
@ -119,8 +141,7 @@ func loadFilters() {
|
|||
|
||||
err := filter.load()
|
||||
if err != nil {
|
||||
// This is okay for the first start, the filter will be loaded later
|
||||
log.Debug("Couldn't load filter %d contents due to %s", filter.ID, err)
|
||||
log.Error("Couldn't load filter %d contents due to %s", filter.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -159,7 +180,12 @@ func assignUniqueFilterID() int64 {
|
|||
|
||||
// Sets up a timer that will be checking for filters updates periodically
|
||||
func periodicallyRefreshFilters() {
|
||||
for range time.Tick(time.Minute) {
|
||||
for {
|
||||
time.Sleep(1 * time.Hour)
|
||||
if config.DNS.FiltersUpdateIntervalHours == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
refreshFiltersIfNecessary(false)
|
||||
}
|
||||
}
|
||||
|
|
@ -180,10 +206,7 @@ func refreshFiltersIfNecessary(force bool) int {
|
|||
var updateFilters []filter
|
||||
var updateFlags []bool // 'true' if filter data has changed
|
||||
|
||||
if config.firstRun {
|
||||
return 0
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
config.RLock()
|
||||
for i := range config.Filters {
|
||||
f := &config.Filters[i] // otherwise we will be operating on a copy
|
||||
|
|
@ -192,7 +215,8 @@ func refreshFiltersIfNecessary(force bool) int {
|
|||
continue
|
||||
}
|
||||
|
||||
if !force && time.Since(f.LastUpdated) <= updatePeriod {
|
||||
expireTime := f.LastUpdated.Unix() + int64(config.DNS.FiltersUpdateIntervalHours)*60*60
|
||||
if !force && expireTime > now.Unix() {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +238,7 @@ func refreshFiltersIfNecessary(force bool) int {
|
|||
log.Printf("Failed to update filter %s: %s\n", uf.URL, err)
|
||||
continue
|
||||
}
|
||||
uf.LastUpdated = time.Now()
|
||||
uf.LastUpdated = now
|
||||
if updated {
|
||||
updateCount++
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
package home
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFilters(t *testing.T) {
|
||||
config.client = &http.Client{
|
||||
Timeout: time.Minute * 5,
|
||||
}
|
||||
|
||||
f := filter{
|
||||
URL: "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt",
|
||||
}
|
||||
|
||||
// download
|
||||
ok, err := f.update()
|
||||
assert.True(t, ok && err == nil)
|
||||
|
||||
// refresh
|
||||
ok, err = f.update()
|
||||
assert.True(t, !ok && err == nil)
|
||||
|
||||
err = f.save()
|
||||
assert.True(t, err == nil)
|
||||
|
||||
err = f.load()
|
||||
assert.True(t, err == nil)
|
||||
|
||||
f.unload()
|
||||
os.Remove(f.Path())
|
||||
}
|
||||
14
home/home.go
14
home/home.go
|
|
@ -135,22 +135,17 @@ func run(args options) {
|
|||
config.BindPort = args.bindPort
|
||||
}
|
||||
|
||||
loadFilters()
|
||||
|
||||
if !config.firstRun {
|
||||
// Save the updated config
|
||||
err := config.write()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Init the DNS server instance before registering HTTP handlers
|
||||
dnsBaseDir := filepath.Join(config.ourWorkingDir, dataDir)
|
||||
initDNSServer(dnsBaseDir)
|
||||
|
||||
if !config.firstRun {
|
||||
err := startDNSServer()
|
||||
err = startDNSServer()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -165,13 +160,6 @@ func run(args options) {
|
|||
config.pidFileName = args.pidFile
|
||||
}
|
||||
|
||||
// Update filters we've just loaded right away, don't wait for periodic update timer
|
||||
go func() {
|
||||
refreshFiltersIfNecessary(false)
|
||||
}()
|
||||
// Schedule automatic filters updates
|
||||
go periodicallyRefreshFilters()
|
||||
|
||||
// Initialize and run the admin Web interface
|
||||
box := packr.NewBox("../build/static")
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ type queryLog struct {
|
|||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
// newQueryLog creates a new instance of the query log
|
||||
// create a new instance of the query log
|
||||
func newQueryLog(conf Config) *queryLog {
|
||||
l := queryLog{}
|
||||
l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
|
||||
|
|
@ -53,7 +53,6 @@ func (l *queryLog) Configure(conf Config) {
|
|||
l.conf = conf
|
||||
}
|
||||
|
||||
// Clear memory buffer and remove the file
|
||||
func (l *queryLog) Clear() {
|
||||
l.fileFlushLock.Lock()
|
||||
defer l.fileFlushLock.Unlock()
|
||||
|
|
@ -164,7 +163,6 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res
|
|||
}
|
||||
}
|
||||
|
||||
// getQueryLogJson returns a map with the current query log ready to be converted to a JSON
|
||||
func (l *queryLog) GetData() []map[string]interface{} {
|
||||
l.lock.RLock()
|
||||
values := make([]*logEntry, len(l.cache))
|
||||
|
|
|
|||
|
|
@ -10,14 +10,20 @@ import (
|
|||
|
||||
// QueryLog - main interface
|
||||
type QueryLog interface {
|
||||
// Close query log object
|
||||
Close()
|
||||
|
||||
// Set new configuration at runtime
|
||||
// Currently only 'Interval' field is supported.
|
||||
Configure(conf Config)
|
||||
|
||||
// Add a log entry
|
||||
Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
|
||||
|
||||
// Get log entries
|
||||
GetData() []map[string]interface{}
|
||||
|
||||
// Clear memory buffer and remove log files
|
||||
Clear()
|
||||
}
|
||||
|
||||
|
|
@ -27,7 +33,7 @@ type Config struct {
|
|||
Interval uint32 // interval to rotate logs (in hours)
|
||||
}
|
||||
|
||||
// New - create instance
|
||||
// New - create a new instance of the query log
|
||||
func New(conf Config) QueryLog {
|
||||
return newQueryLog(conf)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue