Merge: + filters: support filters from local disk

Close #817

* commit '81309c707a70ff15a7075fb5906962764b470ca7':
  + filters: support filters from local disk

AGHTechDoc.md

@@ -1439,7 +1439,7 @@ Request:
 
 	{
 		"name": "..."
-		"url": "..."
+		"url": "..." // URL or an absolute file path
 		"whitelist": true
 	}
 

home/control_filtering.go

@@ -8,15 +8,22 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/util"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/miekg/dns"
 )
 
-// IsValidURL - return TRUE if URL is valid
+// IsValidURL - return TRUE if URL or file path is valid
 func IsValidURL(rawurl string) bool {
+	if filepath.IsAbs(rawurl) {
+		// this is a file path
+		return util.FileExists(rawurl)
+	}
+
 	url, err := url.ParseRequestURI(rawurl)
 	if err != nil {
 		return false //Couldn't even parse the rawurl
@@ -42,7 +49,7 @@ func (f *Filtering) handleFilteringAddURL(w http.ResponseWriter, r *http.Request
 	}
 
 	if !IsValidURL(fj.URL) {
-		http.Error(w, "Invalid URL", http.StatusBadRequest)
+		http.Error(w, "Invalid URL or file path", http.StatusBadRequest)
 		return
 	}
 
@@ -100,11 +107,6 @@ func (f *Filtering) handleFilteringRemoveURL(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if !IsValidURL(req.URL) {
-		http.Error(w, "URL parameter is not valid request URL", http.StatusBadRequest)
-		return
-	}
-
 	// go through each element and delete if url matches
 	config.Lock()
 	newFilters := []filter{}
@@ -154,7 +156,7 @@ func (f *Filtering) handleFilteringSetURL(w http.ResponseWriter, r *http.Request
 	}
 
 	if !IsValidURL(fj.URL) {
-		http.Error(w, "invalid URL", http.StatusBadRequest)
+		http.Error(w, "invalid URL or file path", http.StatusBadRequest)
 		return
 	}
 

home/filter.go

@@ -69,7 +69,7 @@ func defaultFilters() []filter {
 // field ordering is important -- yaml fields will mirror ordering from here
 type filter struct {
 	Enabled     bool
-	URL         string
+	URL         string    // URL or a file path
 	Name        string    `yaml:"name"`
 	RulesCount  int       `yaml:"-"`
 	LastUpdated time.Time `yaml:"-"`
@@ -500,6 +500,7 @@ func (f *Filtering) update(filter *filter) (bool, error) {
 	return b, err
 }
 
+// nolint(gocyclo)
 func (f *Filtering) updateIntl(filter *filter) (bool, error) {
 	log.Tracef("Downloading update for filter %d from %s", filter.ID, filter.URL)
 
@@ -514,18 +515,29 @@ func (f *Filtering) updateIntl(filter *filter) (bool, error) {
 		}
 	}()
 
-	resp, err := Context.client.Get(filter.URL)
+	var reader io.Reader
-	if resp != nil && resp.Body != nil {
+	if filepath.IsAbs(filter.URL) {
-		defer resp.Body.Close()
+		f, err := os.Open(filter.URL)
-	}
+		if err != nil {
-	if err != nil {
+			return false, fmt.Errorf("open file: %s", err)
-		log.Printf("Couldn't request filter from URL %s, skipping: %s", filter.URL, err)
+		}
-		return false, err
+		defer f.Close()
-	}
+		reader = f
+	} else {
+		resp, err := Context.client.Get(filter.URL)
+		if resp != nil && resp.Body != nil {
+			defer resp.Body.Close()
+		}
+		if err != nil {
+			log.Printf("Couldn't request filter from URL %s, skipping: %s", filter.URL, err)
+			return false, err
+		}
 
-	if resp.StatusCode != 200 {
+		if resp.StatusCode != 200 {
-		log.Printf("Got status code %d from URL %s, skipping", resp.StatusCode, filter.URL)
+			log.Printf("Got status code %d from URL %s, skipping", resp.StatusCode, filter.URL)
-		return false, fmt.Errorf("got status code != 200: %d", resp.StatusCode)
+			return false, fmt.Errorf("got status code != 200: %d", resp.StatusCode)
+		}
+		reader = resp.Body
 	}
 
 	htmlTest := true
@@ -534,7 +546,7 @@ func (f *Filtering) updateIntl(filter *filter) (bool, error) {
 	buf := make([]byte, 64*1024)
 	total := 0
 	for {
-		n, err := resp.Body.Read(buf)
+		n, err := reader.Read(buf)
 		total += n
 
 		if htmlTest {

openapi/openapi.yaml

@@ -507,7 +507,7 @@ paths:
             tags:
                 - filtering
             operationId: filteringAddURL
-            summary: 'Add filter URL'
+            summary: 'Add filter URL or an absolute file path'
             consumes:
             - application/json
             parameters:
@@ -1495,7 +1495,7 @@ definitions:
             name:
                 type: "string"
             url:
-                description: "URL containing filtering rules"
+                description: "URL or an absolute path to the file containing filtering rules"
                 type: "string"
                 example: "https://filters.adtidy.org/windows/filters/15.txt"
     RemoveUrlRequest: