diff --git a/AGHTechDoc.md b/AGHTechDoc.md index af62a4f2..c686ecf7 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -1405,6 +1405,10 @@ Request: POST /control/filtering/refresh + { + "whitelist": true + } + Response: 200 OK diff --git a/client/src/actions/filtering.js b/client/src/actions/filtering.js index e1ef2219..e3c2de11 100644 --- a/client/src/actions/filtering.js +++ b/client/src/actions/filtering.js @@ -113,11 +113,11 @@ export const refreshFiltersRequest = createAction('FILTERING_REFRESH_REQUEST'); export const refreshFiltersFailure = createAction('FILTERING_REFRESH_FAILURE'); export const refreshFiltersSuccess = createAction('FILTERING_REFRESH_SUCCESS'); -export const refreshFilters = () => async (dispatch) => { +export const refreshFilters = config => async (dispatch) => { dispatch(refreshFiltersRequest()); dispatch(showLoading()); try { - const data = await apiClient.refreshFilters(); + const data = await apiClient.refreshFilters(config); const { updated } = data; dispatch(refreshFiltersSuccess()); diff --git a/client/src/api/Api.js b/client/src/api/Api.js index a5caaa1c..94a05b19 100644 --- a/client/src/api/Api.js +++ b/client/src/api/Api.js @@ -89,9 +89,14 @@ class Api { return this.makeRequest(path, method); } - refreshFilters() { + refreshFilters(config) { const { path, method } = this.FILTERING_REFRESH; - return this.makeRequest(path, method); + const parameters = { + data: config, + headers: { 'Content-Type': 'application/json' }, + }; + + return this.makeRequest(path, method, parameters); } addFilter(config) { diff --git a/client/src/components/Filters/DnsAllowlist.js b/client/src/components/Filters/DnsAllowlist.js index a2eb57ac..da397af5 100644 --- a/client/src/components/Filters/DnsAllowlist.js +++ b/client/src/components/Filters/DnsAllowlist.js @@ -40,11 +40,14 @@ class DnsAllowlist extends Component { this.props.toggleFilterStatus(url, data, whitelist); }; + handleRefresh = () => { + this.props.refreshFilters({ whitelist: true }); + }; + render() { const { t, toggleFilteringModal, - refreshFilters, addFilter, toggleFilterStatus, filtering: { @@ -89,7 +92,7 @@ class DnsAllowlist extends Component { /> toggleFilteringModal({ type: MODAL_TYPE.ADD })} - handleRefresh={refreshFilters} + handleRefresh={this.handleRefresh} processingRefreshFilters={processingRefreshFilters} whitelist={whitelist} /> diff --git a/client/src/components/Filters/DnsBlocklist.js b/client/src/components/Filters/DnsBlocklist.js index 075749d3..e5d182e7 100644 --- a/client/src/components/Filters/DnsBlocklist.js +++ b/client/src/components/Filters/DnsBlocklist.js @@ -37,11 +37,14 @@ class DnsBlocklist extends Component { this.props.toggleFilterStatus(url, data); }; + handleRefresh = () => { + this.props.refreshFilters({ whitelist: false }); + }; + render() { const { t, toggleFilteringModal, - refreshFilters, addFilter, filtering: { filters, @@ -82,7 +85,7 @@ class DnsBlocklist extends Component { /> toggleFilteringModal({ type: MODAL_TYPE.ADD })} - handleRefresh={refreshFilters} + handleRefresh={this.handleRefresh} processingRefreshFilters={processingRefreshFilters} /> diff --git a/home/control_filtering.go b/home/control_filtering.go index 337173ef..73bd9ca8 100644 --- a/home/control_filtering.go +++ b/home/control_filtering.go @@ -185,17 +185,9 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) { } onConfigModified() - if (status & statusURLChanged) != 0 { - if fj.Data.Enabled { - // download new filter and apply its rules - refreshStatus = 1 - refreshLock.Lock() - _, _ = refreshFiltersIfNecessary(true) - refreshLock.Unlock() - } - - } else if (status & statusEnabledChanged) != 0 { - enableFilters(true) + if (status&(statusURLChanged|statusEnabledChanged)) != 0 && fj.Data.Enabled { + // download new filter and apply its rules + _, _ = refreshFilters(fj.Whitelist, true) } } @@ -217,14 +209,24 @@ func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) { } func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) { + type Req struct { + White bool `json:"whitelist"` + } type Resp struct { Updated int `json:"updated"` } resp := Resp{} var err error + req := Req{} + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + httpError(w, http.StatusBadRequest, "json decode: %s", err) + return + } + Context.controlLock.Unlock() - resp.Updated, err = refreshFilters() + resp.Updated, err = refreshFilters(req.White, false) Context.controlLock.Lock() if err != nil { httpError(w, http.StatusInternalServerError, "%s", err) diff --git a/home/filter.go b/home/filter.go index b54793d9..d5791ef3 100644 --- a/home/filter.go +++ b/home/filter.go @@ -237,7 +237,7 @@ func periodicallyRefreshFilters() { isNetworkErr := false if config.DNS.FiltersUpdateIntervalHours != 0 && atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) { refreshLock.Lock() - _, isNetworkErr = refreshFiltersIfNecessary(false) + _, isNetworkErr = refreshFiltersIfNecessary(FilterRefreshBlocklists | FilterRefreshAllowlists) refreshLock.Unlock() refreshStatus = 0 if !isNetworkErr { @@ -257,45 +257,33 @@ func periodicallyRefreshFilters() { } // Refresh filters -func refreshFilters() (int, error) { - if !atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) { +// important: +// TRUE: ignore the fact that we're currently updating the filters +func refreshFilters(whitelist bool, important bool) (int, error) { + set := atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) + if !important && !set { return 0, fmt.Errorf("Filters update procedure is already running") } refreshLock.Lock() - nUpdated, _ := refreshFiltersIfNecessary(true) + flags := FilterRefreshBlocklists + if whitelist { + flags = FilterRefreshAllowlists + } + nUpdated, _ := refreshFiltersIfNecessary(flags | FilterRefreshForce) refreshLock.Unlock() refreshStatus = 0 return nUpdated, nil } -// Checks filters updates if necessary -// If force is true, it ignores the filter.LastUpdated field value -// -// Algorithm: -// . Get the list of filters to be updated -// . For each filter run the download and checksum check operation -// . For each filter: -// . If filter data hasn't changed, just set new update time on file -// . If filter data has changed: -// . rename the old file (1.txt -> 1.txt.old) -// . store the new data on disk (1.txt) -// . Pass new filters to dnsfilter object - it analyzes new data while the old filters are still active -// . dnsfilter activates new filters -// . Remove the old filter files (1.txt.old) -// -// Return the number of updated filters -// Return TRUE - there was a network error and nothing could be updated -func refreshFiltersIfNecessary(force bool) (int, bool) { +func refreshFiltersArray(filters *[]filter, force bool) (int, []filter, []bool, bool) { var updateFilters []filter var updateFlags []bool // 'true' if filter data has changed - log.Debug("Filters: updating...") - now := time.Now() config.RLock() - for i := range config.Filters { - f := &config.Filters[i] // otherwise we will be operating on a copy + for i := range *filters { + f := &(*filters)[i] // otherwise we will be operating on a copy if !f.Enabled { continue @@ -316,7 +304,7 @@ func refreshFiltersIfNecessary(force bool) (int, bool) { config.RUnlock() if len(updateFilters) == 0 { - return 0, false + return 0, nil, nil, false } nfail := 0 @@ -333,7 +321,7 @@ func refreshFiltersIfNecessary(force bool) (int, bool) { } if nfail == len(updateFilters) { - return 0, true + return 0, nil, nil, true } updateCount := 0 @@ -354,8 +342,8 @@ func refreshFiltersIfNecessary(force bool) (int, bool) { } config.Lock() - for k := range config.Filters { - f := &config.Filters[k] + for k := range *filters { + f := &(*filters)[k] if f.ID != uf.ID || f.URL != uf.URL { continue } @@ -375,6 +363,61 @@ func refreshFiltersIfNecessary(force bool) (int, bool) { config.Unlock() } + return updateCount, updateFilters, updateFlags, false +} + +const ( + FilterRefreshForce = 1 // ignore last file modification date + FilterRefreshAllowlists = 2 // update allow-lists + FilterRefreshBlocklists = 4 // update block-lists +) + +// Checks filters updates if necessary +// If force is true, it ignores the filter.LastUpdated field value +// flags: FilterRefresh* +// +// Algorithm: +// . Get the list of filters to be updated +// . For each filter run the download and checksum check operation +// . For each filter: +// . If filter data hasn't changed, just set new update time on file +// . If filter data has changed: +// . rename the old file (1.txt -> 1.txt.old) +// . store the new data on disk (1.txt) +// . Pass new filters to dnsfilter object - it analyzes new data while the old filters are still active +// . dnsfilter activates new filters +// . Remove the old filter files (1.txt.old) +// +// Return the number of updated filters +// Return TRUE - there was a network error and nothing could be updated +func refreshFiltersIfNecessary(flags int) (int, bool) { + log.Debug("Filters: updating...") + + updateCount := 0 + var updateFilters []filter + var updateFlags []bool + netError := false + netErrorW := false + force := false + if (flags & FilterRefreshForce) != 0 { + force = true + } + if (flags & FilterRefreshBlocklists) != 0 { + updateCount, updateFilters, updateFlags, netError = refreshFiltersArray(&config.Filters, force) + } + if (flags & FilterRefreshAllowlists) != 0 { + updateCountW := 0 + var updateFiltersW []filter + var updateFlagsW []bool + updateCountW, updateFiltersW, updateFlagsW, netErrorW = refreshFiltersArray(&config.WhitelistFilters, force) + updateCount += updateCountW + updateFilters = append(updateFilters, updateFiltersW...) + updateFlags = append(updateFlags, updateFlagsW...) + } + if netError && netErrorW { + return 0, true + } + if updateCount != 0 { enableFilters(false) diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index d611cb27..93a31acf 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -1,6 +1,21 @@ # AdGuard Home API Change Log +## v0.101: API changes + +### API: Refresh filters: POST /control/filtering/refresh + +* Added "whitelist" boolean parameter + +Request: + + POST /control/filtering/refresh + + { + "whitelist": true + } + + ## v0.100: API changes ### API: Get list of clients: GET /control/clients diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 71556333..d2ee5a8b 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -570,6 +570,13 @@ paths: This should work as intended, a `force` parameter is offered as last-resort attempt to make filter lists fresh. If you ever find yourself using `force` to make something work that otherwise wont, this is a bug and report it accordingly. + consumes: + - application/json + parameters: + - in: "body" + name: "body" + schema: + $ref: "#/definitions/FilterRefreshRequest" responses: 200: description: OK @@ -1196,6 +1203,13 @@ definitions: enabled: type: "boolean" + FilterRefreshRequest: + type: "object" + description: "Refresh Filters request data" + properties: + whitelist: + type: "boolean" + FilterCheckHostResponse: type: "object" description: "Check Host Result"