Merge: + DNS: "dns.upstream_dns_file" setting

Merge in DNS/adguard-home from 1680-upstreams-file to master
Fix #1680

* commit '88c67764b9515902514b0a8e4f00af29a353c584':
  * (dnsforward): don't fail when default upstream is not set
  * (dnsforward): upgrade dnsproxy to v0.32.1
  * (ui): fix strings
  + client: Display upstreams list loaded from a file
  + DNS: "dns.upstream_dns_file" setting

AGHTechDoc.md

@@ -988,6 +988,7 @@ Response:
 
 	{
 		"upstream_dns": ["tls://...", ...],
+		"upstream_dns_file": "",
 		"bootstrap_dns": ["1.2.3.4", ...],
 
 		"protection_enabled": true | false,
@@ -1013,6 +1014,7 @@ Request:
 
 	{
 		"upstream_dns": ["tls://...", ...],
+		"upstream_dns_file": "",
 		"bootstrap_dns": ["1.2.3.4", ...],
 
 		"protection_enabled": true | false,

client/src/__locales/en.json

@@ -132,7 +132,8 @@
     "encryption_settings": "Encryption settings",
     "dhcp_settings": "DHCP settings",
     "upstream_dns": "Upstream DNS servers",
-    "upstream_dns_hint": "If you keep this field empty, AdGuard Home will use <a href='https://www.quad9.net/' target='_blank'>Quad9</a> as an upstream.",
+    "upstream_dns_help": "Enter servers addresses one per line. <0>Learn more</0> about configuring upstream DNS servers.",
+    "upstream_dns_configured_in_file": "Configured in {{path}}",
     "test_upstream_btn": "Test upstreams",
     "upstreams": "Upstreams",
     "apply_btn": "Apply",

client/src/components/Settings/Dns/Upstream/Form.js

@@ -7,49 +7,56 @@ import classnames from 'classnames';
 
 import Examples from './Examples';
 import { renderRadioField, renderTextareaField } from '../../../../helpers/form';
-import { DNS_REQUEST_OPTIONS, FORM_NAME } from '../../../../helpers/constants';
+import { DNS_REQUEST_OPTIONS, FORM_NAME, UPSTREAM_CONFIGURATION_WIKI_LINK } from '../../../../helpers/constants';
 import { testUpstream } from '../../../../actions';
 import { removeEmptyLines } from '../../../../helpers/helpers';
 
-const getInputFields = () => [{
+const Title = () => <label className="form__label" htmlFor="upstream_dns">
-    // eslint-disable-next-line react/display-name
+    <Trans components={[<a href={UPSTREAM_CONFIGURATION_WIKI_LINK} key="0">link</a>]}>
-    getTitle: () => <label className="form__label" htmlFor="upstream_dns">
+        upstream_dns_help
-        <Trans>upstream_dns</Trans>
+    </Trans>
-    </label>,
+</label>;
-    name: 'upstream_dns',
+
-    type: 'text',
+const getInputFields = (upstream_dns_file) => [
-    component: renderTextareaField,
+    {
-    className: 'form-control form-control--textarea font-monospace',
+        getTitle: Title,
-    placeholder: 'upstream_dns',
+        name: 'upstream_dns',
-    normalizeOnBlur: removeEmptyLines,
+        type: 'text',
-},
+        value: 'test',
-{
+        component: renderTextareaField,
-    name: 'upstream_mode',
+        className: 'form-control form-control--textarea font-monospace',
-    type: 'radio',
+        placeholder: 'upstream_dns',
-    value: DNS_REQUEST_OPTIONS.LOAD_BALANCING,
+        normalizeOnBlur: removeEmptyLines,
-    component: renderRadioField,
+        disabled: !!upstream_dns_file,
-    subtitle: 'load_balancing_desc',
+    },
-    placeholder: 'load_balancing',
+    {
-},
+        name: 'upstream_mode',
-{
+        type: 'radio',
-    name: 'upstream_mode',
+        value: DNS_REQUEST_OPTIONS.LOAD_BALANCING,
-    type: 'radio',
+        component: renderRadioField,
-    value: DNS_REQUEST_OPTIONS.PARALLEL,
+        subtitle: 'load_balancing_desc',
-    component: renderRadioField,
+        placeholder: 'load_balancing',
-    subtitle: 'upstream_parallel',
+    },
-    placeholder: 'parallel_requests',
+    {
-},
+        name: 'upstream_mode',
-{
+        type: 'radio',
-    name: 'upstream_mode',
+        value: DNS_REQUEST_OPTIONS.PARALLEL,
-    type: 'radio',
+        component: renderRadioField,
-    value: DNS_REQUEST_OPTIONS.FASTEST_ADDR,
+        subtitle: 'upstream_parallel',
-    component: renderRadioField,
+        placeholder: 'parallel_requests',
-    subtitle: 'fastest_addr_desc',
+    },
-    placeholder: 'fastest_addr',
+    {
-}];
+        name: 'upstream_mode',
+        type: 'radio',
+        value: DNS_REQUEST_OPTIONS.FASTEST_ADDR,
+        component: renderRadioField,
+        subtitle: 'fastest_addr_desc',
+        placeholder: 'fastest_addr',
+    },
+];
 
 const Form = ({
-    submitting, invalid, processingSetConfig, processingTestUpstream, handleSubmit,
+    submitting, invalid, handleSubmit,
 }) => {
     const dispatch = useDispatch();
     const { t } = useTranslation();
@@ -57,6 +64,9 @@ const Form = ({
     const bootstrap_dns = useSelector(
         (store) => store.form[FORM_NAME.UPSTREAM].values.bootstrap_dns,
     );
+    const upstream_dns_file = useSelector((state) => state.dnsConfig.upstream_dns_file);
+    const processingTestUpstream = useSelector((state) => state.settings.processingTestUpstream);
+    const processingSetConfig = useSelector((state) => state.dnsConfig.processingSetConfig);
 
     const handleUpstreamTest = () => dispatch(testUpstream({
         upstream_dns,
@@ -67,7 +77,7 @@ const Form = ({
         'btn-loading': processingTestUpstream,
     });
 
-    const INPUT_FIELDS = getInputFields();
+    const INPUT_FIELDS = getInputFields(upstream_dns_file);
 
     return <form onSubmit={handleSubmit}>
         <div className="row">
@@ -146,8 +156,6 @@ Form.propTypes = {
     initialValues: PropTypes.object,
     upstream_dns: PropTypes.string,
     bootstrap_dns: PropTypes.string,
-    processingTestUpstream: PropTypes.bool,
-    processingSetConfig: PropTypes.bool,
 };
 
 export default reduxForm({ form: FORM_NAME.UPSTREAM })(Form);

client/src/components/Settings/Dns/Upstream/index.js

@@ -12,31 +12,41 @@ const Upstream = () => {
         upstream_dns,
         bootstrap_dns,
         upstream_mode,
-        processingSetConfig,
     } = useSelector((state) => state.dnsConfig, shallowEqual);
 
-    const { processingTestUpstream } = useSelector((state) => state.settings, shallowEqual);
+    const upstream_dns_file = useSelector((state) => state.dnsConfig.upstream_dns_file);
 
     const handleSubmit = (values) => {
-        dispatch(setDnsConfig(values));
+        const {
+            bootstrap_dns,
+            upstream_dns,
+            upstream_mode,
+        } = values;
+
+        const dnsConfig = {
+            bootstrap_dns,
+            upstream_mode,
+            ...(upstream_dns_file ? null : { upstream_dns }),
+        };
+
+        dispatch(setDnsConfig(dnsConfig));
     };
 
+    const upstreamDns = upstream_dns_file ? t('upstream_dns_configured_in_file', { path: upstream_dns_file }) : upstream_dns;
+
     return <Card
         title={t('upstream_dns')}
-        subtitle={t('upstream_dns_hint')}
         bodyType="card-body box-body--settings"
     >
         <div className="row">
             <div className="col">
                 <Form
                     initialValues={{
-                        upstream_dns,
+                        upstream_dns: upstreamDns,
                         bootstrap_dns,
                         upstream_mode,
                     }}
                     onSubmit={handleSubmit}
-                    processingTestUpstream={processingTestUpstream}
-                    processingSetConfig={processingSetConfig}
                 />
             </div>
         </div>

client/src/helpers/constants.js

@@ -52,6 +52,7 @@ export const REPOSITORY = {
 
 export const PRIVACY_POLICY_LINK = 'https://adguard.com/privacy/home.html';
 export const PORT_53_FAQ_LINK = 'https://github.com/AdguardTeam/AdGuardHome/wiki/FAQ#bindinuse';
+export const UPSTREAM_CONFIGURATION_WIKI_LINK = 'https://github.com/AdguardTeam/AdGuardHome/wiki/Configuration#upstreams';
 
 export const GETTING_STARTED_LINK = 'https://github.com/AdguardTeam/AdGuardHome/wiki/Getting-Started#update';
 

client/src/reducers/dnsConfig.js

@@ -48,6 +48,7 @@ const dnsConfig = handleActions(
         edns_cs_enabled: false,
         disable_ipv6: false,
         dnssec_enabled: false,
+        upstream_dns_file: '',
     },
 );
 

dnsforward/config.go

@@ -5,11 +5,13 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"sort"
 
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
+	"github.com/AdguardTeam/AdGuardHome/util"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/log"
@@ -54,10 +56,11 @@ type FilteringConfig struct {
 	// Upstream DNS servers configuration
 	// --
 
-	UpstreamDNS  []string `yaml:"upstream_dns"`
+	UpstreamDNS         []string `yaml:"upstream_dns"`
-	BootstrapDNS []string `yaml:"bootstrap_dns"` // a list of bootstrap DNS for DoH and DoT (plain DNS only)
+	UpstreamDNSFileName string   `yaml:"upstream_dns_file"`
-	AllServers   bool     `yaml:"all_servers"`   // if true, parallel queries to all configured upstream servers are enabled
+	BootstrapDNS        []string `yaml:"bootstrap_dns"` // a list of bootstrap DNS for DoH and DoT (plain DNS only)
-	FastestAddr  bool     `yaml:"fastest_addr"`  // use Fastest Address algorithm
+	AllServers          bool     `yaml:"all_servers"`   // if true, parallel queries to all configured upstream servers are enabled
+	FastestAddr         bool     `yaml:"fastest_addr"`  // use Fastest Address algorithm
 
 	// Access settings
 	// --
@@ -184,7 +187,7 @@ func (s *Server) createProxyConfig() (proxy.Config, error) {
 
 	// Validate proxy config
 	if proxyConfig.UpstreamConfig == nil || len(proxyConfig.UpstreamConfig.Upstreams) == 0 {
-		return proxyConfig, errors.New("no upstream servers configured")
+		return proxyConfig, errors.New("no default upstream servers configured")
 	}
 
 	return proxyConfig, nil
@@ -227,10 +230,36 @@ func (s *Server) prepareUpstreamSettings() error {
 		upstream.CipherSuites = s.conf.TLSCiphers
 	}
 
-	upstreamConfig, err := proxy.ParseUpstreamsConfig(s.conf.UpstreamDNS, s.conf.BootstrapDNS, DefaultTimeout)
+	// Load upstreams either from the file, or from the settings
+	var upstreams []string
+	if s.conf.UpstreamDNSFileName != "" {
+		data, err := ioutil.ReadFile(s.conf.UpstreamDNSFileName)
+		if err != nil {
+			return err
+		}
+		d := string(data)
+		for len(d) != 0 {
+			s := util.SplitNext(&d, '\n')
+			upstreams = append(upstreams, s)
+		}
+		log.Debug("DNS: using %d upstream servers from file %s", len(upstreams), s.conf.UpstreamDNSFileName)
+	} else {
+		upstreams = s.conf.UpstreamDNS
+	}
+	upstreamConfig, err := proxy.ParseUpstreamsConfig(upstreams, s.conf.BootstrapDNS, DefaultTimeout)
 	if err != nil {
 		return fmt.Errorf("DNS: proxy.ParseUpstreamsConfig: %s", err)
 	}
+
+	if len(upstreamConfig.Upstreams) == 0 {
+		log.Info("Warning: no default upstream servers specified, using %v", defaultDNS)
+		uc, err := proxy.ParseUpstreamsConfig(defaultDNS, s.conf.BootstrapDNS, DefaultTimeout)
+		if err != nil {
+			return fmt.Errorf("DNS: failed to parse default upstreams: %v", err)
+		}
+		upstreamConfig.Upstreams = uc.Upstreams
+	}
+
 	s.conf.UpstreamConfig = &upstreamConfig
 	return nil
 }

dnsforward/dnsforward_http.go

@@ -22,8 +22,9 @@ func httpError(r *http.Request, w http.ResponseWriter, code int, format string,
 }
 
 type dnsConfigJSON struct {
-	Upstreams  []string `json:"upstream_dns"`
+	Upstreams     []string `json:"upstream_dns"`
-	Bootstraps []string `json:"bootstrap_dns"`
+	UpstreamsFile string   `json:"upstream_dns_file"`
+	Bootstraps    []string `json:"bootstrap_dns"`
 
 	ProtectionEnabled bool   `json:"protection_enabled"`
 	RateLimit         uint32 `json:"ratelimit"`
@@ -43,6 +44,7 @@ func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) {
 	resp := dnsConfigJSON{}
 	s.RLock()
 	resp.Upstreams = stringArrayDup(s.conf.UpstreamDNS)
+	resp.UpstreamsFile = s.conf.UpstreamDNSFileName
 	resp.Bootstraps = stringArrayDup(s.conf.BootstrapDNS)
 
 	resp.ProtectionEnabled = s.conf.ProtectionEnabled
@@ -157,6 +159,11 @@ func (s *Server) handleSetConfig(w http.ResponseWriter, r *http.Request) {
 		restart = true
 	}
 
+	if js.Exists("upstream_dns_file") {
+		s.conf.UpstreamDNSFileName = req.UpstreamsFile
+		restart = true
+	}
+
 	if js.Exists("bootstrap_dns") {
 		s.conf.BootstrapDNS = req.Bootstraps
 		restart = true

go.mod

@@ -3,7 +3,7 @@ module github.com/AdguardTeam/AdGuardHome
 go 1.14
 
 require (
-	github.com/AdguardTeam/dnsproxy v0.32.0
+	github.com/AdguardTeam/dnsproxy v0.32.1
 	github.com/AdguardTeam/golibs v0.4.2
 	github.com/AdguardTeam/urlfilter v0.12.2
 	github.com/NYTimes/gziphandler v1.1.1

go.sum

@@ -7,8 +7,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/AdguardTeam/dnsproxy v0.32.0 h1:taULDOMubiQSvRLynn8GlfMunhKaVryCBd/OkM++YFU=
+github.com/AdguardTeam/dnsproxy v0.32.1 h1:UoiFt/aT8YCBFUGe7hG8ehLRXyvoIf22mOQqeIQxhWI=
-github.com/AdguardTeam/dnsproxy v0.32.0/go.mod h1:ZLDrKIypYxBDz2N9FQHgeehuHrwTbuhZXdGwNySshbw=
+github.com/AdguardTeam/dnsproxy v0.32.1/go.mod h1:ZLDrKIypYxBDz2N9FQHgeehuHrwTbuhZXdGwNySshbw=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.4.2 h1:7M28oTZFoFwNmp8eGPb3ImmYbxGaJLyQXeIFVHjME0o=
 github.com/AdguardTeam/golibs v0.4.2/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=

openapi/openapi.yaml

@@ -997,6 +997,8 @@ components:
                     example:
                         - tls://1.1.1.1
                         - tls://1.0.0.1
+                upstream_dns_file:
+                    type: string
                 protection_enabled:
                     type: boolean
                 dhcp_available: