From ca418de16f809cd1240eb881a1b55f99e5a54764 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 14 Apr 2020 00:04:38 +0300 Subject: [PATCH 01/10] +(global): windows travis build --- .travis.yml | 44 ++++++++++++++++++++++++++++++++++++++------ util/os_windows.go | 2 +- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57f4cbba..3cf8d23c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,31 @@ go: os: - linux - osx + - windows before_install: - - nvm install node - - npm install -g npm - - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.23.8 + - |- + case $TRAVIS_OS_NAME in + linux | osx) + nvm install node + npm install -g npm + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.23.8 + ;; + windows) + choco install node + ;; + esac install: - - npm --prefix client ci + - |- + case $TRAVIS_OS_NAME in + linux | osx) + npm --prefix client ci + ;; + windows) + npm --prefix client ci + ;; + esac cache: directories: @@ -23,10 +40,23 @@ cache: - $HOME/Library/Caches/go-build script: - - /bin/bash ci.sh + - |- + case $TRAVIS_OS_NAME in + linux | osx) + /bin/bash ci.sh + ;; + windows) + go test -race -v -bench=. -coverprofile=coverage.txt -covermode=atomic ./... + ;; + esac after_success: - - bash <(curl -s https://codecov.io/bash) + - |- + case $TRAVIS_OS_NAME in + linux) + bash <(curl -s https://codecov.io/bash) + ;; + esac notifications: slack: performix:yXTihlSzsLFSZiqbXMNzvTSX @@ -60,6 +90,8 @@ matrix: file_glob: true skip_cleanup: true + # Docker build configuration + - if: repo = AdguardTeam/AdGuardHome - name: docker if: type != pull_request AND (branch = master OR tag IS present) AND repo = AdguardTeam/AdGuardHome go: diff --git a/util/os_windows.go b/util/os_windows.go index 1c09e588..2a3742fa 100644 --- a/util/os_windows.go +++ b/util/os_windows.go @@ -13,7 +13,7 @@ func SetRlimit(val uint) { func HaveAdminRights() (bool, error) { var token windows.Token - h, _ := windows.GetCurrentProcess() + h := windows.CurrentProcess() err := windows.OpenProcessToken(h, windows.TOKEN_QUERY, &token) if err != nil { return false, err From 6cdc59db26be2161c805c8aaeec57c79769fedd7 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 14 Apr 2020 00:08:54 +0300 Subject: [PATCH 02/10] *(global): travis: choco install nodejs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3cf8d23c..ef5b63f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_install: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.23.8 ;; windows) - choco install node + choco install nodejs ;; esac From 814a1f6cb38f9bf55da15f493a7f5024ebe71cb7 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 14 Apr 2020 00:16:58 +0300 Subject: [PATCH 03/10] *: travis-win -- try nvm --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ef5b63f3..ddcf05c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,9 @@ before_install: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.23.8 ;; windows) - choco install nodejs + choco install nvm + nvm install node + npm install -g npm ;; esac From 344a0336b0f10451eb252c451a8c1e6baa836609 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 14 Apr 2020 00:26:44 +0300 Subject: [PATCH 04/10] *: travis - use nvs on Windows --- .travis.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddcf05c0..37dff1d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,12 @@ before_install: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.23.8 ;; windows) - choco install nvm - nvm install node - npm install -g npm + # Using NVS for managing Node.js versions on Windows + NVS_HOME="C:\ProgramData" + git clone --single-branch https://github.com/jasongin/nvs $NVS_HOME + source $NVS_HOME/nvs.sh + nvs add latest + nvs use latest ;; esac @@ -28,9 +31,14 @@ install: - |- case $TRAVIS_OS_NAME in linux | osx) + node --version + npm --version npm --prefix client ci ;; windows) + node --version + npm --version + nvs --version npm --prefix client ci ;; esac From b9b42997af9f50351da6047d625c416421ae68d1 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Tue, 14 Apr 2020 00:30:20 +0300 Subject: [PATCH 05/10] *: fix nvs script g --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 37dff1d6..60abe861 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_install: ;; windows) # Using NVS for managing Node.js versions on Windows - NVS_HOME="C:\ProgramData" + NVS_HOME="C:\ProgramData\nvs" git clone --single-branch https://github.com/jasongin/nvs $NVS_HOME source $NVS_HOME/nvs.sh nvs add latest From e96fd6e42dd1545df6ffbfed02a6e27575d3b5a1 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Wed, 15 Apr 2020 14:11:31 +0300 Subject: [PATCH 06/10] -: fix qlog test on Windows --- home/home_test.go | 2 +- querylog/qlog.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/home/home_test.go b/home/home_test.go index 8a46e493..4e22e1ba 100644 --- a/home/home_test.go +++ b/home/home_test.go @@ -112,9 +112,9 @@ func TestHome(t *testing.T) { dir := prepareTestDir() defer func() { _ = os.RemoveAll(dir) }() - _ = os.MkdirAll(filepath.Join(Context.getDataDir(), filterDir), 0755) fn := filepath.Join(dir, "AdGuardHome.yaml") + // Prepare the test config assert.True(t, ioutil.WriteFile(fn, []byte(yamlConf), 0644) == nil) fn, _ = filepath.Abs(fn) diff --git a/querylog/qlog.go b/querylog/qlog.go index 9dab0acb..f8792d8f 100644 --- a/querylog/qlog.go +++ b/querylog/qlog.go @@ -206,7 +206,9 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} { fileEntries, oldest, total := l.searchFiles(params) if params.OlderThan.IsZero() { - params.OlderThan = now + // In case if the timer is not precise (for instance, on Windows) + // We really want to get all records including those added just before the call + params.OlderThan = now.Add(time.Millisecond) } // add from memory buffer From 4bdf22eadcd5d3451d592edb8db23cea504b9e75 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Wed, 15 Apr 2020 14:36:47 +0300 Subject: [PATCH 07/10] -: fix autohosts tests on Windows --- util/auto_hosts.go | 70 +++++++++++++++++++++++------------------ util/auto_hosts_test.go | 61 +++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 43 deletions(-) diff --git a/util/auto_hosts.go b/util/auto_hosts.go index 0af5bf08..f310528c 100644 --- a/util/auto_hosts.go +++ b/util/auto_hosts.go @@ -23,7 +23,7 @@ type AutoHosts struct { hostsFn string // path to the main hosts-file hostsDirs []string // paths to OS-specific directories with hosts-files watcher *fsnotify.Watcher // file and directory watcher object - updateChan chan bool // signal for 'update' goroutine + updateChan chan bool // signal for 'updateLoop' goroutine onChanged onChangedT // notification to other modules } @@ -68,20 +68,22 @@ func (a *AutoHosts) Init(hostsFn string) { // Start - start module func (a *AutoHosts) Start() { - go a.update() + log.Debug("Start AutoHosts module") + + go a.updateLoop() a.updateChan <- true go a.watcherLoop() err := a.watcher.Add(a.hostsFn) if err != nil { - log.Error("AutoHosts: %s", err) + log.Error("Error while initializing watcher for a file %s: %s", a.hostsFn, err) } for _, dir := range a.hostsDirs { err = a.watcher.Add(dir) if err != nil { - log.Error("AutoHosts: %s", err) + log.Error("Error while initializing watcher for a directory %s: %s", dir, err) } } } @@ -89,7 +91,8 @@ func (a *AutoHosts) Start() { // Close - close module func (a *AutoHosts) Close() { a.updateChan <- false - a.watcher.Close() + close(a.updateChan) + _ = a.watcher.Close() } // Read IP-hostname pairs from file @@ -174,7 +177,7 @@ func (a *AutoHosts) watcherLoop() { log.Debug("AutoHosts: modified: %s", event.Name) select { case a.updateChan <- true: - // sent a signal to 'update' goroutine + // sent a signal to 'updateLoop' goroutine default: // queue is full } @@ -189,41 +192,48 @@ func (a *AutoHosts) watcherLoop() { } } -// Read static hosts from system files -func (a *AutoHosts) update() { +// updateLoop - read static hosts from system files +func (a *AutoHosts) updateLoop() { for { select { case ok := <-a.updateChan: if !ok { + log.Debug("Finished AutoHosts update loop") return } - table := make(map[string][]net.IP) - - a.load(table, a.hostsFn) - - for _, dir := range a.hostsDirs { - fis, err := ioutil.ReadDir(dir) - if err != nil { - if !os.IsNotExist(err) { - log.Error("AutoHosts: Opening directory: %s: %s", dir, err) - } - continue - } - - for _, fi := range fis { - a.load(table, dir+"/"+fi.Name()) - } - } - - a.lock.Lock() - a.table = table - a.lock.Unlock() - a.notify() + a.updateHosts() } } } +// updateHosts - loads system hosts +func (a *AutoHosts) updateHosts() { + table := make(map[string][]net.IP) + + a.load(table, a.hostsFn) + + for _, dir := range a.hostsDirs { + fis, err := ioutil.ReadDir(dir) + if err != nil { + if !os.IsNotExist(err) { + log.Error("AutoHosts: Opening directory: %s: %s", dir, err) + } + continue + } + + for _, fi := range fis { + a.load(table, dir+"/"+fi.Name()) + } + } + + a.lock.Lock() + a.table = table + a.lock.Unlock() + + a.notify() +} + // Process - get the list of IP addresses for the hostname // Return nil if not found func (a *AutoHosts) Process(host string) []net.IP { diff --git a/util/auto_hosts_test.go b/util/auto_hosts_test.go index ce06e361..4aaa519e 100644 --- a/util/auto_hosts_test.go +++ b/util/auto_hosts_test.go @@ -17,38 +17,73 @@ func prepareTestDir() string { return dir } -func TestAutoHosts(t *testing.T) { +func TestAutoHostsResolution(t *testing.T) { ah := AutoHosts{} dir := prepareTestDir() defer func() { _ = os.RemoveAll(dir) }() f, _ := ioutil.TempFile(dir, "") - defer os.Remove(f.Name()) + defer func() { _ = os.Remove(f.Name()) }() defer f.Close() _, _ = f.WriteString(" 127.0.0.1 host localhost \n") - ah.Init(f.Name()) - ah.Start() - // wait until we parse the file - time.Sleep(50 * time.Millisecond) + // Update from the hosts file + ah.updateHosts() + + // Existing host ips := ah.Process("localhost") - assert.True(t, ips[0].Equal(net.ParseIP("127.0.0.1"))) - ips = ah.Process("newhost") - assert.True(t, ips == nil) + assert.NotNil(t, ips) + assert.Equal(t, 1, len(ips)) + assert.Equal(t, net.ParseIP("127.0.0.1"), ips[0]) + // Unknown host + ips = ah.Process("newhost") + assert.Nil(t, ips) + + // Test hosts file table := ah.List() ips, _ = table["host"] - assert.True(t, ips[0].String() == "127.0.0.1") + assert.NotNil(t, ips) + assert.Equal(t, 1, len(ips)) + assert.Equal(t, "127.0.0.1", ips[0].String()) +} +func TestAutoHostsFSNotify(t *testing.T) { + ah := AutoHosts{} + + dir := prepareTestDir() + defer func() { _ = os.RemoveAll(dir) }() + + f, _ := ioutil.TempFile(dir, "") + defer func() { _ = os.Remove(f.Name()) }() + defer f.Close() + + // Init + _, _ = f.WriteString(" 127.0.0.1 host localhost \n") + ah.Init(f.Name()) + ah.updateHosts() + + // Unknown host + ips := ah.Process("newhost") + assert.Nil(t, ips) + + // Stat monitoring for changes + ah.Start() + defer ah.Close() + + // Update file _, _ = f.WriteString("127.0.0.2 newhost\n") + _ = f.Sync() + // wait until fsnotify has triggerred and processed the file-modification event time.Sleep(50 * time.Millisecond) + // Check if we are notified about changes ips = ah.Process("newhost") - assert.True(t, ips[0].Equal(net.ParseIP("127.0.0.2"))) - - ah.Close() + assert.NotNil(t, ips) + assert.Equal(t, 1, len(ips)) + assert.Equal(t, "127.0.0.2", ips[0].String()) } From 1a34b96d032c7e4468571e83d7d2aafcb5cc6e27 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Wed, 15 Apr 2020 15:17:57 +0300 Subject: [PATCH 08/10] *: added more logging --- home/auth.go | 4 +++- home/home.go | 15 +++++++++------ home/home_test.go | 6 +++--- home/web.go | 2 ++ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/home/auth.go b/home/auth.go index c1c00594..c72ca546 100644 --- a/home/auth.go +++ b/home/auth.go @@ -71,6 +71,8 @@ type User struct { // InitAuth - create a global object func InitAuth(dbFilename string, users []User, sessionTTL uint32) *Auth { + log.Info("Initializing auth module: %s", dbFilename) + a := Auth{} a.sessionTTL = sessionTTL a.sessions = make(map[string]*session) @@ -83,7 +85,7 @@ func InitAuth(dbFilename string, users []User, sessionTTL uint32) *Auth { } a.loadSessions() a.users = users - log.Debug("Auth: initialized. users:%d sessions:%d", len(a.users), len(a.sessions)) + log.Info("Auth: initialized. users:%d sessions:%d", len(a.users), len(a.sessions)) return &a } diff --git a/home/home.go b/home/home.go index 63bb308e..98ad8841 100644 --- a/home/home.go +++ b/home/home.go @@ -43,10 +43,10 @@ const ( // Update-related variables var ( - versionString string - updateChannel string - versionCheckURL string - ARMVersion string + versionString = "dev" + updateChannel = "none" + versionCheckURL = "" + ARMVersion = "" ) const versionCheckPeriod = time.Hour * 8 @@ -155,11 +155,11 @@ func run(args options) { configureLogger(args) // print the first message after logger is configured - msg := "AdGuard Home, version %s, channel %s\n, arch %s %s" + msg := "AdGuard Home, version %s, channel %s, arch %s %s" if ARMVersion != "" { msg = msg + " v" + ARMVersion } - log.Printf(msg, versionString, updateChannel, runtime.GOOS, runtime.GOARCH, ARMVersion) + log.Printf(msg, versionString, updateChannel, runtime.GOOS, runtime.GOARCH) log.Debug("Current working directory is %s", Context.workDir) if args.runningAsService { log.Info("AdGuard Home is running as a service") @@ -169,6 +169,7 @@ func run(args options) { Context.firstRun = detectFirstRun() if Context.firstRun { + log.Info("This is the first time AdGuard Home is launched") requireAdminRights() } @@ -197,6 +198,7 @@ func run(args options) { err = parseConfig() if err != nil { + log.Error("Failed to parse configuration, exiting") os.Exit(1) } @@ -211,6 +213,7 @@ func run(args options) { config.DHCP.ConfigModified = onConfigModified Context.dhcpServer = dhcpd.Create(config.DHCP) if Context.dhcpServer == nil { + log.Error("Failed to initialize DHCP server, exiting") os.Exit(1) } Context.autoHosts.Init("") diff --git a/home/home_test.go b/home/home_test.go index 4e22e1ba..89901387 100644 --- a/home/home_test.go +++ b/home/home_test.go @@ -107,7 +107,7 @@ schema_version: 5 // . Wait until the filters are downloaded // . Stop and cleanup func TestHome(t *testing.T) { - // Reinit context + // Init new context Context = homeContext{} dir := prepareTestDir() @@ -135,11 +135,11 @@ func TestHome(t *testing.T) { time.Sleep(100 * time.Millisecond) } assert.Truef(t, err == nil, "%s", err) - assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, http.StatusOK, resp.StatusCode) resp, err = h.Get("http://127.0.0.1:3000/control/status") assert.Truef(t, err == nil, "%s", err) - assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, http.StatusOK, resp.StatusCode) // test DNS over UDP r := upstream.NewResolver("127.0.0.1:5354", 3*time.Second) diff --git a/home/web.go b/home/web.go index 04b2696e..c2145caa 100644 --- a/home/web.go +++ b/home/web.go @@ -42,6 +42,8 @@ type Web struct { // CreateWeb - create module func CreateWeb(conf *WebConfig) *Web { + log.Info("Initialize web module") + w := Web{} w.conf = conf From af34500729b29aedf13df3de09486a403509a9d4 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Wed, 15 Apr 2020 15:24:12 +0300 Subject: [PATCH 09/10] run npm build on windows --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 60abe861..fd369c2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,6 +56,7 @@ script: /bin/bash ci.sh ;; windows) + npm --prefix client run build-prod go test -race -v -bench=. -coverprofile=coverage.txt -covermode=atomic ./... ;; esac From 6503bab1aaa27ff43175d069413630a420093be0 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Wed, 15 Apr 2020 15:41:48 +0300 Subject: [PATCH 10/10] *: added cross-env to npm scripts --- client/package-lock.json | 52 ++++++++++++++++++++++++++++++++++++++++ client/package.json | 11 +++++---- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 0e43676f..0f020587 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3522,6 +3522,58 @@ "gud": "^1.0.0" } }, + "cross-env": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.2.tgz", + "integrity": "sha512-KZP/bMEOJEDCkDQAyRhu3RL2ZO/SUVrxQVI0G3YEQ+OLbRA3c6zgixe8Mq8a/z7+HKlNEjo8oiLUs8iRijY2Rw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.1" + }, + "dependencies": { + "cross-spawn": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.2.tgz", + "integrity": "sha512-PD6G8QG3S4FK/XCGFbEQrDqO2AnMMsy0meR7lerlIOHAAbkuavGU/pOqprrlvfTNjvowivTeBsjebAL0NSoMxw==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", diff --git a/client/package.json b/client/package.json index 4ee1c1c3..daa3ab38 100644 --- a/client/package.json +++ b/client/package.json @@ -3,9 +3,9 @@ "version": "0.1.0", "private": true, "scripts": { - "build-dev": "NODE_ENV=development ./node_modules/.bin/webpack --config webpack.dev.js", - "watch": "NODE_ENV=development ./node_modules/.bin/webpack --config webpack.dev.js --watch", - "build-prod": "NODE_ENV=production ./node_modules/.bin/webpack --config webpack.prod.js", + "build-dev": "cross-env NODE_ENV=development webpack --config webpack.dev.js", + "watch": "cross-env NODE_ENV=development webpack --config webpack.dev.js --watch", + "build-prod": "cross-env NODE_ENV=production webpack --config webpack.prod.js", "lint": "eslint client/" }, "dependencies": { @@ -49,6 +49,7 @@ "clean-webpack-plugin": "^0.1.19", "compression-webpack-plugin": "^1.1.11", "copy-webpack-plugin": "^4.6.0", + "cross-env": "^7.0.2", "css-loader": "^2.1.1", "eslint": "^4.19.1", "eslint-config-airbnb-base": "^12.1.0", @@ -70,11 +71,11 @@ "style-loader": "^0.21.0", "stylelint": "^9.10.1", "stylelint-webpack-plugin": "0.10.4", + "svg-url-loader": "^2.3.2", "uglifyjs-webpack-plugin": "^1.2.7", "url-loader": "^1.0.1", "webpack": "3.8.1", "webpack-dev-server": "^3.1.14", - "webpack-merge": "^4.1.3", - "svg-url-loader": "^2.3.2" + "webpack-merge": "^4.1.3" } }