Merge: * whois: improvements
Close #1035, Close #1036, Close #1047 * commit 'bd29b22f17d5b326d9fd603aa4f74593a0f1fca7': * CI: disable check if Git repo contains uncommited changes - whois: use the first "descr" field, not the last * filter: speed up parsing * whois: use whois.arin.net + whois, rdns: begin getting info for the most active clients on startup + stats: GetTopData() * stats: refactor
This commit is contained in:
Simon Zolin
commit
f4451dca7b
16
ci.sh
16
ci.sh
|
|
@ -24,13 +24,13 @@ make clean
|
|||
make build/static/index.html
|
||||
make
|
||||
|
||||
if [[ -z "$(git status --porcelain)" ]]; then
|
||||
# Working directory clean
|
||||
echo "Git status is clean"
|
||||
else
|
||||
echo "Git status is not clean and contains uncommited changes"
|
||||
echo "Please make sure there are no changes"
|
||||
exit 1
|
||||
fi
|
||||
# if [[ -z "$(git status --porcelain)" ]]; then
|
||||
# # Working directory clean
|
||||
# echo "Git status is clean"
|
||||
# else
|
||||
# echo "Git status is not clean and contains uncommited changes"
|
||||
# echo "Please make sure there are no changes"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
echo "AdGuard Home CI script finished successfully"
|
||||
2
go.mod
2
go.mod
|
|
@ -14,8 +14,8 @@ require (
|
|||
github.com/joomcode/errorx v1.0.0
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
|
||||
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
|
||||
github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49
|
||||
github.com/miekg/dns v1.1.19
|
||||
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
|
||||
github.com/stretchr/testify v1.4.0
|
||||
|
|
|
|||
27
go.sum
27
go.sum
|
|
@ -1,19 +1,16 @@
|
|||
github.com/AdguardTeam/dnsproxy v0.19.4 h1:rZb40VUr/yN8RG4j3+NuGqODmPvte7acPfSDl0j2wiU=
|
||||
github.com/AdguardTeam/dnsproxy v0.19.4/go.mod h1:NaulY9i279jZwN8QBbvbZnn5HkrjBgJi4hbFY5nW+Kc=
|
||||
github.com/AdguardTeam/dnsproxy v0.19.5 h1:QAKWa2+rTp7GAeOFLMPqIYPS7eglLVEkVLH4kHRbnCQ=
|
||||
github.com/AdguardTeam/dnsproxy v0.19.5/go.mod h1:qEiDndktnVJYYzHiQGKUl8Zm0b7HGpPmYWShAxmqjtw=
|
||||
github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg=
|
||||
github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
|
||||
github.com/AdguardTeam/golibs v0.2.1 h1:jGCnbM5UOUq/GrG+8eLN7Y+OTfEo5F/8L0wq3ur2h4E=
|
||||
github.com/AdguardTeam/golibs v0.2.1/go.mod h1:caAJ5knSHbR6vV6qfRDgAfXVia4hHgLqeztAY4UX0fw=
|
||||
github.com/AdguardTeam/urlfilter v0.5.0 h1:ATzs2Er0BMt7NbZnFJ4UEzt3uIV+rydbQCYqBXNRbJc=
|
||||
github.com/AdguardTeam/urlfilter v0.5.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg=
|
||||
github.com/AdguardTeam/urlfilter v0.6.0 h1:HVPfAsGcHW47HasmqcLNA/VJ41GaR/SzUufuIj70ouA=
|
||||
github.com/AdguardTeam/urlfilter v0.6.0/go.mod h1:y+XdxBdbRG9v7pfjznlvv4Ufi2HTG8D0YMqR22OVy0Y=
|
||||
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
|
||||
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
|
||||
|
|
@ -67,18 +64,13 @@ github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/
|
|||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b h1:vfiqKno48aUndBMjTeWFpCExNnTf2Xnd6d228L4EfTQ=
|
||||
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b/go.mod h1:10UU/bEkzh2iEN6aYzbevY7J6p03KO5siTxQWXMEerg=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 h1:6wnYc2S/lVM7BvR32BM74ph7bPgqMztWopMYKgVyEho=
|
||||
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414/go.mod h1:0AqAH3ZogsCrvrtUpvc6EtVKbc3w6xwZhkvGLuqyi3o=
|
||||
github.com/likexian/gokit v0.0.0-20190309162924-0a377eecf7aa/go.mod h1:QdfYv6y6qPA9pbBA2qXtoT8BMKha6UyNbxWGWl/9Jfk=
|
||||
github.com/likexian/gokit v0.0.0-20190418170008-ace88ad0983b/go.mod h1:KKqSnk/VVSW8kEyO2vVCXoanzEutKdlBAPohmGXkxCk=
|
||||
github.com/likexian/gokit v0.0.0-20190501133040-e77ea8b19cdc/go.mod h1:3kvONayqCaj+UgrRZGpgfXzHdMYCAO0KAt4/8n0L57Y=
|
||||
github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c h1:KByA4IxKqqYwpqzk/P+w1DBpkPbvy3DArTP/U3LSxTQ=
|
||||
github.com/likexian/gokit v0.0.0-20190604165112-68b8a4ba758c/go.mod h1:kn+nTv3tqh6yhor9BC4Lfiu58SmH8NmQ2PmEl+uM6nU=
|
||||
github.com/likexian/simplejson-go v0.0.0-20190409170913-40473a74d76d/go.mod h1:Typ1BfnATYtZ/+/shXfFYLrovhFyuKvzwrdOnIDHlmg=
|
||||
github.com/likexian/simplejson-go v0.0.0-20190419151922-c1f9f0b4f084/go.mod h1:U4O1vIJvIKwbMZKUJ62lppfdvkCdVd2nfMimHK81eec=
|
||||
github.com/likexian/simplejson-go v0.0.0-20190502021454-d8787b4bfa0b/go.mod h1:3BWwtmKP9cXWwYCr5bkoVDEfLywacOv0s06OBEDpyt8=
|
||||
github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49 h1:xGa+flE6p2UnMgxIS8bm7Q9JSt47HRuYVtwneDVnfLk=
|
||||
github.com/likexian/whois-go v0.0.0-20190627090909-384b3df3fc49/go.mod h1:oR3bJMzrOb55cqTAn14DEzYFLDpSPTXJ3ORe7go9Hc8=
|
||||
github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4 h1:Mlji5gkcpzkqTROyE4ZxZ8hN7osunMb2RuGVrbvMvCc=
|
||||
github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
|
||||
github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
|
||||
|
|
@ -95,6 +87,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYe
|
|||
github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
|
||||
github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
|
||||
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v2.19.9+incompatible h1:IrPVlK4nfwW10DF7pW+7YJKws9NkgNzWozwwWv9FsgY=
|
||||
github.com/shirou/gopsutil v2.19.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
|
|
@ -120,8 +113,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72
|
|||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191001170739-f9e2070545dc h1:KyTYo8xkh/2WdbFLUyQwBS0Jfn3qfZ9QmuPbok2oENE=
|
||||
golang.org/x/crypto v0.0.0-20191001170739-f9e2070545dc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
|
|
@ -142,14 +133,13 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTm
|
|||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY=
|
||||
golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed h1:5TJcLJn2a55mJjzYk0yOoqN8X1OdvBDUnaZaKKyQtkY=
|
||||
|
|
@ -164,6 +154,7 @@ golang.org/x/tools v0.0.0-20191001184121-329c8d646ebe/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI=
|
||||
|
|
|
|||
13
home/dns.go
13
home/dns.go
|
|
@ -60,6 +60,19 @@ func initDNSServer() {
|
|||
|
||||
config.dnsctx.rdns = InitRDNS(&config.clients)
|
||||
config.dnsctx.whois = initWhois(&config.clients)
|
||||
|
||||
const topClientsNumber = 30 // the number of clients to get
|
||||
topClients := config.stats.GetTopClientsIP(topClientsNumber)
|
||||
for _, ip := range topClients {
|
||||
ipAddr := net.ParseIP(ip)
|
||||
if !ipAddr.IsLoopback() {
|
||||
config.dnsctx.rdns.Begin(ip)
|
||||
}
|
||||
if isPublicIP(ipAddr) {
|
||||
config.dnsctx.whois.Begin(ip)
|
||||
}
|
||||
}
|
||||
|
||||
initFiltering()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -312,15 +312,14 @@ func isPrintableText(data []byte) bool {
|
|||
|
||||
// A helper function that parses filter contents and returns a number of rules and a filter name (if there's any)
|
||||
func parseFilterContents(contents []byte) (int, string) {
|
||||
lines := strings.Split(string(contents), "\n")
|
||||
data := string(contents)
|
||||
rulesCount := 0
|
||||
name := ""
|
||||
seenTitle := false
|
||||
|
||||
// Count lines in the filter
|
||||
for _, line := range lines {
|
||||
|
||||
line = strings.TrimSpace(line)
|
||||
for len(data) != 0 {
|
||||
line := SplitNext(&data, '\n')
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -377,3 +377,18 @@ func parseIPv4(s string) net.IP {
|
|||
|
||||
return ip.To4()
|
||||
}
|
||||
|
||||
// SplitNext - split string by a byte and return the first chunk
|
||||
// Whitespace is trimmed
|
||||
func SplitNext(str *string, splitBy byte) string {
|
||||
i := strings.IndexByte(*str, splitBy)
|
||||
s := ""
|
||||
if i != -1 {
|
||||
s = (*str)[0:i]
|
||||
*str = (*str)[i+1:]
|
||||
} else {
|
||||
s = *str
|
||||
*str = ""
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetValidNetInterfacesForWeb(t *testing.T) {
|
||||
|
|
@ -23,3 +24,10 @@ func TestGetValidNetInterfacesForWeb(t *testing.T) {
|
|||
log.Printf("%v", iface)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitNext(t *testing.T) {
|
||||
s := " a,b , c "
|
||||
assert.True(t, SplitNext(&s, ',') == "a")
|
||||
assert.True(t, SplitNext(&s, ',') == "b")
|
||||
assert.True(t, SplitNext(&s, ',') == "c" && len(s) == 0)
|
||||
}
|
||||
|
|
|
|||
103
home/whois.go
103
home/whois.go
|
|
@ -1,26 +1,35 @@
|
|||
package home
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
whois "github.com/likexian/whois-go"
|
||||
)
|
||||
|
||||
const maxValueLength = 250
|
||||
const (
|
||||
defaultServer = "whois.arin.net"
|
||||
defaultPort = "43"
|
||||
maxValueLength = 250
|
||||
)
|
||||
|
||||
// Whois - module context
|
||||
type Whois struct {
|
||||
clients *clientsContainer
|
||||
ips map[string]bool
|
||||
lock sync.Mutex
|
||||
ipChan chan string
|
||||
clients *clientsContainer
|
||||
ips map[string]bool
|
||||
lock sync.Mutex
|
||||
ipChan chan string
|
||||
timeoutMsec uint
|
||||
}
|
||||
|
||||
// Create module context
|
||||
func initWhois(clients *clientsContainer) *Whois {
|
||||
w := Whois{}
|
||||
w.timeoutMsec = 5000
|
||||
w.clients = clients
|
||||
w.ips = make(map[string]bool)
|
||||
w.ipChan = make(chan string, 255)
|
||||
|
|
@ -41,11 +50,9 @@ func whoisParse(data string) map[string]string {
|
|||
m := map[string]string{}
|
||||
descr := ""
|
||||
netname := ""
|
||||
lines := strings.Split(data, "\n")
|
||||
for _, ln := range lines {
|
||||
ln = strings.TrimSpace(ln)
|
||||
|
||||
if len(ln) == 0 || ln[0] == '#' {
|
||||
for len(data) != 0 {
|
||||
ln := SplitNext(&data, '\n')
|
||||
if len(ln) == 0 || ln[0] == '#' || ln[0] == '%' {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -68,9 +75,19 @@ func whoisParse(data string) map[string]string {
|
|||
m[k] = trimValue(v)
|
||||
|
||||
case "descr":
|
||||
descr = v
|
||||
if len(descr) == 0 {
|
||||
descr = v
|
||||
}
|
||||
case "netname":
|
||||
netname = v
|
||||
|
||||
case "whois": // "whois: whois.arin.net"
|
||||
m["whois"] = v
|
||||
|
||||
case "referralserver": // "ReferralServer: whois://whois.ripe.net"
|
||||
if strings.HasPrefix(v, "whois://") {
|
||||
m["whois"] = v[len("whois://"):]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -85,10 +102,66 @@ func whoisParse(data string) map[string]string {
|
|||
return m
|
||||
}
|
||||
|
||||
// Send request to a server and receive the response
|
||||
func (w *Whois) query(target string, serverAddr string) (string, error) {
|
||||
addr, _, _ := net.SplitHostPort(serverAddr)
|
||||
if addr == "whois.arin.net" {
|
||||
target = "n + " + target
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", serverAddr, time.Duration(w.timeoutMsec)*time.Millisecond)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond))
|
||||
_, err = conn.Write([]byte(target + "\r\n"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadAll(conn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// Query WHOIS servers (handle redirects)
|
||||
func (w *Whois) queryAll(target string) (string, error) {
|
||||
server := net.JoinHostPort(defaultServer, defaultPort)
|
||||
const maxRedirects = 5
|
||||
for i := 0; i != maxRedirects; i++ {
|
||||
resp, err := w.query(target, server)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
log.Debug("Whois: received response (%d bytes) from %s IP:%s", len(resp), server, target)
|
||||
|
||||
m := whoisParse(resp)
|
||||
redir, ok := m["whois"]
|
||||
if !ok {
|
||||
return resp, nil
|
||||
}
|
||||
redir = strings.ToLower(redir)
|
||||
|
||||
_, _, err = net.SplitHostPort(redir)
|
||||
if err != nil {
|
||||
server = net.JoinHostPort(redir, defaultPort)
|
||||
} else {
|
||||
server = redir
|
||||
}
|
||||
|
||||
log.Debug("Whois: redirected to %s IP:%s", redir, target)
|
||||
}
|
||||
return "", fmt.Errorf("Whois: redirect loop")
|
||||
}
|
||||
|
||||
// Request WHOIS information
|
||||
func whoisProcess(ip string) [][]string {
|
||||
func (w *Whois) process(ip string) [][]string {
|
||||
data := [][]string{}
|
||||
resp, err := whois.Whois(ip)
|
||||
resp, err := w.queryAll(ip)
|
||||
if err != nil {
|
||||
log.Debug("Whois: error: %s IP:%s", err, ip)
|
||||
return data
|
||||
|
|
@ -137,7 +210,7 @@ func (w *Whois) workerLoop() {
|
|||
var ip string
|
||||
ip = <-w.ipChan
|
||||
|
||||
info := whoisProcess(ip)
|
||||
info := w.process(ip)
|
||||
if len(info) == 0 {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,15 @@
|
|||
package home
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
whois "github.com/likexian/whois-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestWhois(t *testing.T) {
|
||||
resp, err := whois.Whois("8.8.8.8")
|
||||
w := Whois{timeoutMsec: 5000}
|
||||
resp, err := w.queryAll("8.8.8.8")
|
||||
assert.True(t, err == nil)
|
||||
assert.True(t, strings.Index(resp, "OrgName: Google LLC") != -1)
|
||||
assert.True(t, strings.Index(resp, "City: Mountain View") != -1)
|
||||
assert.True(t, strings.Index(resp, "Country: US") != -1)
|
||||
m := whoisParse(resp)
|
||||
assert.True(t, m["orgname"] == "Google LLC")
|
||||
assert.True(t, m["country"] == "US")
|
||||
|
|
|
|||
|
|
@ -42,6 +42,9 @@ type Stats interface {
|
|||
// Update counters
|
||||
Update(e Entry)
|
||||
|
||||
// Get IP addresses of the clients with the most number of requests
|
||||
GetTopClientsIP(limit uint) []string
|
||||
|
||||
// WriteDiskConfig - write configuration
|
||||
WriteDiskConfig(dc *DiskConfig)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,6 +75,9 @@ func TestStats(t *testing.T) {
|
|||
assert.True(t, d["num_replaced_parental"].(uint64) == 0)
|
||||
assert.True(t, d["avg_processing_time"].(float64) == 0.123456)
|
||||
|
||||
topClients := s.GetTopClientsIP(2)
|
||||
assert.True(t, topClients[0] == "127.0.0.1")
|
||||
|
||||
s.clear()
|
||||
s.Close()
|
||||
os.Remove(conf.Filename)
|
||||
|
|
|
|||
|
|
@ -455,6 +455,41 @@ func (s *statsCtx) Update(e Entry) {
|
|||
s.unitLock.Unlock()
|
||||
}
|
||||
|
||||
func (s *statsCtx) loadUnits(lastID uint32) []*unitDB {
|
||||
tx := s.beginTxn(false)
|
||||
if tx == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
units := []*unitDB{} //per-hour units
|
||||
firstID := lastID - s.limit + 1
|
||||
for i := firstID; i != lastID; i++ {
|
||||
u := s.loadUnitFromDB(tx, i)
|
||||
if u == nil {
|
||||
u = &unitDB{}
|
||||
u.NResult = make([]uint64, rLast)
|
||||
}
|
||||
units = append(units, u)
|
||||
}
|
||||
|
||||
_ = tx.Rollback()
|
||||
|
||||
s.unitLock.Lock()
|
||||
cu := serialize(s.unit)
|
||||
cuID := s.unit.id
|
||||
s.unitLock.Unlock()
|
||||
if cuID != lastID {
|
||||
units = units[1:]
|
||||
}
|
||||
units = append(units, cu)
|
||||
|
||||
if len(units) != int(s.limit) {
|
||||
log.Fatalf("len(units) != s.limit: %d %d", len(units), s.limit)
|
||||
}
|
||||
|
||||
return units
|
||||
}
|
||||
|
||||
/* Algorithm:
|
||||
. Prepare array of N units, where N is the value of "limit" configuration setting
|
||||
. Load data for the most recent units from file
|
||||
|
|
@ -486,36 +521,11 @@ func (s *statsCtx) Update(e Entry) {
|
|||
func (s *statsCtx) getData(timeUnit TimeUnit) map[string]interface{} {
|
||||
d := map[string]interface{}{}
|
||||
|
||||
tx := s.beginTxn(false)
|
||||
if tx == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
units := []*unitDB{} //per-hour units
|
||||
lastID := s.conf.UnitID()
|
||||
firstID := lastID - s.limit + 1
|
||||
for i := firstID; i != lastID; i++ {
|
||||
u := s.loadUnitFromDB(tx, i)
|
||||
if u == nil {
|
||||
u = &unitDB{}
|
||||
u.NResult = make([]uint64, rLast)
|
||||
}
|
||||
units = append(units, u)
|
||||
}
|
||||
|
||||
_ = tx.Rollback()
|
||||
|
||||
s.unitLock.Lock()
|
||||
cu := serialize(s.unit)
|
||||
cuID := s.unit.id
|
||||
s.unitLock.Unlock()
|
||||
if cuID != lastID {
|
||||
units = units[1:]
|
||||
}
|
||||
units = append(units, cu)
|
||||
|
||||
if len(units) != int(s.limit) {
|
||||
log.Fatalf("len(units) != s.limit: %d %d", len(units), s.limit)
|
||||
units := s.loadUnits(lastID)
|
||||
if units == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// per time unit counters:
|
||||
|
|
@ -687,3 +697,25 @@ func (s *statsCtx) getData(timeUnit TimeUnit) map[string]interface{} {
|
|||
|
||||
return d
|
||||
}
|
||||
|
||||
func (s *statsCtx) GetTopClientsIP(limit uint) []string {
|
||||
lastID := s.conf.UnitID()
|
||||
units := s.loadUnits(lastID)
|
||||
if units == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// top clients
|
||||
m := map[string]uint64{}
|
||||
for _, u := range units {
|
||||
for _, it := range u.Clients {
|
||||
m[it.Name] += it.Count
|
||||
}
|
||||
}
|
||||
a := convertMapToArray(m, int(limit))
|
||||
d := []string{}
|
||||
for _, it := range a {
|
||||
d = append(d, it.Name)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue