From 4126e8ed0c6fbdd220d394f3453a2e93242ed2b4 Mon Sep 17 00:00:00 2001 From: Azareal Date: Tue, 2 Mar 2021 17:22:32 +1000 Subject: [PATCH] add qwant, xenforo, twingly, linkfluence, and new toutiao user agent. add Users.CountSearch test cases. add Users.SearchOffset test cases. add more Users.Count test cases. add more Users.BulkGetByName test cases. reduce boilerplate in other tests. fix bugs in Users.CountSearch and Users.BulkGetByName --- common/user_store.go | 4 +- gen_router.go | 320 +++++++++++++++++++++++-------------------- langs/english.json | 4 + misc_test.go | 107 ++++++++++----- router_gen/main.go | 9 ++ 5 files changed, 254 insertions(+), 190 deletions(-) diff --git a/common/user_store.go b/common/user_store.go index 3f7ae084..02bc4a50 100644 --- a/common/user_store.go +++ b/common/user_store.go @@ -78,7 +78,7 @@ func NewDefaultUserStore(cache UserCache) (*DefaultUserStore, error) { nameExists: acc.Exists(u, "name").Prepare(), count: acc.Count(u).Prepare(), - countSearch: acc.Count(u).Where("(name LIKE ('%'+?+'%') OR ?='') AND (email=? OR ?='') AND (group=? OR ?=0)").Prepare(), + countSearch: acc.Count(u).Where("(name=? OR ?='') AND (email=? OR ?='') AND (group=? OR ?=0)").Prepare(), }, acc.FirstError() } @@ -193,7 +193,7 @@ func (s *DefaultUserStore) BulkGetByName(names []string) (list []*User, err erro } u.Init() s.cache.Set(u) - list[u.ID] = u + list = append(list, u) } if err = rows.Err(); err != nil { return list, err diff --git a/gen_router.go b/gen_router.go index e9acd36c..e33fe057 100644 --- a/gen_router.go +++ b/gen_router.go @@ -598,54 +598,58 @@ var agentMapEnum = map[string]int{ "exabot": 16, "mojeek": 17, "cliqz": 18, - "datenbank": 19, - "baidu": 20, - "sogou": 21, - "toutiao": 22, - "haosou": 23, - "duckduckgo": 24, - "seznambot": 25, - "discord": 26, - "telegram": 27, - "twitter": 28, - "facebook": 29, - "cloudflare": 30, - "archive_org": 31, - "uptimebot": 32, - "slackbot": 33, - "apple": 34, - "discourse": 35, - "mattermost": 36, - "alexa": 37, - "lynx": 38, - "blank": 39, - "malformed": 40, - "suspicious": 41, - "semrush": 42, - "dotbot": 43, - "ahrefs": 44, - "proximic": 45, - "megaindex": 46, - "majestic": 47, - "cocolyze": 48, - "babbar": 49, - "surdotly": 50, - "domcop": 51, - "netcraft": 52, - "blexbot": 53, - "wappalyzer": 54, - "burf": 55, - "aspiegel": 56, - "mail_ru": 57, - "ccbot": 58, - "yacy": 59, - "zgrab": 60, - "cloudsystemnetworks": 61, - "maui": 62, - "curl": 63, - "python": 64, - "headlesschrome": 65, - "awesome_bot": 66, + "qwant": 19, + "datenbank": 20, + "baidu": 21, + "sogou": 22, + "toutiao": 23, + "haosou": 24, + "duckduckgo": 25, + "seznambot": 26, + "discord": 27, + "telegram": 28, + "twitter": 29, + "facebook": 30, + "cloudflare": 31, + "archive_org": 32, + "uptimebot": 33, + "slackbot": 34, + "apple": 35, + "discourse": 36, + "xenforo": 37, + "mattermost": 38, + "alexa": 39, + "lynx": 40, + "blank": 41, + "malformed": 42, + "suspicious": 43, + "semrush": 44, + "dotbot": 45, + "ahrefs": 46, + "proximic": 47, + "megaindex": 48, + "majestic": 49, + "cocolyze": 50, + "babbar": 51, + "surdotly": 52, + "domcop": 53, + "netcraft": 54, + "blexbot": 55, + "wappalyzer": 56, + "twingly": 57, + "linkfluence": 58, + "burf": 59, + "aspiegel": 60, + "mail_ru": 61, + "ccbot": 62, + "yacy": 63, + "zgrab": 64, + "cloudsystemnetworks": 65, + "maui": 66, + "curl": 67, + "python": 68, + "headlesschrome": 69, + "awesome_bot": 70, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -667,54 +671,58 @@ var reverseAgentMapEnum = map[int]string{ 16: "exabot", 17: "mojeek", 18: "cliqz", - 19: "datenbank", - 20: "baidu", - 21: "sogou", - 22: "toutiao", - 23: "haosou", - 24: "duckduckgo", - 25: "seznambot", - 26: "discord", - 27: "telegram", - 28: "twitter", - 29: "facebook", - 30: "cloudflare", - 31: "archive_org", - 32: "uptimebot", - 33: "slackbot", - 34: "apple", - 35: "discourse", - 36: "mattermost", - 37: "alexa", - 38: "lynx", - 39: "blank", - 40: "malformed", - 41: "suspicious", - 42: "semrush", - 43: "dotbot", - 44: "ahrefs", - 45: "proximic", - 46: "megaindex", - 47: "majestic", - 48: "cocolyze", - 49: "babbar", - 50: "surdotly", - 51: "domcop", - 52: "netcraft", - 53: "blexbot", - 54: "wappalyzer", - 55: "burf", - 56: "aspiegel", - 57: "mail_ru", - 58: "ccbot", - 59: "yacy", - 60: "zgrab", - 61: "cloudsystemnetworks", - 62: "maui", - 63: "curl", - 64: "python", - 65: "headlesschrome", - 66: "awesome_bot", + 19: "qwant", + 20: "datenbank", + 21: "baidu", + 22: "sogou", + 23: "toutiao", + 24: "haosou", + 25: "duckduckgo", + 26: "seznambot", + 27: "discord", + 28: "telegram", + 29: "twitter", + 30: "facebook", + 31: "cloudflare", + 32: "archive_org", + 33: "uptimebot", + 34: "slackbot", + 35: "apple", + 36: "discourse", + 37: "xenforo", + 38: "mattermost", + 39: "alexa", + 40: "lynx", + 41: "blank", + 42: "malformed", + 43: "suspicious", + 44: "semrush", + 45: "dotbot", + 46: "ahrefs", + 47: "proximic", + 48: "megaindex", + 49: "majestic", + 50: "cocolyze", + 51: "babbar", + 52: "surdotly", + 53: "domcop", + 54: "netcraft", + 55: "blexbot", + 56: "wappalyzer", + 57: "twingly", + 58: "linkfluence", + 59: "burf", + 60: "aspiegel", + 61: "mail_ru", + 62: "ccbot", + 63: "yacy", + 64: "zgrab", + 65: "cloudsystemnetworks", + 66: "maui", + 67: "curl", + 68: "python", + 69: "headlesschrome", + 70: "awesome_bot", } var markToAgent = map[string]string{ "OPR": "opera", @@ -735,6 +743,7 @@ var markToAgent = map[string]string{ "Baiduspider": "baidu", "Sogou": "sogou", "ToutiaoSpider": "toutiao", + "Bytespider": "toutiao", "360Spider": "haosou", "bingbot": "bing", "BingPreview": "bing", @@ -743,6 +752,7 @@ var markToAgent = map[string]string{ "Exabot": "exabot", "MojeekBot": "mojeek", "Cliqzbot": "cliqz", + "Qwantify": "qwant", "netEstate": "datenbank", "SeznamBot": "seznambot", "CloudFlare": "cloudflare", @@ -757,6 +767,7 @@ var markToAgent = map[string]string{ "Facebot": "facebook", "Applebot": "apple", "Discourse": "discourse", + "XenForo": "xenforo", "mattermost": "mattermost", "ia_archiver": "alexa", "SemrushBot": "semrush", @@ -773,6 +784,8 @@ var markToAgent = map[string]string{ "NetcraftSurveyAgent": "netcraft", "BLEXBot": "blexbot", "Wappalyzer": "wappalyzer", + "Twingly": "twingly", + "linkfluence": "linkfluence", "Burf": "burf", "AspiegelBot": "aspiegel", "PetalBot": "aspiegel", @@ -795,18 +808,19 @@ var markToID = map[string]int{ "MSIE": 6, "Trident": 7, "Edge": 5, - "Lynx": 38, + "Lynx": 40, "SamsungBrowser": 10, "UCBrowser": 11, "Google": 12, "Googlebot": 12, "yandex": 13, - "DuckDuckBot": 24, - "DuckDuckGo": 24, - "Baiduspider": 20, - "Sogou": 21, - "ToutiaoSpider": 22, - "360Spider": 23, + "DuckDuckBot": 25, + "DuckDuckGo": 25, + "Baiduspider": 21, + "Sogou": 22, + "ToutiaoSpider": 23, + "Bytespider": 23, + "360Spider": 24, "bingbot": 14, "BingPreview": 14, "msnbot": 14, @@ -814,49 +828,53 @@ var markToID = map[string]int{ "Exabot": 16, "MojeekBot": 17, "Cliqzbot": 18, - "netEstate": 19, - "SeznamBot": 25, - "CloudFlare": 30, - "archive": 31, - "Uptimebot": 32, - "Slackbot": 33, - "Slack": 33, - "Discordbot": 26, - "TelegramBot": 27, - "Twitterbot": 28, - "facebookexternalhit": 29, - "Facebot": 29, - "Applebot": 34, - "Discourse": 35, - "mattermost": 36, - "ia_archiver": 37, - "SemrushBot": 42, - "DotBot": 43, - "AhrefsBot": 44, - "proximic": 45, - "MegaIndex": 46, - "MJ12bot": 47, - "mj12bot": 47, - "Cocolyzebot": 48, - "Barkrowler": 49, - "SurdotlyBot": 50, - "DomCopBot": 51, - "NetcraftSurveyAgent": 52, - "BLEXBot": 53, - "Wappalyzer": 54, - "Burf": 55, - "AspiegelBot": 56, - "PetalBot": 56, - "RU_Bot": 57, - "CCBot": 58, - "yacybot": 59, - "zgrab": 60, - "Nimbostratus": 61, - "MauiBot": 62, - "curl": 63, - "python": 64, - "HeadlessChrome": 65, - "awesome_bot": 66, + "Qwantify": 19, + "netEstate": 20, + "SeznamBot": 26, + "CloudFlare": 31, + "archive": 32, + "Uptimebot": 33, + "Slackbot": 34, + "Slack": 34, + "Discordbot": 27, + "TelegramBot": 28, + "Twitterbot": 29, + "facebookexternalhit": 30, + "Facebot": 30, + "Applebot": 35, + "Discourse": 36, + "XenForo": 37, + "mattermost": 38, + "ia_archiver": 39, + "SemrushBot": 44, + "DotBot": 45, + "AhrefsBot": 46, + "proximic": 47, + "MegaIndex": 48, + "MJ12bot": 49, + "mj12bot": 49, + "Cocolyzebot": 50, + "Barkrowler": 51, + "SurdotlyBot": 52, + "DomCopBot": 53, + "NetcraftSurveyAgent": 54, + "BLEXBot": 55, + "Wappalyzer": 56, + "Twingly": 57, + "linkfluence": 58, + "Burf": 59, + "AspiegelBot": 60, + "PetalBot": 60, + "RU_Bot": 61, + "CCBot": 62, + "yacybot": 63, + "zgrab": 64, + "Nimbostratus": 65, + "MauiBot": 66, + "curl": 67, + "python": 68, + "HeadlessChrome": 69, + "awesome_bot": 70, } /*var agentRank = map[string]int{ "opera":9, @@ -1022,7 +1040,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { pre = "Suspicious Request" } r.dumpRequest(req,pre,r.suspReqLogger) - co.AgentViewCounter.Bump(41) + co.AgentViewCounter.Bump(43) } func isLocalHost(h string) bool { @@ -1041,7 +1059,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.WriteHeader(200) // 400 w.Write([]byte("")) r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ)) - co.AgentViewCounter.Bump(40) + co.AgentViewCounter.Bump(42) } // Split the Host and Port string @@ -1181,7 +1199,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another if ua == "" { - co.AgentViewCounter.Bump(39) + co.AgentViewCounter.Bump(41) if c.Dev.DebugMode { var pre string for _, char := range req.UserAgent() { @@ -1275,11 +1293,11 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if strings.Contains(ua,"rv:11") { agent = 6 } - case 60: + case 64: w.WriteHeader(200) // 400 w.Write([]byte("")) r.DumpRequest(req,"Blocked Scanner") - co.AgentViewCounter.Bump(60) + co.AgentViewCounter.Bump(64) return } diff --git a/langs/english.json b/langs/english.json index e32ea5aa..b0999e4b 100644 --- a/langs/english.json +++ b/langs/english.json @@ -205,6 +205,7 @@ "exabot":"Exabot", "mojeek":"MojeekBot", "cliqz":"Cliqzbot", + "qwant":"Qwant", "datenbank":"Website Datenbank", "sogou":"Sogou", "toutiao":"Toutiao", @@ -222,6 +223,7 @@ "facebook":"Facebook", "apple":"AppleBot", "discourse":"Discourse Forum Onebox", + "xenforo":"XenForo", "mattermost":"Mattermost", "alexa":"Alexa", "lynx":"Lynx", @@ -239,6 +241,8 @@ "netcraft":"Netcraft", "blexbot":"BLEXBot", "wappalyzer":"Wappalyzer", + "twingly":"Twingly", + "linkfluence":"Linkfluence", "burf":"Burf.co", "aspiegel":"Aspiegel", "mail_ru":"Mail.ru bot", diff --git a/misc_test.go b/misc_test.go index 6fef6b66..6e5c5bd9 100644 --- a/misc_test.go +++ b/misc_test.go @@ -139,18 +139,16 @@ func userStoreTest(t *testing.T, newUserID int) { } // TODO: Lock onto the specific error type. Is this even possible without sacrificing the detailed information in the error message? - var userList map[int]*c.User - userList, _ = c.Users.BulkGetMap([]int{-1}) - expectf(t, len(userList) == 0, "The userList length should be 0, not %d", len(userList)) - expectf(t, isCacheLengthZero(uc), "User cache length should be 0, not %d", cacheLength(uc)) + bulkGetMapEmpty := func(id int) { + userList, _ := c.Users.BulkGetMap([]int{id}) + expectf(t, len(userList) == 0, "The userList length should be 0, not %d", len(userList)) + expectf(t, isCacheLengthZero(uc), "User cache length should be 0, not %d", cacheLength(uc)) + } + bulkGetMapEmpty(-1) + bulkGetMapEmpty(0) - userList, _ = c.Users.BulkGetMap([]int{0}) - expectf(t, len(userList) == 0, "The userList length should be 0, not %d", len(userList)) - expectf(t, isCacheLengthZero(uc), "User cache length should be 0, not %d", cacheLength(uc)) - - userList, _ = c.Users.BulkGetMap([]int{1}) + userList, _ := c.Users.BulkGetMap([]int{1}) expectf(t, len(userList) == 1, "Returned map should have one result (UID #1), not %d", len(userList)) - user, ok := userList[1] if !ok { t.Error("We couldn't find UID #1 in the returned map") @@ -174,7 +172,39 @@ func userStoreTest(t *testing.T, newUserID int) { expectf(t, !c.Users.Exists(newUserID), "UID #%d shouldn't exist", newUserID) expectf(t, isCacheLengthZero(uc), "User cache length should be 0, not %d", cacheLength(uc)) - expectIntToBeX(t, c.Users.Count(), 1, "The number of users should be one, not %d") + expectIntToBeX(t, c.Users.Count(), 1, "The number of users should be 1, not %d") + searchUser := func(name, email string, gid, count int) { + f := func(name, email string, gid, count int, m string) { + expectIntToBeX(t, c.Users.CountSearch(name, email, gid), count, "The number of users for "+m+", not %d") + } + f(name, email, 0, count, fmt.Sprintf("name '%s' and email '%s' should be %d", name, email, count)) + f(name, "", 0, count, fmt.Sprintf("name '%s' should be %d", name, count)) + f("", email, 0, count, fmt.Sprintf("email '%s' should be %d", email, count)) + + f2 := func(name, email string, gid, offset int, m string, args ...interface{}) { + ulist, err := c.Users.SearchOffset(name, email, gid, offset, 15) + expectNilErr(t, err) + expectIntToBeX(t, len(ulist), count, "The number of users for "+fmt.Sprintf(m, args...)+", not %d") + } + f2(name, email, 0, 0, "name '%s' and email '%s' should be %d", name, email, count) + f2(name, "", 0, 0, "name '%s' should be %d", name, count) + f2("", email, 0, 0, "email '%s' should be %d", email, count) + + count = 0 + f2(name, email, 0, 10, "name '%s' and email '%s' should be %d", name, email, count) + f2(name, "", 0, 10, "name '%s' should be %d", name, count) + f2("", email, 0, 10, "email '%s' should be %d", email, count) + + f2(name, email, 999, 0, "name '%s' and email '%s' should be %d", name, email, 0) + f2(name, "", 999, 0, "name '%s' should be %d", name, 0) + f2("", email, 999, 0, "email '%s' should be %d", email, 0) + + f2(name, email, 999, 10, "name '%s' and email '%s' should be %d", name, email, 0) + f2(name, "", 999, 10, "name '%s' should be %d", name, 0) + f2("", email, 999, 10, "email '%s' should be %d", email, 0) + } + searchUser("Sam", "sam@localhost.loc", 0, 0) + // TODO: CountSearch gid test awaitingActivation := 5 // TODO: Write tests for the registration validators @@ -182,6 +212,9 @@ func userStoreTest(t *testing.T, newUserID int) { expectNilErr(t, err) expectf(t, uid == newUserID, "The UID of the new user should be %d not %d", newUserID, uid) expectf(t, c.Users.Exists(newUserID), "UID #%d should exist", newUserID) + expectIntToBeX(t, c.Users.Count(), 2, "The number of users should be 2, not %d") + searchUser("Sam", "sam@localhost.loc", 0, 1) + // TODO: CountSearch gid test user, err = c.Users.Get(newUserID) recordMustExist(t, err, "Couldn't find UID #%d", newUserID) @@ -195,7 +228,13 @@ func userStoreTest(t *testing.T, newUserID int) { } userList, _ = c.Users.BulkGetMap([]int{1, uid}) - expectf(t, len(userList) == 2, "Returned map should have two results, not %d", len(userList)) + expectf(t, len(userList) == 2, "Returned map should have 2 results, not %d", len(userList)) + // TODO: More tests on userList + + { + userList, _ := c.Users.BulkGetByName([]string{"Admin", "Sam"}) + expectf(t, len(userList) == 2, "Returned list should have 2 results, not %d", len(userList)) + } if uc != nil { expectIntToBeX(t, uc.Length(), 2, "User cache length should be 2, not %d") @@ -327,10 +366,12 @@ func userStoreTest(t *testing.T, newUserID int) { expectNilErr(t, err) expect(t, user.Group == 6, "Someone's mutated this pointer elsewhere") - err = user.Delete() - expectNilErr(t, err) + expectNilErr(t, user.Delete()) expectf(t, !c.Users.Exists(newUserID), "UID #%d should no longer exist", newUserID) afterUserFlush(newUserID) + expectIntToBeX(t, c.Users.Count(), 1, "The number of users should be 1, not %d") + searchUser("Sam", "sam@localhost.loc", 0, 0) + // TODO: CountSearch gid test _, err = c.Users.Get(newUserID) recordMustNotExist(t, err, "UID #%d shouldn't exist", newUserID) @@ -1836,15 +1877,13 @@ func TestMetaStore(t *testing.T) { expect(t, m == "", "meta var magic should be empty") recordMustNotExist(t, err, "meta var magic should not exist") - err = c.Meta.Set("magic", "lol") - expectNilErr(t, err) + expectNilErr(t, c.Meta.Set("magic", "lol")) m, err = c.Meta.Get("magic") expectNilErr(t, err) expect(t, m == "lol", "meta var magic should be lol") - err = c.Meta.Set("magic", "wha") - expectNilErr(t, err) + expectNilErr(t, c.Meta.Set("magic", "wha")) m, err = c.Meta.Get("magic") expectNilErr(t, err) @@ -1925,19 +1964,20 @@ func TestWordFilters(t *testing.T) { expect(t, c.WordFilters.EstCount() == 1, "Word filter list should not be empty") expect(t, c.WordFilters.Count() == 1, "Word filter list should not be empty") + ftest := func(f *c.WordFilter, id int, find, replace string) { + expectf(t, f.ID == id, "Word filter ID should be %d, not %d", id, f.ID) + expectf(t, f.Find == find, "Word filter needle should be '%s', not '%s'", find, f.Find) + expectf(t, f.Replace == replace, "Word filter replacement should be '%s', not '%s'", replace, f.Replace) + } + filters, err = c.WordFilters.GetAll() expectNilErr(t, err) expect(t, len(filters) == 1, "Word filter map should not be empty") - filter := filters[1] - expect(t, filter.ID == 1, "Word filter ID should be 1") - expect(t, filter.Find == "imbecile", "Word filter needle should be imbecile") - expect(t, filter.Replace == "lovely", "Word filter replacement should be lovely") + ftest(filters[1], 1, "imbecile", "lovely") - filter, err = c.WordFilters.Get(1) + filter, err := c.WordFilters.Get(1) expectNilErr(t, err) - expect(t, filter.ID == 1, "Word filter ID should be 1") - expect(t, filter.Find == "imbecile", "Word filter needle should be imbecile") - expect(t, filter.Replace == "lovely", "Word filter replacement should be lovely") + ftest(filter, 1, "imbecile", "lovely") // Update expectNilErr(t, c.WordFilters.Update(1, "b", "a")) @@ -1949,21 +1989,15 @@ func TestWordFilters(t *testing.T) { filters, err = c.WordFilters.GetAll() expectNilErr(t, err) expect(t, len(filters) == 1, "Word filter map should not be empty") - filter = filters[1] - expect(t, filter.ID == 1, "Word filter ID should be 1") - expect(t, filter.Find == "b", "Word filter needle should be b") - expect(t, filter.Replace == "a", "Word filter replacement should be a") + ftest(filters[1], 1, "b", "a") filter, err = c.WordFilters.Get(1) expectNilErr(t, err) - expect(t, filter.ID == 1, "Word filter ID should be 1") - expect(t, filter.Find == "b", "Word filter needle should be imbecile") - expect(t, filter.Replace == "a", "Word filter replacement should be a") + ftest(filter, 1, "b", "a") // TODO: Add a test for ParseMessage relating to word filters - err = c.WordFilters.Delete(1) - expectNilErr(t, err) + expectNilErr(t, c.WordFilters.Delete(1)) expect(t, c.WordFilters.Length() == 0, "Word filter list should be empty") expect(t, c.WordFilters.EstCount() == 0, "Word filter list should be empty") @@ -2095,8 +2129,7 @@ func TestWidgets(t *testing.T) { widget2.Enabled = false ewidget = &c.WidgetEdit{widget2, map[string]string{"Name": "Test", "Text": "Testing"}} - err = ewidget.Commit() - expectNilErr(t, err) + expectNilErr(t, ewidget.Commit()) widget2, err = c.Widgets.Get(1) expectNilErr(t, err) diff --git a/router_gen/main.go b/router_gen/main.go index 25d57065..e14bcabb 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -250,6 +250,7 @@ func main() { "exabot", "mojeek", "cliqz", + "qwant", "datenbank", "baidu", "sogou", @@ -267,6 +268,7 @@ func main() { "slackbot", "apple", "discourse", + "xenforo", "mattermost", "alexa", "lynx", @@ -286,6 +288,8 @@ func main() { "netcraft", "blexbot", "wappalyzer", + "twingly", + "linkfluence", "burf", "aspiegel", "mail_ru", @@ -333,6 +337,7 @@ func main() { a("Baiduspider", "baidu") a("Sogou", "sogou") a("ToutiaoSpider", "toutiao") + a("Bytespider", "toutiao") a("360Spider", "haosou") a("bingbot", "bing") a("BingPreview", "bing") @@ -341,6 +346,7 @@ func main() { a("Exabot", "exabot") a("MojeekBot", "mojeek") a("Cliqzbot", "cliqz") + a("Qwantify", "qwant") a("netEstate", "datenbank") a("SeznamBot", "seznambot") a("CloudFlare", "cloudflare") // Track alwayson specifically in case there are other bots? @@ -355,6 +361,7 @@ func main() { a("Facebot", "facebook") a("Applebot", "apple") a("Discourse", "discourse") + a("XenForo", "xenforo") a("mattermost", "mattermost") a("ia_archiver", "alexa") @@ -372,6 +379,8 @@ func main() { a("NetcraftSurveyAgent", "netcraft") a("BLEXBot", "blexbot") a("Wappalyzer", "wappalyzer") + a("Twingly", "twingly") + a("linkfluence", "linkfluence") a("Burf", "burf") a("AspiegelBot", "aspiegel") a("PetalBot", "aspiegel")