add MojeekBot user agent
This commit is contained in:
Azareal
parent
d3336245d5
commit
64ec820453
180
gen_router.go
180
gen_router.go
|
|
@ -588,34 +588,35 @@ var agentMapEnum = map[string]int{
|
|||
"bing": 14,
|
||||
"slurp": 15,
|
||||
"exabot": 16,
|
||||
"baidu": 17,
|
||||
"sogou": 18,
|
||||
"toutiao": 19,
|
||||
"haosou": 20,
|
||||
"duckduckgo": 21,
|
||||
"seznambot": 22,
|
||||
"discord": 23,
|
||||
"twitter": 24,
|
||||
"facebook": 25,
|
||||
"cloudflare": 26,
|
||||
"archive_org": 27,
|
||||
"uptimebot": 28,
|
||||
"slackbot": 29,
|
||||
"apple": 30,
|
||||
"discourse": 31,
|
||||
"alexa": 32,
|
||||
"lynx": 33,
|
||||
"blank": 34,
|
||||
"malformed": 35,
|
||||
"suspicious": 36,
|
||||
"semrush": 37,
|
||||
"dotbot": 38,
|
||||
"ahrefs": 39,
|
||||
"proximic": 40,
|
||||
"majestic": 41,
|
||||
"aspiegel": 42,
|
||||
"mail_ru": 43,
|
||||
"zgrab": 44,
|
||||
"mojeek": 17,
|
||||
"baidu": 18,
|
||||
"sogou": 19,
|
||||
"toutiao": 20,
|
||||
"haosou": 21,
|
||||
"duckduckgo": 22,
|
||||
"seznambot": 23,
|
||||
"discord": 24,
|
||||
"twitter": 25,
|
||||
"facebook": 26,
|
||||
"cloudflare": 27,
|
||||
"archive_org": 28,
|
||||
"uptimebot": 29,
|
||||
"slackbot": 30,
|
||||
"apple": 31,
|
||||
"discourse": 32,
|
||||
"alexa": 33,
|
||||
"lynx": 34,
|
||||
"blank": 35,
|
||||
"malformed": 36,
|
||||
"suspicious": 37,
|
||||
"semrush": 38,
|
||||
"dotbot": 39,
|
||||
"ahrefs": 40,
|
||||
"proximic": 41,
|
||||
"majestic": 42,
|
||||
"aspiegel": 43,
|
||||
"mail_ru": 44,
|
||||
"zgrab": 45,
|
||||
}
|
||||
var reverseAgentMapEnum = map[int]string{
|
||||
0: "unknown",
|
||||
|
|
@ -635,34 +636,35 @@ var reverseAgentMapEnum = map[int]string{
|
|||
14: "bing",
|
||||
15: "slurp",
|
||||
16: "exabot",
|
||||
17: "baidu",
|
||||
18: "sogou",
|
||||
19: "toutiao",
|
||||
20: "haosou",
|
||||
21: "duckduckgo",
|
||||
22: "seznambot",
|
||||
23: "discord",
|
||||
24: "twitter",
|
||||
25: "facebook",
|
||||
26: "cloudflare",
|
||||
27: "archive_org",
|
||||
28: "uptimebot",
|
||||
29: "slackbot",
|
||||
30: "apple",
|
||||
31: "discourse",
|
||||
32: "alexa",
|
||||
33: "lynx",
|
||||
34: "blank",
|
||||
35: "malformed",
|
||||
36: "suspicious",
|
||||
37: "semrush",
|
||||
38: "dotbot",
|
||||
39: "ahrefs",
|
||||
40: "proximic",
|
||||
41: "majestic",
|
||||
42: "aspiegel",
|
||||
43: "mail_ru",
|
||||
44: "zgrab",
|
||||
17: "mojeek",
|
||||
18: "baidu",
|
||||
19: "sogou",
|
||||
20: "toutiao",
|
||||
21: "haosou",
|
||||
22: "duckduckgo",
|
||||
23: "seznambot",
|
||||
24: "discord",
|
||||
25: "twitter",
|
||||
26: "facebook",
|
||||
27: "cloudflare",
|
||||
28: "archive_org",
|
||||
29: "uptimebot",
|
||||
30: "slackbot",
|
||||
31: "apple",
|
||||
32: "discourse",
|
||||
33: "alexa",
|
||||
34: "lynx",
|
||||
35: "blank",
|
||||
36: "malformed",
|
||||
37: "suspicious",
|
||||
38: "semrush",
|
||||
39: "dotbot",
|
||||
40: "ahrefs",
|
||||
41: "proximic",
|
||||
42: "majestic",
|
||||
43: "aspiegel",
|
||||
44: "mail_ru",
|
||||
45: "zgrab",
|
||||
}
|
||||
var markToAgent = map[string]string{
|
||||
"OPR": "opera",
|
||||
|
|
@ -688,6 +690,7 @@ var markToAgent = map[string]string{
|
|||
"BingPreview": "bing",
|
||||
"Slurp": "slurp",
|
||||
"Exabot": "exabot",
|
||||
"MojeekBot": "mojeek",
|
||||
"SeznamBot": "seznambot",
|
||||
"CloudFlare": "cloudflare",
|
||||
"archive": "archive_org",
|
||||
|
|
@ -718,43 +721,44 @@ var markToID = map[string]int{
|
|||
"MSIE": 6,
|
||||
"Trident": 7,
|
||||
"Edge": 5,
|
||||
"Lynx": 33,
|
||||
"Lynx": 34,
|
||||
"SamsungBrowser": 10,
|
||||
"UCBrowser": 11,
|
||||
"Google": 12,
|
||||
"Googlebot": 12,
|
||||
"yandex": 13,
|
||||
"DuckDuckBot": 21,
|
||||
"DuckDuckGo": 21,
|
||||
"Baiduspider": 17,
|
||||
"Sogou": 18,
|
||||
"ToutiaoSpider": 19,
|
||||
"360Spider": 20,
|
||||
"DuckDuckBot": 22,
|
||||
"DuckDuckGo": 22,
|
||||
"Baiduspider": 18,
|
||||
"Sogou": 19,
|
||||
"ToutiaoSpider": 20,
|
||||
"360Spider": 21,
|
||||
"bingbot": 14,
|
||||
"BingPreview": 14,
|
||||
"Slurp": 15,
|
||||
"Exabot": 16,
|
||||
"SeznamBot": 22,
|
||||
"CloudFlare": 26,
|
||||
"archive": 27,
|
||||
"Uptimebot": 28,
|
||||
"Slackbot": 29,
|
||||
"Slack": 29,
|
||||
"Discordbot": 23,
|
||||
"Twitterbot": 24,
|
||||
"facebookexternalhit": 25,
|
||||
"Facebot": 25,
|
||||
"Applebot": 30,
|
||||
"Discourse": 31,
|
||||
"ia_archiver": 32,
|
||||
"SemrushBot": 37,
|
||||
"DotBot": 38,
|
||||
"AhrefsBot": 39,
|
||||
"proximic": 40,
|
||||
"MJ12bot": 41,
|
||||
"AspiegelBot": 42,
|
||||
"RU_Bot": 43,
|
||||
"zgrab": 44,
|
||||
"MojeekBot": 17,
|
||||
"SeznamBot": 23,
|
||||
"CloudFlare": 27,
|
||||
"archive": 28,
|
||||
"Uptimebot": 29,
|
||||
"Slackbot": 30,
|
||||
"Slack": 30,
|
||||
"Discordbot": 24,
|
||||
"Twitterbot": 25,
|
||||
"facebookexternalhit": 26,
|
||||
"Facebot": 26,
|
||||
"Applebot": 31,
|
||||
"Discourse": 32,
|
||||
"ia_archiver": 33,
|
||||
"SemrushBot": 38,
|
||||
"DotBot": 39,
|
||||
"AhrefsBot": 40,
|
||||
"proximic": 41,
|
||||
"MJ12bot": 42,
|
||||
"AspiegelBot": 43,
|
||||
"RU_Bot": 44,
|
||||
"zgrab": 45,
|
||||
}
|
||||
/*var agentRank = map[string]int{
|
||||
"opera":9,
|
||||
|
|
@ -882,7 +886,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
|
|||
pre += "\n"
|
||||
}
|
||||
r.DumpRequest(req,pre+"Suspicious Request")
|
||||
co.AgentViewCounter.Bump(36)
|
||||
co.AgentViewCounter.Bump(37)
|
||||
}
|
||||
|
||||
func isLocalHost(h string) bool {
|
||||
|
|
@ -897,7 +901,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
w.WriteHeader(200) // 400
|
||||
w.Write([]byte(""))
|
||||
r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ))
|
||||
co.AgentViewCounter.Bump(35)
|
||||
co.AgentViewCounter.Bump(36)
|
||||
}
|
||||
|
||||
// Split the Host and Port string
|
||||
|
|
@ -1040,7 +1044,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
|
||||
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
|
||||
if ua == "" {
|
||||
co.AgentViewCounter.Bump(34)
|
||||
co.AgentViewCounter.Bump(35)
|
||||
if c.Dev.DebugMode {
|
||||
var pre string
|
||||
for _, char := range req.UserAgent() {
|
||||
|
|
@ -1121,7 +1125,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
if strings.Contains(ua,"rv:11") {
|
||||
agent = 6
|
||||
}
|
||||
case 44:
|
||||
case 45:
|
||||
r.SuspiciousRequest(req,"Vulnerability Scanner")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@
|
|||
"bing":"Bing",
|
||||
"slurp":"Yahoo! Slurp",
|
||||
"exabot":"Exabot",
|
||||
"mojeek":"MojeekBot",
|
||||
"sogou":"Sogou",
|
||||
"toutiao":"Toutiao",
|
||||
"haosou":"Qihoo 360 Search",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,7 @@ func main() {
|
|||
"bing",
|
||||
"slurp",
|
||||
"exabot",
|
||||
"mojeek",
|
||||
"baidu",
|
||||
"sogou",
|
||||
"toutiao",
|
||||
|
|
@ -308,6 +309,7 @@ func main() {
|
|||
"BingPreview",
|
||||
"Slurp",
|
||||
"Exabot",
|
||||
"MojeekBot",
|
||||
"SeznamBot",
|
||||
"CloudFlare",
|
||||
"archive", //archive.org_bot
|
||||
|
|
@ -357,6 +359,7 @@ func main() {
|
|||
"BingPreview": "bing",
|
||||
"Slurp": "slurp",
|
||||
"Exabot": "exabot",
|
||||
"MojeekBot": "mojeek",
|
||||
"SeznamBot": "seznambot",
|
||||
"CloudFlare": "cloudflare", // Track alwayson specifically in case there are other bots?
|
||||
"archive": "archive_org", //archive.org_bot
|
||||
|
|
|
|||
Loading…
Reference in New Issue