From 75d1d6fcbe32794101cd425e2d99a1e150ef218f Mon Sep 17 00:00:00 2001 From: Azareal Date: Fri, 29 May 2020 08:41:19 +1000 Subject: [PATCH] add telegram, domcop, maui and alt aspiegel user agents reduce boilerplate in routergen --- gen_router.go | 241 ++++++++++++++++++++++++--------------------- langs/english.json | 3 + router_gen/main.go | 47 +++++---- 3 files changed, 157 insertions(+), 134 deletions(-) diff --git a/gen_router.go b/gen_router.go index a1d58a9b..b3f52d7c 100644 --- a/gen_router.go +++ b/gen_router.go @@ -600,42 +600,45 @@ var agentMapEnum = map[string]int{ "duckduckgo": 24, "seznambot": 25, "discord": 26, - "twitter": 27, - "facebook": 28, - "cloudflare": 29, - "archive_org": 30, - "uptimebot": 31, - "slackbot": 32, - "apple": 33, - "discourse": 34, - "mattermost": 35, - "alexa": 36, - "lynx": 37, - "blank": 38, - "malformed": 39, - "suspicious": 40, - "semrush": 41, - "dotbot": 42, - "ahrefs": 43, - "proximic": 44, - "megaindex": 45, - "majestic": 46, - "cocolyze": 47, - "babbar": 48, - "surdotly": 49, - "netcraft": 50, - "blexbot": 51, - "burf": 52, - "aspiegel": 53, - "mail_ru": 54, - "ccbot": 55, - "zgrab": 56, - "cloudsystemnetworks": 57, - "curl": 58, - "python": 59, - "go": 60, - "headlesschrome": 61, - "awesome_bot": 62, + "telegram": 27, + "twitter": 28, + "facebook": 29, + "cloudflare": 30, + "archive_org": 31, + "uptimebot": 32, + "slackbot": 33, + "apple": 34, + "discourse": 35, + "mattermost": 36, + "alexa": 37, + "lynx": 38, + "blank": 39, + "malformed": 40, + "suspicious": 41, + "semrush": 42, + "dotbot": 43, + "ahrefs": 44, + "proximic": 45, + "megaindex": 46, + "majestic": 47, + "cocolyze": 48, + "babbar": 49, + "surdotly": 50, + "domcop": 51, + "netcraft": 52, + "blexbot": 53, + "burf": 54, + "aspiegel": 55, + "mail_ru": 56, + "ccbot": 57, + "zgrab": 58, + "cloudsystemnetworks": 59, + "maui": 60, + "curl": 61, + "python": 62, + "go": 63, + "headlesschrome": 64, + "awesome_bot": 65, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -665,42 +668,45 @@ var reverseAgentMapEnum = map[int]string{ 24: "duckduckgo", 25: "seznambot", 26: "discord", - 27: "twitter", - 28: "facebook", - 29: "cloudflare", - 30: "archive_org", - 31: "uptimebot", - 32: "slackbot", - 33: "apple", - 34: "discourse", - 35: "mattermost", - 36: "alexa", - 37: "lynx", - 38: "blank", - 39: "malformed", - 40: "suspicious", - 41: "semrush", - 42: "dotbot", - 43: "ahrefs", - 44: "proximic", - 45: "megaindex", - 46: "majestic", - 47: "cocolyze", - 48: "babbar", - 49: "surdotly", - 50: "netcraft", - 51: "blexbot", - 52: "burf", - 53: "aspiegel", - 54: "mail_ru", - 55: "ccbot", - 56: "zgrab", - 57: "cloudsystemnetworks", - 58: "curl", - 59: "python", - 60: "go", - 61: "headlesschrome", - 62: "awesome_bot", + 27: "telegram", + 28: "twitter", + 29: "facebook", + 30: "cloudflare", + 31: "archive_org", + 32: "uptimebot", + 33: "slackbot", + 34: "apple", + 35: "discourse", + 36: "mattermost", + 37: "alexa", + 38: "lynx", + 39: "blank", + 40: "malformed", + 41: "suspicious", + 42: "semrush", + 43: "dotbot", + 44: "ahrefs", + 45: "proximic", + 46: "megaindex", + 47: "majestic", + 48: "cocolyze", + 49: "babbar", + 50: "surdotly", + 51: "domcop", + 52: "netcraft", + 53: "blexbot", + 54: "burf", + 55: "aspiegel", + 56: "mail_ru", + 57: "ccbot", + 58: "zgrab", + 59: "cloudsystemnetworks", + 60: "maui", + 61: "curl", + 62: "python", + 63: "go", + 64: "headlesschrome", + 65: "awesome_bot", } var markToAgent = map[string]string{ "OPR": "opera", @@ -737,6 +743,7 @@ var markToAgent = map[string]string{ "Slackbot": "slackbot", "Slack": "slackbot", "Discordbot": "discord", + "TelegramBot": "telegram", "Twitterbot": "twitter", "facebookexternalhit": "facebook", "Facebot": "facebook", @@ -754,6 +761,7 @@ var markToAgent = map[string]string{ "Cocolyzebot": "cocolyze", "Barkrowler": "babbar", "SurdotlyBot": "surdotly", + "DomCopBot": "domcop", "NetcraftSurveyAgent": "netcraft", "BLEXBot": "blexbot", "Burf": "burf", @@ -762,6 +770,7 @@ var markToAgent = map[string]string{ "CCBot": "ccbot", "zgrab": "zgrab", "Nimbostratus": "cloudsystemnetworks", + "MauiBot": "maui", "curl": "curl", "python": "python", "Go": "go", @@ -776,7 +785,7 @@ var markToID = map[string]int{ "MSIE": 6, "Trident": 7, "Edge": 5, - "Lynx": 37, + "Lynx": 38, "SamsungBrowser": 10, "UCBrowser": 11, "Google": 12, @@ -797,42 +806,45 @@ var markToID = map[string]int{ "Cliqzbot": 18, "netEstate": 19, "SeznamBot": 25, - "CloudFlare": 29, - "archive": 30, - "Uptimebot": 31, - "Slackbot": 32, - "Slack": 32, + "CloudFlare": 30, + "archive": 31, + "Uptimebot": 32, + "Slackbot": 33, + "Slack": 33, "Discordbot": 26, - "Twitterbot": 27, - "facebookexternalhit": 28, - "Facebot": 28, - "Applebot": 33, - "Discourse": 34, - "mattermost": 35, - "ia_archiver": 36, - "SemrushBot": 41, - "DotBot": 42, - "AhrefsBot": 43, - "proximic": 44, - "MegaIndex": 45, - "MJ12bot": 46, - "mj12bot": 46, - "Cocolyzebot": 47, - "Barkrowler": 48, - "SurdotlyBot": 49, - "NetcraftSurveyAgent": 50, - "BLEXBot": 51, - "Burf": 52, - "AspiegelBot": 53, - "RU_Bot": 54, - "CCBot": 55, - "zgrab": 56, - "Nimbostratus": 57, - "curl": 58, - "python": 59, - "Go": 60, - "HeadlessChrome": 61, - "awesome_bot": 62, + "TelegramBot": 27, + "Twitterbot": 28, + "facebookexternalhit": 29, + "Facebot": 29, + "Applebot": 34, + "Discourse": 35, + "mattermost": 36, + "ia_archiver": 37, + "SemrushBot": 42, + "DotBot": 43, + "AhrefsBot": 44, + "proximic": 45, + "MegaIndex": 46, + "MJ12bot": 47, + "mj12bot": 47, + "Cocolyzebot": 48, + "Barkrowler": 49, + "SurdotlyBot": 50, + "DomCopBot": 51, + "NetcraftSurveyAgent": 52, + "BLEXBot": 53, + "Burf": 54, + "AspiegelBot": 55, + "RU_Bot": 56, + "CCBot": 57, + "zgrab": 58, + "Nimbostratus": 59, + "MauiBot": 60, + "curl": 61, + "python": 62, + "Go": 63, + "HeadlessChrome": 64, + "awesome_bot": 65, } /*var agentRank = map[string]int{ "opera":9, @@ -902,11 +914,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) { if err != nil { return nil, err } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, req *http.Request) { + UploadHandler: func(w http.ResponseWriter, r *http.Request) { writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req) + http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) }, extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), requestLogger: log.New(f, "", log.LstdFlags), @@ -968,7 +979,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { pre += "\n" } r.DumpRequest(req,pre+"Suspicious Request") - co.AgentViewCounter.Bump(40) + co.AgentViewCounter.Bump(41) } func isLocalHost(h string) bool { @@ -987,7 +998,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.WriteHeader(200) // 400 w.Write([]byte("")) r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ)) - co.AgentViewCounter.Bump(39) + co.AgentViewCounter.Bump(40) } // Split the Host and Port string @@ -1126,7 +1137,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another if ua == "" { - co.AgentViewCounter.Bump(38) + co.AgentViewCounter.Bump(39) if c.Dev.DebugMode { var pre string for _, char := range req.UserAgent() { @@ -1216,7 +1227,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if strings.Contains(ua,"rv:11") { agent = 6 } - case 56: + case 58: r.SuspiciousRequest(req,"Vuln Scanner") } diff --git a/langs/english.json b/langs/english.json index fa36dac8..29bfcfa8 100644 --- a/langs/english.json +++ b/langs/english.json @@ -212,6 +212,7 @@ "duckduckgo":"DuckDuckBot", "seznambot":"SeznamBot", "discord":"Discord", + "telegram":"TelegramBot", "twitter":"Twitterbot", "cloudflare":"Cloudflare Alwayson", "archive_org":"Archive.org", @@ -233,6 +234,7 @@ "cocolyze":"Cocolyze", "babbar":"Babbar", "surdotly":"Surdotly", + "domcop":"DomCopBot", "netcraft":"Netcraft", "blexbot":"BLEXBot", "burf":"Burf.co", @@ -241,6 +243,7 @@ "ccbot":"CCBot", "zgrab":"Zgrab App Scanner", "cloudsystemnetworks":"Nimbostratus / Cloud System Networks", + "maui":"MauiBot", "curl":"curl", "python":"Python Bot", "go":"Go Bot", diff --git a/router_gen/main.go b/router_gen/main.go index 9ba19631..65818884 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -258,6 +258,7 @@ func main() { "duckduckgo", "seznambot", "discord", + "telegram", "twitter", "facebook", "cloudflare", @@ -281,6 +282,7 @@ func main() { "cocolyze", "babbar", "surdotly", + "domcop", "netcraft", "blexbot", "burf", @@ -289,6 +291,7 @@ func main() { "ccbot", "zgrab", "cloudsystemnetworks", + "maui", "curl", "python", "go", @@ -337,6 +340,7 @@ func main() { "Slackbot", "Slack", "Discordbot", + "TelegramBot", "Twitterbot", "facebookexternalhit", "Facebot", @@ -355,6 +359,7 @@ func main() { "Cocolyzebot", "Barkrowler", "SurdotlyBot", + "DomCopBot", "NetcraftSurveyAgent", "BLEXBot", "Burf", @@ -363,6 +368,7 @@ func main() { "CCBot", "zgrab", "Nimbostratus", + "MauiBot", "curl", "python", "Go", @@ -406,6 +412,7 @@ func main() { "Slackbot": "slackbot", "Slack": "slackbot", "Discordbot": "discord", + "TelegramBot": "telegram", "Twitterbot": "twitter", "facebookexternalhit": "facebook", "Facebot": "facebook", @@ -424,14 +431,17 @@ func main() { "Cocolyzebot": "cocolyze", "Barkrowler": "babbar", "SurdotlyBot": "surdotly", + "DomCopBot":"domcop", "NetcraftSurveyAgent": "netcraft", "BLEXBot": "blexbot", "Burf": "burf", "AspiegelBot": "aspiegel", + "PetalBot": "aspiegel", "RU_Bot": "mail_ru", // Mail.RU_Bot "CCBot": "ccbot", "zgrab": "zgrab", "Nimbostratus": "cloudsystemnetworks", + "MauiBot":"maui", "curl": "curl", "python": "python", "Go": "go", @@ -477,29 +487,29 @@ var RouteMap = map[string]interface{}{ {{range .AllRouteNames}} } // ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS -var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}} - "{{$element.Plain}}": {{$index}},{{end}} +var routeMapEnum = map[string]int{ {{range $index, $el := .AllRouteNames}} + "{{$el.Plain}}": {{$index}},{{end}} } -var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} - {{$index}}: "{{$element.Plain}}",{{end}} +var reverseRouteMapEnum = map[int]string{ {{range $index, $el := .AllRouteNames}} + {{$index}}: "{{$el.Plain}}",{{end}} } -var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}} - "{{$element}}": {{$index}},{{end}} +var osMapEnum = map[string]int{ {{range $index, $el := .AllOSNames}} + "{{$el}}": {{$index}},{{end}} } -var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}} - {{$index}}: "{{$element}}",{{end}} +var reverseOSMapEnum = map[int]string{ {{range $index, $el := .AllOSNames}} + {{$index}}: "{{$el}}",{{end}} } -var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} - "{{$element}}": {{$index}},{{end}} +var agentMapEnum = map[string]int{ {{range $index, $el := .AllAgentNames}} + "{{$el}}": {{$index}},{{end}} } -var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}} - {{$index}}: "{{$element}}",{{end}} +var reverseAgentMapEnum = map[int]string{ {{range $index, $el := .AllAgentNames}} + {{$index}}: "{{$el}}",{{end}} } -var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}} - "{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}} +var markToAgent = map[string]string{ {{range $index, $el := .AllAgentMarkNames}} + "{{$el}}": "{{index $.AllAgentMarks $el}}",{{end}} } -var markToID = map[string]int{ {{range $index, $element := .AllAgentMarkNames}} - "{{$element}}": {{index $.AllAgentMarkIDs $element}},{{end}} +var markToID = map[string]int{ {{range $index, $el := .AllAgentMarkNames}} + "{{$el}}": {{index $.AllAgentMarkIDs $el}},{{end}} } /*var agentRank = map[string]int{ "opera":9, @@ -569,11 +579,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) { if err != nil { return nil, err } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, req *http.Request) { + UploadHandler: func(w http.ResponseWriter, r *http.Request) { writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req) + http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) }, extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), requestLogger: log.New(f, "", log.LstdFlags),