add telegram, domcop, maui and alt aspiegel user agents

reduce boilerplate in routergen
This commit is contained in:
Azareal 2020-05-29 08:41:19 +10:00
parent 7211ed4f0e
commit 75d1d6fcbe
3 changed files with 157 additions and 134 deletions

View File

@ -600,42 +600,45 @@ var agentMapEnum = map[string]int{
"duckduckgo": 24,
"seznambot": 25,
"discord": 26,
"twitter": 27,
"facebook": 28,
"cloudflare": 29,
"archive_org": 30,
"uptimebot": 31,
"slackbot": 32,
"apple": 33,
"discourse": 34,
"mattermost": 35,
"alexa": 36,
"lynx": 37,
"blank": 38,
"malformed": 39,
"suspicious": 40,
"semrush": 41,
"dotbot": 42,
"ahrefs": 43,
"proximic": 44,
"megaindex": 45,
"majestic": 46,
"cocolyze": 47,
"babbar": 48,
"surdotly": 49,
"netcraft": 50,
"blexbot": 51,
"burf": 52,
"aspiegel": 53,
"mail_ru": 54,
"ccbot": 55,
"zgrab": 56,
"cloudsystemnetworks": 57,
"curl": 58,
"python": 59,
"go": 60,
"headlesschrome": 61,
"awesome_bot": 62,
"telegram": 27,
"twitter": 28,
"facebook": 29,
"cloudflare": 30,
"archive_org": 31,
"uptimebot": 32,
"slackbot": 33,
"apple": 34,
"discourse": 35,
"mattermost": 36,
"alexa": 37,
"lynx": 38,
"blank": 39,
"malformed": 40,
"suspicious": 41,
"semrush": 42,
"dotbot": 43,
"ahrefs": 44,
"proximic": 45,
"megaindex": 46,
"majestic": 47,
"cocolyze": 48,
"babbar": 49,
"surdotly": 50,
"domcop": 51,
"netcraft": 52,
"blexbot": 53,
"burf": 54,
"aspiegel": 55,
"mail_ru": 56,
"ccbot": 57,
"zgrab": 58,
"cloudsystemnetworks": 59,
"maui": 60,
"curl": 61,
"python": 62,
"go": 63,
"headlesschrome": 64,
"awesome_bot": 65,
}
var reverseAgentMapEnum = map[int]string{
0: "unknown",
@ -665,42 +668,45 @@ var reverseAgentMapEnum = map[int]string{
24: "duckduckgo",
25: "seznambot",
26: "discord",
27: "twitter",
28: "facebook",
29: "cloudflare",
30: "archive_org",
31: "uptimebot",
32: "slackbot",
33: "apple",
34: "discourse",
35: "mattermost",
36: "alexa",
37: "lynx",
38: "blank",
39: "malformed",
40: "suspicious",
41: "semrush",
42: "dotbot",
43: "ahrefs",
44: "proximic",
45: "megaindex",
46: "majestic",
47: "cocolyze",
48: "babbar",
49: "surdotly",
50: "netcraft",
51: "blexbot",
52: "burf",
53: "aspiegel",
54: "mail_ru",
55: "ccbot",
56: "zgrab",
57: "cloudsystemnetworks",
58: "curl",
59: "python",
60: "go",
61: "headlesschrome",
62: "awesome_bot",
27: "telegram",
28: "twitter",
29: "facebook",
30: "cloudflare",
31: "archive_org",
32: "uptimebot",
33: "slackbot",
34: "apple",
35: "discourse",
36: "mattermost",
37: "alexa",
38: "lynx",
39: "blank",
40: "malformed",
41: "suspicious",
42: "semrush",
43: "dotbot",
44: "ahrefs",
45: "proximic",
46: "megaindex",
47: "majestic",
48: "cocolyze",
49: "babbar",
50: "surdotly",
51: "domcop",
52: "netcraft",
53: "blexbot",
54: "burf",
55: "aspiegel",
56: "mail_ru",
57: "ccbot",
58: "zgrab",
59: "cloudsystemnetworks",
60: "maui",
61: "curl",
62: "python",
63: "go",
64: "headlesschrome",
65: "awesome_bot",
}
var markToAgent = map[string]string{
"OPR": "opera",
@ -737,6 +743,7 @@ var markToAgent = map[string]string{
"Slackbot": "slackbot",
"Slack": "slackbot",
"Discordbot": "discord",
"TelegramBot": "telegram",
"Twitterbot": "twitter",
"facebookexternalhit": "facebook",
"Facebot": "facebook",
@ -754,6 +761,7 @@ var markToAgent = map[string]string{
"Cocolyzebot": "cocolyze",
"Barkrowler": "babbar",
"SurdotlyBot": "surdotly",
"DomCopBot": "domcop",
"NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot",
"Burf": "burf",
@ -762,6 +770,7 @@ var markToAgent = map[string]string{
"CCBot": "ccbot",
"zgrab": "zgrab",
"Nimbostratus": "cloudsystemnetworks",
"MauiBot": "maui",
"curl": "curl",
"python": "python",
"Go": "go",
@ -776,7 +785,7 @@ var markToID = map[string]int{
"MSIE": 6,
"Trident": 7,
"Edge": 5,
"Lynx": 37,
"Lynx": 38,
"SamsungBrowser": 10,
"UCBrowser": 11,
"Google": 12,
@ -797,42 +806,45 @@ var markToID = map[string]int{
"Cliqzbot": 18,
"netEstate": 19,
"SeznamBot": 25,
"CloudFlare": 29,
"archive": 30,
"Uptimebot": 31,
"Slackbot": 32,
"Slack": 32,
"CloudFlare": 30,
"archive": 31,
"Uptimebot": 32,
"Slackbot": 33,
"Slack": 33,
"Discordbot": 26,
"Twitterbot": 27,
"facebookexternalhit": 28,
"Facebot": 28,
"Applebot": 33,
"Discourse": 34,
"mattermost": 35,
"ia_archiver": 36,
"SemrushBot": 41,
"DotBot": 42,
"AhrefsBot": 43,
"proximic": 44,
"MegaIndex": 45,
"MJ12bot": 46,
"mj12bot": 46,
"Cocolyzebot": 47,
"Barkrowler": 48,
"SurdotlyBot": 49,
"NetcraftSurveyAgent": 50,
"BLEXBot": 51,
"Burf": 52,
"AspiegelBot": 53,
"RU_Bot": 54,
"CCBot": 55,
"zgrab": 56,
"Nimbostratus": 57,
"curl": 58,
"python": 59,
"Go": 60,
"HeadlessChrome": 61,
"awesome_bot": 62,
"TelegramBot": 27,
"Twitterbot": 28,
"facebookexternalhit": 29,
"Facebot": 29,
"Applebot": 34,
"Discourse": 35,
"mattermost": 36,
"ia_archiver": 37,
"SemrushBot": 42,
"DotBot": 43,
"AhrefsBot": 44,
"proximic": 45,
"MegaIndex": 46,
"MJ12bot": 47,
"mj12bot": 47,
"Cocolyzebot": 48,
"Barkrowler": 49,
"SurdotlyBot": 50,
"DomCopBot": 51,
"NetcraftSurveyAgent": 52,
"BLEXBot": 53,
"Burf": 54,
"AspiegelBot": 55,
"RU_Bot": 56,
"CCBot": 57,
"zgrab": 58,
"Nimbostratus": 59,
"MauiBot": 60,
"curl": 61,
"python": 62,
"Go": 63,
"HeadlessChrome": 64,
"awesome_bot": 65,
}
/*var agentRank = map[string]int{
"opera":9,
@ -902,11 +914,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
if err != nil {
return nil, err
}
return &GenRouter{
UploadHandler: func(w http.ResponseWriter, req *http.Request) {
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
},
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags),
@ -968,7 +979,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
pre += "\n"
}
r.DumpRequest(req,pre+"Suspicious Request")
co.AgentViewCounter.Bump(40)
co.AgentViewCounter.Bump(41)
}
func isLocalHost(h string) bool {
@ -987,7 +998,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(200) // 400
w.Write([]byte(""))
r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ))
co.AgentViewCounter.Bump(39)
co.AgentViewCounter.Bump(40)
}
// Split the Host and Port string
@ -1126,7 +1137,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
if ua == "" {
co.AgentViewCounter.Bump(38)
co.AgentViewCounter.Bump(39)
if c.Dev.DebugMode {
var pre string
for _, char := range req.UserAgent() {
@ -1216,7 +1227,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if strings.Contains(ua,"rv:11") {
agent = 6
}
case 56:
case 58:
r.SuspiciousRequest(req,"Vuln Scanner")
}

View File

@ -212,6 +212,7 @@
"duckduckgo":"DuckDuckBot",
"seznambot":"SeznamBot",
"discord":"Discord",
"telegram":"TelegramBot",
"twitter":"Twitterbot",
"cloudflare":"Cloudflare Alwayson",
"archive_org":"Archive.org",
@ -233,6 +234,7 @@
"cocolyze":"Cocolyze",
"babbar":"Babbar",
"surdotly":"Surdotly",
"domcop":"DomCopBot",
"netcraft":"Netcraft",
"blexbot":"BLEXBot",
"burf":"Burf.co",
@ -241,6 +243,7 @@
"ccbot":"CCBot",
"zgrab":"Zgrab App Scanner",
"cloudsystemnetworks":"Nimbostratus / Cloud System Networks",
"maui":"MauiBot",
"curl":"curl",
"python":"Python Bot",
"go":"Go Bot",

View File

@ -258,6 +258,7 @@ func main() {
"duckduckgo",
"seznambot",
"discord",
"telegram",
"twitter",
"facebook",
"cloudflare",
@ -281,6 +282,7 @@ func main() {
"cocolyze",
"babbar",
"surdotly",
"domcop",
"netcraft",
"blexbot",
"burf",
@ -289,6 +291,7 @@ func main() {
"ccbot",
"zgrab",
"cloudsystemnetworks",
"maui",
"curl",
"python",
"go",
@ -337,6 +340,7 @@ func main() {
"Slackbot",
"Slack",
"Discordbot",
"TelegramBot",
"Twitterbot",
"facebookexternalhit",
"Facebot",
@ -355,6 +359,7 @@ func main() {
"Cocolyzebot",
"Barkrowler",
"SurdotlyBot",
"DomCopBot",
"NetcraftSurveyAgent",
"BLEXBot",
"Burf",
@ -363,6 +368,7 @@ func main() {
"CCBot",
"zgrab",
"Nimbostratus",
"MauiBot",
"curl",
"python",
"Go",
@ -406,6 +412,7 @@ func main() {
"Slackbot": "slackbot",
"Slack": "slackbot",
"Discordbot": "discord",
"TelegramBot": "telegram",
"Twitterbot": "twitter",
"facebookexternalhit": "facebook",
"Facebot": "facebook",
@ -424,14 +431,17 @@ func main() {
"Cocolyzebot": "cocolyze",
"Barkrowler": "babbar",
"SurdotlyBot": "surdotly",
"DomCopBot":"domcop",
"NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot",
"Burf": "burf",
"AspiegelBot": "aspiegel",
"PetalBot": "aspiegel",
"RU_Bot": "mail_ru", // Mail.RU_Bot
"CCBot": "ccbot",
"zgrab": "zgrab",
"Nimbostratus": "cloudsystemnetworks",
"MauiBot":"maui",
"curl": "curl",
"python": "python",
"Go": "go",
@ -477,29 +487,29 @@ var RouteMap = map[string]interface{}{ {{range .AllRouteNames}}
}
// ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS
var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
"{{$element.Plain}}": {{$index}},{{end}}
var routeMapEnum = map[string]int{ {{range $index, $el := .AllRouteNames}}
"{{$el.Plain}}": {{$index}},{{end}}
}
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
{{$index}}: "{{$element.Plain}}",{{end}}
var reverseRouteMapEnum = map[int]string{ {{range $index, $el := .AllRouteNames}}
{{$index}}: "{{$el.Plain}}",{{end}}
}
var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}}
"{{$element}}": {{$index}},{{end}}
var osMapEnum = map[string]int{ {{range $index, $el := .AllOSNames}}
"{{$el}}": {{$index}},{{end}}
}
var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}}
{{$index}}: "{{$element}}",{{end}}
var reverseOSMapEnum = map[int]string{ {{range $index, $el := .AllOSNames}}
{{$index}}: "{{$el}}",{{end}}
}
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}}
var agentMapEnum = map[string]int{ {{range $index, $el := .AllAgentNames}}
"{{$el}}": {{$index}},{{end}}
}
var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}}
{{$index}}: "{{$element}}",{{end}}
var reverseAgentMapEnum = map[int]string{ {{range $index, $el := .AllAgentNames}}
{{$index}}: "{{$el}}",{{end}}
}
var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}}
"{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}}
var markToAgent = map[string]string{ {{range $index, $el := .AllAgentMarkNames}}
"{{$el}}": "{{index $.AllAgentMarks $el}}",{{end}}
}
var markToID = map[string]int{ {{range $index, $element := .AllAgentMarkNames}}
"{{$element}}": {{index $.AllAgentMarkIDs $element}},{{end}}
var markToID = map[string]int{ {{range $index, $el := .AllAgentMarkNames}}
"{{$el}}": {{index $.AllAgentMarkIDs $el}},{{end}}
}
/*var agentRank = map[string]int{
"opera":9,
@ -569,11 +579,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
if err != nil {
return nil, err
}
return &GenRouter{
UploadHandler: func(w http.ResponseWriter, req *http.Request) {
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
},
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags),