a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add telegram, domcop, maui and alt aspiegel user agents 

reduce boilerplate in routergen
This commit is contained in:
Azareal 2020-05-29 08:41:19 +10:00
parent 7211ed4f0e
commit 75d1d6fcbe
3 changed files with 157 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
gen_router.go
View File

@ -600,42 +600,45 @@ var agentMapEnum = map[string]int{
"duckduckgo": 24, "duckduckgo": 24,
"seznambot": 25, "seznambot": 25,
"discord": 26, "discord": 26,
"twitter": 27, "telegram": 27,
"facebook": 28, "twitter": 28,
"cloudflare": 29, "facebook": 29,
"archive_org": 30, "cloudflare": 30,
"uptimebot": 31, "archive_org": 31,
"slackbot": 32, "uptimebot": 32,
"apple": 33, "slackbot": 33,
"discourse": 34, "apple": 34,
"mattermost": 35, "discourse": 35,
"alexa": 36, "mattermost": 36,
"lynx": 37, "alexa": 37,
"blank": 38, "lynx": 38,
"malformed": 39, "blank": 39,
"suspicious": 40, "malformed": 40,
"semrush": 41, "suspicious": 41,
"dotbot": 42, "semrush": 42,
"ahrefs": 43, "dotbot": 43,
"proximic": 44, "ahrefs": 44,
"megaindex": 45, "proximic": 45,
"majestic": 46, "megaindex": 46,
"cocolyze": 47, "majestic": 47,
"babbar": 48, "cocolyze": 48,
"surdotly": 49, "babbar": 49,
"netcraft": 50, "surdotly": 50,
"blexbot": 51, "domcop": 51,
"burf": 52, "netcraft": 52,
"aspiegel": 53, "blexbot": 53,
"mail_ru": 54, "burf": 54,
"ccbot": 55, "aspiegel": 55,
"zgrab": 56, "mail_ru": 56,
"cloudsystemnetworks": 57, "ccbot": 57,
"curl": 58, "zgrab": 58,
"python": 59, "cloudsystemnetworks": 59,
"go": 60, "maui": 60,
"headlesschrome": 61, "curl": 61,
"awesome_bot": 62, "python": 62,
"go": 63,
"headlesschrome": 64,
"awesome_bot": 65,
} }
var reverseAgentMapEnum = map[int]string{ var reverseAgentMapEnum = map[int]string{
0: "unknown", 0: "unknown",
@ -665,42 +668,45 @@ var reverseAgentMapEnum = map[int]string{
24: "duckduckgo", 24: "duckduckgo",
25: "seznambot", 25: "seznambot",
26: "discord", 26: "discord",
27: "twitter", 27: "telegram",
28: "facebook", 28: "twitter",
29: "cloudflare", 29: "facebook",
30: "archive_org", 30: "cloudflare",
31: "uptimebot", 31: "archive_org",
32: "slackbot", 32: "uptimebot",
33: "apple", 33: "slackbot",
34: "discourse", 34: "apple",
35: "mattermost", 35: "discourse",
36: "alexa", 36: "mattermost",
37: "lynx", 37: "alexa",
38: "blank", 38: "lynx",
39: "malformed", 39: "blank",
40: "suspicious", 40: "malformed",
41: "semrush", 41: "suspicious",
42: "dotbot", 42: "semrush",
43: "ahrefs", 43: "dotbot",
44: "proximic", 44: "ahrefs",
45: "megaindex", 45: "proximic",
46: "majestic", 46: "megaindex",
47: "cocolyze", 47: "majestic",
48: "babbar", 48: "cocolyze",
49: "surdotly", 49: "babbar",
50: "netcraft", 50: "surdotly",
51: "blexbot", 51: "domcop",
52: "burf", 52: "netcraft",
53: "aspiegel", 53: "blexbot",
54: "mail_ru", 54: "burf",
55: "ccbot", 55: "aspiegel",
56: "zgrab", 56: "mail_ru",
57: "cloudsystemnetworks", 57: "ccbot",
58: "curl", 58: "zgrab",
59: "python", 59: "cloudsystemnetworks",
60: "go", 60: "maui",
61: "headlesschrome", 61: "curl",
62: "awesome_bot", 62: "python",
63: "go",
64: "headlesschrome",
65: "awesome_bot",
} }
var markToAgent = map[string]string{ var markToAgent = map[string]string{
"OPR": "opera", "OPR": "opera",
@ -737,6 +743,7 @@ var markToAgent = map[string]string{
"Slackbot": "slackbot", "Slackbot": "slackbot",
"Slack": "slackbot", "Slack": "slackbot",
"Discordbot": "discord", "Discordbot": "discord",
"TelegramBot": "telegram",
"Twitterbot": "twitter", "Twitterbot": "twitter",
"facebookexternalhit": "facebook", "facebookexternalhit": "facebook",
"Facebot": "facebook", "Facebot": "facebook",
@ -754,6 +761,7 @@ var markToAgent = map[string]string{
"Cocolyzebot": "cocolyze", "Cocolyzebot": "cocolyze",
"Barkrowler": "babbar", "Barkrowler": "babbar",
"SurdotlyBot": "surdotly", "SurdotlyBot": "surdotly",
"DomCopBot": "domcop",
"NetcraftSurveyAgent": "netcraft", "NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot", "BLEXBot": "blexbot",
"Burf": "burf", "Burf": "burf",
@ -762,6 +770,7 @@ var markToAgent = map[string]string{
"CCBot": "ccbot", "CCBot": "ccbot",
"zgrab": "zgrab", "zgrab": "zgrab",
"Nimbostratus": "cloudsystemnetworks", "Nimbostratus": "cloudsystemnetworks",
"MauiBot": "maui",
"curl": "curl", "curl": "curl",
"python": "python", "python": "python",
"Go": "go", "Go": "go",
@ -776,7 +785,7 @@ var markToID = map[string]int{
"MSIE": 6, "MSIE": 6,
"Trident": 7, "Trident": 7,
"Edge": 5, "Edge": 5,
"Lynx": 37, "Lynx": 38,
"SamsungBrowser": 10, "SamsungBrowser": 10,
"UCBrowser": 11, "UCBrowser": 11,
"Google": 12, "Google": 12,
@ -797,42 +806,45 @@ var markToID = map[string]int{
"Cliqzbot": 18, "Cliqzbot": 18,
"netEstate": 19, "netEstate": 19,
"SeznamBot": 25, "SeznamBot": 25,
"CloudFlare": 29, "CloudFlare": 30,
"archive": 30, "archive": 31,
"Uptimebot": 31, "Uptimebot": 32,
"Slackbot": 32, "Slackbot": 33,
"Slack": 32, "Slack": 33,
"Discordbot": 26, "Discordbot": 26,
"Twitterbot": 27, "TelegramBot": 27,
"facebookexternalhit": 28, "Twitterbot": 28,
"Facebot": 28, "facebookexternalhit": 29,
"Applebot": 33, "Facebot": 29,
"Discourse": 34, "Applebot": 34,
"mattermost": 35, "Discourse": 35,
"ia_archiver": 36, "mattermost": 36,
"SemrushBot": 41, "ia_archiver": 37,
"DotBot": 42, "SemrushBot": 42,
"AhrefsBot": 43, "DotBot": 43,
"proximic": 44, "AhrefsBot": 44,
"MegaIndex": 45, "proximic": 45,
"MJ12bot": 46, "MegaIndex": 46,
"mj12bot": 46, "MJ12bot": 47,
"Cocolyzebot": 47, "mj12bot": 47,
"Barkrowler": 48, "Cocolyzebot": 48,
"SurdotlyBot": 49, "Barkrowler": 49,
"NetcraftSurveyAgent": 50, "SurdotlyBot": 50,
"BLEXBot": 51, "DomCopBot": 51,
"Burf": 52, "NetcraftSurveyAgent": 52,
"AspiegelBot": 53, "BLEXBot": 53,
"RU_Bot": 54, "Burf": 54,
"CCBot": 55, "AspiegelBot": 55,
"zgrab": 56, "RU_Bot": 56,
"Nimbostratus": 57, "CCBot": 57,
"curl": 58, "zgrab": 58,
"python": 59, "Nimbostratus": 59,
"Go": 60, "MauiBot": 60,
"HeadlessChrome": 61, "curl": 61,
"awesome_bot": 62, "python": 62,
"Go": 63,
"HeadlessChrome": 64,
"awesome_bot": 65,
} }
/*var agentRank = map[string]int{ /*var agentRank = map[string]int{
"opera":9, "opera":9,
@ -902,11 +914,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &GenRouter{ return &GenRouter{
UploadHandler: func(w http.ResponseWriter, req *http.Request) { UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w) writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req) http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
}, },
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags), requestLogger: log.New(f, "", log.LstdFlags),
@ -968,7 +979,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
pre += "\n" pre += "\n"
} }
r.DumpRequest(req,pre+"Suspicious Request") r.DumpRequest(req,pre+"Suspicious Request")
co.AgentViewCounter.Bump(40) co.AgentViewCounter.Bump(41)
} }
func isLocalHost(h string) bool { func isLocalHost(h string) bool {
@ -987,7 +998,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(200) // 400 w.WriteHeader(200) // 400
w.Write([]byte("")) w.Write([]byte(""))
r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ)) r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ))
co.AgentViewCounter.Bump(39) co.AgentViewCounter.Bump(40)
} }
// Split the Host and Port string // Split the Host and Port string
@ -1126,7 +1137,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
if ua == "" { if ua == "" {
co.AgentViewCounter.Bump(38) co.AgentViewCounter.Bump(39)
if c.Dev.DebugMode { if c.Dev.DebugMode {
var pre string var pre string
for _, char := range req.UserAgent() { for _, char := range req.UserAgent() {
@ -1216,7 +1227,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if strings.Contains(ua,"rv:11") { if strings.Contains(ua,"rv:11") {
agent = 6 agent = 6
} }
case 56: case 58:
r.SuspiciousRequest(req,"Vuln Scanner") r.SuspiciousRequest(req,"Vuln Scanner")
} }

3
langs/english.json
View File

@ -212,6 +212,7 @@
"duckduckgo":"DuckDuckBot", "duckduckgo":"DuckDuckBot",
"seznambot":"SeznamBot", "seznambot":"SeznamBot",
"discord":"Discord", "discord":"Discord",
"telegram":"TelegramBot",
"twitter":"Twitterbot", "twitter":"Twitterbot",
"cloudflare":"Cloudflare Alwayson", "cloudflare":"Cloudflare Alwayson",
"archive_org":"Archive.org", "archive_org":"Archive.org",
@ -233,6 +234,7 @@
"cocolyze":"Cocolyze", "cocolyze":"Cocolyze",
"babbar":"Babbar", "babbar":"Babbar",
"surdotly":"Surdotly", "surdotly":"Surdotly",
"domcop":"DomCopBot",
"netcraft":"Netcraft", "netcraft":"Netcraft",
"blexbot":"BLEXBot", "blexbot":"BLEXBot",
"burf":"Burf.co", "burf":"Burf.co",
@ -241,6 +243,7 @@
"ccbot":"CCBot", "ccbot":"CCBot",
"zgrab":"Zgrab App Scanner", "zgrab":"Zgrab App Scanner",
"cloudsystemnetworks":"Nimbostratus / Cloud System Networks", "cloudsystemnetworks":"Nimbostratus / Cloud System Networks",
"maui":"MauiBot",
"curl":"curl", "curl":"curl",
"python":"Python Bot", "python":"Python Bot",
"go":"Go Bot", "go":"Go Bot",

47
router_gen/main.go
View File

@ -258,6 +258,7 @@ func main() {
"duckduckgo", "duckduckgo",
"seznambot", "seznambot",
"discord", "discord",
"telegram",
"twitter", "twitter",
"facebook", "facebook",
"cloudflare", "cloudflare",
@ -281,6 +282,7 @@ func main() {
"cocolyze", "cocolyze",
"babbar", "babbar",
"surdotly", "surdotly",
"domcop",
"netcraft", "netcraft",
"blexbot", "blexbot",
"burf", "burf",
@ -289,6 +291,7 @@ func main() {
"ccbot", "ccbot",
"zgrab", "zgrab",
"cloudsystemnetworks", "cloudsystemnetworks",
"maui",
"curl", "curl",
"python", "python",
"go", "go",
@ -337,6 +340,7 @@ func main() {
"Slackbot", "Slackbot",
"Slack", "Slack",
"Discordbot", "Discordbot",
"TelegramBot",
"Twitterbot", "Twitterbot",
"facebookexternalhit", "facebookexternalhit",
"Facebot", "Facebot",
@ -355,6 +359,7 @@ func main() {
"Cocolyzebot", "Cocolyzebot",
"Barkrowler", "Barkrowler",
"SurdotlyBot", "SurdotlyBot",
"DomCopBot",
"NetcraftSurveyAgent", "NetcraftSurveyAgent",
"BLEXBot", "BLEXBot",
"Burf", "Burf",
@ -363,6 +368,7 @@ func main() {
"CCBot", "CCBot",
"zgrab", "zgrab",
"Nimbostratus", "Nimbostratus",
"MauiBot",
"curl", "curl",
"python", "python",
"Go", "Go",
@ -406,6 +412,7 @@ func main() {
"Slackbot": "slackbot", "Slackbot": "slackbot",
"Slack": "slackbot", "Slack": "slackbot",
"Discordbot": "discord", "Discordbot": "discord",
"TelegramBot": "telegram",
"Twitterbot": "twitter", "Twitterbot": "twitter",
"facebookexternalhit": "facebook", "facebookexternalhit": "facebook",
"Facebot": "facebook", "Facebot": "facebook",
@ -424,14 +431,17 @@ func main() {
"Cocolyzebot": "cocolyze", "Cocolyzebot": "cocolyze",
"Barkrowler": "babbar", "Barkrowler": "babbar",
"SurdotlyBot": "surdotly", "SurdotlyBot": "surdotly",
"DomCopBot":"domcop",
"NetcraftSurveyAgent": "netcraft", "NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot", "BLEXBot": "blexbot",
"Burf": "burf", "Burf": "burf",
"AspiegelBot": "aspiegel", "AspiegelBot": "aspiegel",
"PetalBot": "aspiegel",
"RU_Bot": "mail_ru", // Mail.RU_Bot "RU_Bot": "mail_ru", // Mail.RU_Bot
"CCBot": "ccbot", "CCBot": "ccbot",
"zgrab": "zgrab", "zgrab": "zgrab",
"Nimbostratus": "cloudsystemnetworks", "Nimbostratus": "cloudsystemnetworks",
"MauiBot":"maui",
"curl": "curl", "curl": "curl",
"python": "python", "python": "python",
"Go": "go", "Go": "go",
@ -477,29 +487,29 @@ var RouteMap = map[string]interface{}{ {{range .AllRouteNames}}
} }
// ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS // ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS
var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}} var routeMapEnum = map[string]int{ {{range $index, $el := .AllRouteNames}}
"{{$element.Plain}}": {{$index}},{{end}} "{{$el.Plain}}": {{$index}},{{end}}
} }
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $el := .AllRouteNames}}
{{$index}}: "{{$element.Plain}}",{{end}} {{$index}}: "{{$el.Plain}}",{{end}}
} }
var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}} var osMapEnum = map[string]int{ {{range $index, $el := .AllOSNames}}
"{{$element}}": {{$index}},{{end}} "{{$el}}": {{$index}},{{end}}
} }
var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}} var reverseOSMapEnum = map[int]string{ {{range $index, $el := .AllOSNames}}
{{$index}}: "{{$element}}",{{end}} {{$index}}: "{{$el}}",{{end}}
} }
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} var agentMapEnum = map[string]int{ {{range $index, $el := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}} "{{$el}}": {{$index}},{{end}}
} }
var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}} var reverseAgentMapEnum = map[int]string{ {{range $index, $el := .AllAgentNames}}
{{$index}}: "{{$element}}",{{end}} {{$index}}: "{{$el}}",{{end}}
} }
var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}} var markToAgent = map[string]string{ {{range $index, $el := .AllAgentMarkNames}}
"{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}} "{{$el}}": "{{index $.AllAgentMarks $el}}",{{end}}
} }
var markToID = map[string]int{ {{range $index, $element := .AllAgentMarkNames}} var markToID = map[string]int{ {{range $index, $el := .AllAgentMarkNames}}
"{{$element}}": {{index $.AllAgentMarkIDs $element}},{{end}} "{{$el}}": {{index $.AllAgentMarkIDs $el}},{{end}}
} }
/*var agentRank = map[string]int{ /*var agentRank = map[string]int{
"opera":9, "opera":9,
@ -569,11 +579,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &GenRouter{ return &GenRouter{
UploadHandler: func(w http.ResponseWriter, req *http.Request) { UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w) writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req) http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
}, },
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags), requestLogger: log.New(f, "", log.LstdFlags),