add cocolyze, babbar, surdotly, awesome_bot user agents
reduce matching in the router slightly use hookgen for router_after_filters use hookgen for router_pre_route shorten ua buffer variable name skip more symbols in initial ua loop shorten vulnscan string avoid unneccessary fork in ua logic
This commit is contained in:
parent
6be9a25e89
commit
d0bc74f58f
191
gen_router.go
191
gen_router.go
|
@ -20,6 +20,8 @@ import (
|
||||||
"github.com/Azareal/Gosora/uutils"
|
"github.com/Azareal/Gosora/uutils"
|
||||||
"github.com/Azareal/Gosora/routes"
|
"github.com/Azareal/Gosora/routes"
|
||||||
"github.com/Azareal/Gosora/routes/panel"
|
"github.com/Azareal/Gosora/routes/panel"
|
||||||
|
|
||||||
|
//"github.com/andybalholm/brotli"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrNoRoute = errors.New("That route doesn't exist.")
|
var ErrNoRoute = errors.New("That route doesn't exist.")
|
||||||
|
@ -618,18 +620,22 @@ var agentMapEnum = map[string]int{
|
||||||
"proximic": 44,
|
"proximic": 44,
|
||||||
"megaindex": 45,
|
"megaindex": 45,
|
||||||
"majestic": 46,
|
"majestic": 46,
|
||||||
"netcraft": 47,
|
"cocolyze": 47,
|
||||||
"blexbot": 48,
|
"babbar": 48,
|
||||||
"burf": 49,
|
"surdotly": 49,
|
||||||
"aspiegel": 50,
|
"netcraft": 50,
|
||||||
"mail_ru": 51,
|
"blexbot": 51,
|
||||||
"ccbot": 52,
|
"burf": 52,
|
||||||
"zgrab": 53,
|
"aspiegel": 53,
|
||||||
"cloudsystemnetworks": 54,
|
"mail_ru": 54,
|
||||||
"curl": 55,
|
"ccbot": 55,
|
||||||
"python": 56,
|
"zgrab": 56,
|
||||||
"go": 57,
|
"cloudsystemnetworks": 57,
|
||||||
"headlesschrome": 58,
|
"curl": 58,
|
||||||
|
"python": 59,
|
||||||
|
"go": 60,
|
||||||
|
"headlesschrome": 61,
|
||||||
|
"awesome_bot": 62,
|
||||||
}
|
}
|
||||||
var reverseAgentMapEnum = map[int]string{
|
var reverseAgentMapEnum = map[int]string{
|
||||||
0: "unknown",
|
0: "unknown",
|
||||||
|
@ -679,18 +685,22 @@ var reverseAgentMapEnum = map[int]string{
|
||||||
44: "proximic",
|
44: "proximic",
|
||||||
45: "megaindex",
|
45: "megaindex",
|
||||||
46: "majestic",
|
46: "majestic",
|
||||||
47: "netcraft",
|
47: "cocolyze",
|
||||||
48: "blexbot",
|
48: "babbar",
|
||||||
49: "burf",
|
49: "surdotly",
|
||||||
50: "aspiegel",
|
50: "netcraft",
|
||||||
51: "mail_ru",
|
51: "blexbot",
|
||||||
52: "ccbot",
|
52: "burf",
|
||||||
53: "zgrab",
|
53: "aspiegel",
|
||||||
54: "cloudsystemnetworks",
|
54: "mail_ru",
|
||||||
55: "curl",
|
55: "ccbot",
|
||||||
56: "python",
|
56: "zgrab",
|
||||||
57: "go",
|
57: "cloudsystemnetworks",
|
||||||
58: "headlesschrome",
|
58: "curl",
|
||||||
|
59: "python",
|
||||||
|
60: "go",
|
||||||
|
61: "headlesschrome",
|
||||||
|
62: "awesome_bot",
|
||||||
}
|
}
|
||||||
var markToAgent = map[string]string{
|
var markToAgent = map[string]string{
|
||||||
"OPR": "opera",
|
"OPR": "opera",
|
||||||
|
@ -741,6 +751,9 @@ var markToAgent = map[string]string{
|
||||||
"MegaIndex": "megaindex",
|
"MegaIndex": "megaindex",
|
||||||
"MJ12bot": "majestic",
|
"MJ12bot": "majestic",
|
||||||
"mj12bot": "majestic",
|
"mj12bot": "majestic",
|
||||||
|
"Cocolyzebot": "cocolyze",
|
||||||
|
"Barkrowler": "babbar",
|
||||||
|
"SurdotlyBot": "surdotly",
|
||||||
"NetcraftSurveyAgent": "netcraft",
|
"NetcraftSurveyAgent": "netcraft",
|
||||||
"BLEXBot": "blexbot",
|
"BLEXBot": "blexbot",
|
||||||
"Burf": "burf",
|
"Burf": "burf",
|
||||||
|
@ -753,6 +766,7 @@ var markToAgent = map[string]string{
|
||||||
"python": "python",
|
"python": "python",
|
||||||
"Go": "go",
|
"Go": "go",
|
||||||
"HeadlessChrome": "headlesschrome",
|
"HeadlessChrome": "headlesschrome",
|
||||||
|
"awesome_bot": "awesome_bot",
|
||||||
}
|
}
|
||||||
var markToID = map[string]int{
|
var markToID = map[string]int{
|
||||||
"OPR": 3,
|
"OPR": 3,
|
||||||
|
@ -803,18 +817,22 @@ var markToID = map[string]int{
|
||||||
"MegaIndex": 45,
|
"MegaIndex": 45,
|
||||||
"MJ12bot": 46,
|
"MJ12bot": 46,
|
||||||
"mj12bot": 46,
|
"mj12bot": 46,
|
||||||
"NetcraftSurveyAgent": 47,
|
"Cocolyzebot": 47,
|
||||||
"BLEXBot": 48,
|
"Barkrowler": 48,
|
||||||
"Burf": 49,
|
"SurdotlyBot": 49,
|
||||||
"AspiegelBot": 50,
|
"NetcraftSurveyAgent": 50,
|
||||||
"RU_Bot": 51,
|
"BLEXBot": 51,
|
||||||
"CCBot": 52,
|
"Burf": 52,
|
||||||
"zgrab": 53,
|
"AspiegelBot": 53,
|
||||||
"Nimbostratus": 54,
|
"RU_Bot": 54,
|
||||||
"curl": 55,
|
"CCBot": 55,
|
||||||
"python": 56,
|
"zgrab": 56,
|
||||||
"Go": 57,
|
"Nimbostratus": 57,
|
||||||
"HeadlessChrome": 58,
|
"curl": 58,
|
||||||
|
"python": 59,
|
||||||
|
"Go": 60,
|
||||||
|
"HeadlessChrome": 61,
|
||||||
|
"awesome_bot": 62,
|
||||||
}
|
}
|
||||||
/*var agentRank = map[string]int{
|
/*var agentRank = map[string]int{
|
||||||
"opera":9,
|
"opera":9,
|
||||||
|
@ -957,7 +975,9 @@ func isLocalHost(h string) bool {
|
||||||
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//var brPool = sync.Pool{}
|
||||||
var gzipPool = sync.Pool{}
|
var gzipPool = sync.Pool{}
|
||||||
|
//var uaBufPool = sync.Pool{}
|
||||||
|
|
||||||
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
||||||
// TODO: SetDefaultPath
|
// TODO: SetDefaultPath
|
||||||
|
@ -1040,7 +1060,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
lp := strings.ToLower(req.URL.Path)
|
lp := strings.ToLower(req.URL.Path)
|
||||||
// TODO: Flag any requests which has a dot with anything but a number after that
|
// TODO: Flag any requests which has a dot with anything but a number after that
|
||||||
// TODO: Use HasSuffix to avoid over-scanning?
|
// TODO: Use HasSuffix to avoid over-scanning?
|
||||||
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") {
|
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
|
||||||
r.SuspiciousRequest(req,"Bad snippet in path")
|
r.SuspiciousRequest(req,"Bad snippet in path")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1053,7 +1073,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
|
|
||||||
// TODO: Use the same hook table as downstream
|
// TODO: Use the same hook table as downstream
|
||||||
hTbl := c.GetHookTable()
|
hTbl := c.GetHookTable()
|
||||||
skip, ferr := hTbl.VhookSkippable("router_after_filters", w, req, prefix)
|
skip, ferr := c.H_router_after_filters_hook(hTbl, w, req, prefix)
|
||||||
if skip || ferr != nil {
|
if skip || ferr != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -1116,18 +1136,22 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// WIP UA Parser
|
// WIP UA Parser
|
||||||
|
//var ii = uaBufPool.Get()
|
||||||
|
var buf []byte
|
||||||
|
//if ii != nil {
|
||||||
|
// buf = ii.([]byte)
|
||||||
|
//}
|
||||||
var items []string
|
var items []string
|
||||||
var buffer []byte
|
|
||||||
var os int
|
var os int
|
||||||
for _, it := range uutils.StringToBytes(ua) {
|
for _, it := range uutils.StringToBytes(ua) {
|
||||||
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
|
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
|
||||||
// TODO: Store an index and slice that instead?
|
// TODO: Store an index and slice that instead?
|
||||||
buffer = append(buffer, it)
|
buf = append(buf, it)
|
||||||
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buffer,[]byte("http")))*/ || it == ',' || it == '/' {
|
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buf,[]byte("http")))*/ || it == ',' || it == '/' {
|
||||||
if len(buffer) != 0 {
|
if len(buf) != 0 {
|
||||||
if len(buffer) > 2 {
|
if len(buf) > 2 {
|
||||||
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
|
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
|
||||||
switch(uutils.BytesToString(buffer)) {
|
switch(uutils.BytesToString(buf)) {
|
||||||
case "Windows":
|
case "Windows":
|
||||||
os = 1
|
os = 1
|
||||||
case "Linux":
|
case "Linux":
|
||||||
|
@ -1138,23 +1162,24 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
os = 5
|
os = 5
|
||||||
case "Android":
|
case "Android":
|
||||||
os = 4
|
os = 4
|
||||||
case "like","compatible":
|
case "like","compatible","NT","X","KHTML":
|
||||||
// Skip these words
|
// Skip these words
|
||||||
default:
|
default:
|
||||||
items = append(items, string(buffer))
|
items = append(items, string(buf))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
buffer = buffer[:0]
|
buf = buf[:0]
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// TODO: Test this
|
// TODO: Test this
|
||||||
items = items[:0]
|
items = items[:0]
|
||||||
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
||||||
r.requestLogger.Print("UA Buf: ", buffer)
|
r.requestLogger.Print("UA Buf: ", buf)
|
||||||
r.requestLogger.Print("UA Buf String: ", string(buffer))
|
r.requestLogger.Print("UA Buf String: ", string(buf))
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
//uaBufPool.Put(buf)
|
||||||
|
|
||||||
// Iterate over this in reverse as the real UA tends to be on the right side
|
// Iterate over this in reverse as the real UA tends to be on the right side
|
||||||
for i := len(items) - 1; i >= 0; i-- {
|
for i := len(items) - 1; i >= 0; i-- {
|
||||||
|
@ -1171,6 +1196,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
r.requestLogger.Print("parsed agent: ", agent)
|
r.requestLogger.Print("parsed agent: ", agent)
|
||||||
r.requestLogger.Print("os: ", os)
|
r.requestLogger.Print("os: ", os)
|
||||||
r.requestLogger.Printf("items: %+v\n",items)
|
r.requestLogger.Printf("items: %+v\n",items)
|
||||||
|
/*for _, it := range items {
|
||||||
|
r.requestLogger.Printf("it: %+v\n",string(it))
|
||||||
|
}*/
|
||||||
}
|
}
|
||||||
|
|
||||||
// Special handling
|
// Special handling
|
||||||
|
@ -1188,12 +1216,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
if strings.Contains(ua,"rv:11") {
|
if strings.Contains(ua,"rv:11") {
|
||||||
agent = 6
|
agent = 6
|
||||||
}
|
}
|
||||||
case 53:
|
case 56:
|
||||||
r.SuspiciousRequest(req,"Vulnerability Scanner")
|
r.SuspiciousRequest(req,"Vuln Scanner")
|
||||||
}
|
}
|
||||||
|
|
||||||
if agent == 0 {
|
if agent == 0 {
|
||||||
co.AgentViewCounter.Bump(0)
|
//co.AgentViewCounter.Bump(0)
|
||||||
if c.Dev.DebugMode {
|
if c.Dev.DebugMode {
|
||||||
var pre string
|
var pre string
|
||||||
for _, char := range req.UserAgent() {
|
for _, char := range req.UserAgent() {
|
||||||
|
@ -1203,10 +1231,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
} else {
|
} else {
|
||||||
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
|
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
|
||||||
}
|
}
|
||||||
} else {
|
}// else {
|
||||||
//co.AgentViewCounter.Bump(agentMapEnum[agent])
|
//co.AgentViewCounter.Bump(agentMapEnum[agent])
|
||||||
co.AgentViewCounter.Bump(agent)
|
co.AgentViewCounter.Bump(agent)
|
||||||
}
|
//}
|
||||||
co.OSViewCounter.Bump(os)
|
co.OSViewCounter.Bump(os)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1270,7 +1298,31 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
//log.Println("req: ", req)
|
//log.Println("req: ", req)
|
||||||
|
|
||||||
// Disable Gzip when SSL is disabled for security reasons?
|
// Disable Gzip when SSL is disabled for security reasons?
|
||||||
if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
|
if prefix != "/ws" {
|
||||||
|
ae := req.Header.Get("Accept-Encoding")
|
||||||
|
/*if strings.Contains(ae, "br") {
|
||||||
|
h := w.Header()
|
||||||
|
h.Set("Content-Encoding", "br")
|
||||||
|
var ii = brPool.Get()
|
||||||
|
var igzw *brotli.Writer
|
||||||
|
if ii == nil {
|
||||||
|
igzw = brotli.NewWriter(w)
|
||||||
|
} else {
|
||||||
|
igzw = ii.(*brotli.Writer)
|
||||||
|
igzw.Reset(w)
|
||||||
|
}
|
||||||
|
gzw := c.BrResponseWriter{Writer: igzw, ResponseWriter: w}
|
||||||
|
defer func() {
|
||||||
|
//h := w.Header()
|
||||||
|
if h.Get("Content-Encoding") == "br" && h.Get("X-I") == "" {
|
||||||
|
//log.Print("push br close")
|
||||||
|
igzw := gzw.Writer.(*brotli.Writer)
|
||||||
|
igzw.Close()
|
||||||
|
brPool.Put(igzw)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
w = gzw
|
||||||
|
} else */if strings.Contains(ae, "gzip") {
|
||||||
h := w.Header()
|
h := w.Header()
|
||||||
h.Set("Content-Encoding", "gzip")
|
h.Set("Content-Encoding", "gzip")
|
||||||
var ii = gzipPool.Get()
|
var ii = gzipPool.Get()
|
||||||
|
@ -1293,8 +1345,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
}()
|
}()
|
||||||
w = gzw
|
w = gzw
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
skip, ferr = hTbl.VhookSkippable("router_pre_route", w, req, user, prefix)
|
skip, ferr = c.H_router_pre_route_hook(hTbl, w, req, user, prefix)
|
||||||
if skip || ferr != nil {
|
if skip || ferr != nil {
|
||||||
r.handleError(ferr,w,req,user)
|
r.handleError(ferr,w,req,user)
|
||||||
return
|
return
|
||||||
|
@ -1363,8 +1416,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
|
||||||
if ok {
|
if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
w.Header().Del("Content-Encoding")
|
w.Header().Del("Content-Encoding")
|
||||||
}
|
}
|
||||||
|
@ -1929,8 +1982,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
|
||||||
if ok {
|
if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
w.Header().Del("Content-Encoding")
|
w.Header().Del("Content-Encoding")
|
||||||
}
|
}
|
||||||
|
@ -2821,11 +2874,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
co.RouteViewCounter.Bump3(166, cn)
|
co.RouteViewCounter.Bump3(166, cn)
|
||||||
return c.NotFound(w,req,nil)
|
return c.NotFound(w,req,nil)
|
||||||
}
|
}
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||||
if ok {
|
w = bzw.ResponseWriter
|
||||||
|
w.Header().Del("Content-Encoding")
|
||||||
|
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
h := w.Header()
|
w.Header().Del("Content-Encoding")
|
||||||
h.Del("Content-Encoding")
|
|
||||||
}
|
}
|
||||||
req.URL.Path += extraData
|
req.URL.Path += extraData
|
||||||
// TODO: Find a way to propagate errors up from this?
|
// TODO: Find a way to propagate errors up from this?
|
||||||
|
@ -2840,11 +2894,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
co.RouteViewCounter.Bump3(168, cn)
|
co.RouteViewCounter.Bump3(168, cn)
|
||||||
return routes.RobotsTxt(w,req)
|
return routes.RobotsTxt(w,req)
|
||||||
case "favicon.ico":
|
case "favicon.ico":
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||||
if ok {
|
w = bzw.ResponseWriter
|
||||||
|
w.Header().Del("Content-Encoding")
|
||||||
|
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
h := w.Header()
|
w.Header().Del("Content-Encoding")
|
||||||
h.Del("Content-Encoding")
|
|
||||||
}
|
}
|
||||||
req.URL.Path = "/s/favicon.ico"
|
req.URL.Path = "/s/favicon.ico"
|
||||||
routes.StaticFile(w,req)
|
routes.StaticFile(w,req)
|
||||||
|
|
|
@ -230,6 +230,9 @@
|
||||||
"proximic":"Comscore",
|
"proximic":"Comscore",
|
||||||
"megaindex":"MegaIndex",
|
"megaindex":"MegaIndex",
|
||||||
"majestic":"MJ12bot",
|
"majestic":"MJ12bot",
|
||||||
|
"cocolyze":"Cocolyze",
|
||||||
|
"babbar":"Babbar",
|
||||||
|
"surdotly":"Surdotly",
|
||||||
"netcraft":"Netcraft",
|
"netcraft":"Netcraft",
|
||||||
"blexbot":"BLEXBot",
|
"blexbot":"BLEXBot",
|
||||||
"burf":"Burf.co",
|
"burf":"Burf.co",
|
||||||
|
@ -242,6 +245,7 @@
|
||||||
"python":"Python Bot",
|
"python":"Python Bot",
|
||||||
"go":"Go Bot",
|
"go":"Go Bot",
|
||||||
"headlesschrome":"Headless Chrome",
|
"headlesschrome":"Headless Chrome",
|
||||||
|
"awesome_bot":"Awesome Bot",
|
||||||
"suspicious":"Suspicious",
|
"suspicious":"Suspicious",
|
||||||
"unknown":"Unknown",
|
"unknown":"Unknown",
|
||||||
"blank":"Blank",
|
"blank":"Blank",
|
||||||
|
|
|
@ -278,6 +278,9 @@ func main() {
|
||||||
"proximic",
|
"proximic",
|
||||||
"megaindex",
|
"megaindex",
|
||||||
"majestic",
|
"majestic",
|
||||||
|
"cocolyze",
|
||||||
|
"babbar",
|
||||||
|
"surdotly",
|
||||||
"netcraft",
|
"netcraft",
|
||||||
"blexbot",
|
"blexbot",
|
||||||
"burf",
|
"burf",
|
||||||
|
@ -290,6 +293,7 @@ func main() {
|
||||||
"python",
|
"python",
|
||||||
"go",
|
"go",
|
||||||
"headlesschrome",
|
"headlesschrome",
|
||||||
|
"awesome_bot",
|
||||||
}
|
}
|
||||||
|
|
||||||
tmplVars.AllAgentMap = make(map[string]int)
|
tmplVars.AllAgentMap = make(map[string]int)
|
||||||
|
@ -348,6 +352,9 @@ func main() {
|
||||||
"MegaIndex",
|
"MegaIndex",
|
||||||
"MJ12bot",
|
"MJ12bot",
|
||||||
"mj12bot",
|
"mj12bot",
|
||||||
|
"Cocolyzebot",
|
||||||
|
"Barkrowler",
|
||||||
|
"SurdotlyBot",
|
||||||
"NetcraftSurveyAgent",
|
"NetcraftSurveyAgent",
|
||||||
"BLEXBot",
|
"BLEXBot",
|
||||||
"Burf",
|
"Burf",
|
||||||
|
@ -360,6 +367,7 @@ func main() {
|
||||||
"python",
|
"python",
|
||||||
"Go",
|
"Go",
|
||||||
"HeadlessChrome",
|
"HeadlessChrome",
|
||||||
|
"awesome_bot",
|
||||||
}
|
}
|
||||||
|
|
||||||
tmplVars.AllAgentMarks = map[string]string{
|
tmplVars.AllAgentMarks = map[string]string{
|
||||||
|
@ -413,6 +421,9 @@ func main() {
|
||||||
"MegaIndex": "megaindex",
|
"MegaIndex": "megaindex",
|
||||||
"MJ12bot": "majestic",
|
"MJ12bot": "majestic",
|
||||||
"mj12bot": "majestic",
|
"mj12bot": "majestic",
|
||||||
|
"Cocolyzebot": "cocolyze",
|
||||||
|
"Barkrowler": "babbar",
|
||||||
|
"SurdotlyBot": "surdotly",
|
||||||
"NetcraftSurveyAgent": "netcraft",
|
"NetcraftSurveyAgent": "netcraft",
|
||||||
"BLEXBot": "blexbot",
|
"BLEXBot": "blexbot",
|
||||||
"Burf": "burf",
|
"Burf": "burf",
|
||||||
|
@ -425,6 +436,7 @@ func main() {
|
||||||
"python": "python",
|
"python": "python",
|
||||||
"Go": "go",
|
"Go": "go",
|
||||||
"HeadlessChrome": "headlesschrome",
|
"HeadlessChrome": "headlesschrome",
|
||||||
|
"awesome_bot": "awesome_bot",
|
||||||
}
|
}
|
||||||
|
|
||||||
tmplVars.AllAgentMarkIDs = make(map[string]int)
|
tmplVars.AllAgentMarkIDs = make(map[string]int)
|
||||||
|
@ -454,6 +466,8 @@ import (
|
||||||
"github.com/Azareal/Gosora/uutils"
|
"github.com/Azareal/Gosora/uutils"
|
||||||
"github.com/Azareal/Gosora/routes"
|
"github.com/Azareal/Gosora/routes"
|
||||||
"github.com/Azareal/Gosora/routes/panel"
|
"github.com/Azareal/Gosora/routes/panel"
|
||||||
|
|
||||||
|
//"github.com/andybalholm/brotli"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrNoRoute = errors.New("That route doesn't exist.")
|
var ErrNoRoute = errors.New("That route doesn't exist.")
|
||||||
|
@ -628,7 +642,9 @@ func isLocalHost(h string) bool {
|
||||||
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//var brPool = sync.Pool{}
|
||||||
var gzipPool = sync.Pool{}
|
var gzipPool = sync.Pool{}
|
||||||
|
//var uaBufPool = sync.Pool{}
|
||||||
|
|
||||||
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
||||||
// TODO: SetDefaultPath
|
// TODO: SetDefaultPath
|
||||||
|
@ -711,7 +727,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
lp := strings.ToLower(req.URL.Path)
|
lp := strings.ToLower(req.URL.Path)
|
||||||
// TODO: Flag any requests which has a dot with anything but a number after that
|
// TODO: Flag any requests which has a dot with anything but a number after that
|
||||||
// TODO: Use HasSuffix to avoid over-scanning?
|
// TODO: Use HasSuffix to avoid over-scanning?
|
||||||
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") {
|
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
|
||||||
r.SuspiciousRequest(req,"Bad snippet in path")
|
r.SuspiciousRequest(req,"Bad snippet in path")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -724,7 +740,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
|
|
||||||
// TODO: Use the same hook table as downstream
|
// TODO: Use the same hook table as downstream
|
||||||
hTbl := c.GetHookTable()
|
hTbl := c.GetHookTable()
|
||||||
skip, ferr := hTbl.VhookSkippable("router_after_filters", w, req, prefix)
|
skip, ferr := c.H_router_after_filters_hook(hTbl, w, req, prefix)
|
||||||
if skip || ferr != nil {
|
if skip || ferr != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -787,18 +803,22 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// WIP UA Parser
|
// WIP UA Parser
|
||||||
|
//var ii = uaBufPool.Get()
|
||||||
|
var buf []byte
|
||||||
|
//if ii != nil {
|
||||||
|
// buf = ii.([]byte)
|
||||||
|
//}
|
||||||
var items []string
|
var items []string
|
||||||
var buffer []byte
|
|
||||||
var os int
|
var os int
|
||||||
for _, it := range uutils.StringToBytes(ua) {
|
for _, it := range uutils.StringToBytes(ua) {
|
||||||
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
|
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
|
||||||
// TODO: Store an index and slice that instead?
|
// TODO: Store an index and slice that instead?
|
||||||
buffer = append(buffer, it)
|
buf = append(buf, it)
|
||||||
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buffer,[]byte("http")))*/ || it == ',' || it == '/' {
|
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buf,[]byte("http")))*/ || it == ',' || it == '/' {
|
||||||
if len(buffer) != 0 {
|
if len(buf) != 0 {
|
||||||
if len(buffer) > 2 {
|
if len(buf) > 2 {
|
||||||
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
|
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
|
||||||
switch(uutils.BytesToString(buffer)) {
|
switch(uutils.BytesToString(buf)) {
|
||||||
case "Windows":
|
case "Windows":
|
||||||
os = {{.AllOSMap.windows}}
|
os = {{.AllOSMap.windows}}
|
||||||
case "Linux":
|
case "Linux":
|
||||||
|
@ -809,23 +829,24 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
os = {{.AllOSMap.iphone}}
|
os = {{.AllOSMap.iphone}}
|
||||||
case "Android":
|
case "Android":
|
||||||
os = {{.AllOSMap.android}}
|
os = {{.AllOSMap.android}}
|
||||||
case "like","compatible":
|
case "like","compatible","NT","X","KHTML":
|
||||||
// Skip these words
|
// Skip these words
|
||||||
default:
|
default:
|
||||||
items = append(items, string(buffer))
|
items = append(items, string(buf))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
buffer = buffer[:0]
|
buf = buf[:0]
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// TODO: Test this
|
// TODO: Test this
|
||||||
items = items[:0]
|
items = items[:0]
|
||||||
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
||||||
r.requestLogger.Print("UA Buf: ", buffer)
|
r.requestLogger.Print("UA Buf: ", buf)
|
||||||
r.requestLogger.Print("UA Buf String: ", string(buffer))
|
r.requestLogger.Print("UA Buf String: ", string(buf))
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
//uaBufPool.Put(buf)
|
||||||
|
|
||||||
// Iterate over this in reverse as the real UA tends to be on the right side
|
// Iterate over this in reverse as the real UA tends to be on the right side
|
||||||
for i := len(items) - 1; i >= 0; i-- {
|
for i := len(items) - 1; i >= 0; i-- {
|
||||||
|
@ -842,6 +863,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
r.requestLogger.Print("parsed agent: ", agent)
|
r.requestLogger.Print("parsed agent: ", agent)
|
||||||
r.requestLogger.Print("os: ", os)
|
r.requestLogger.Print("os: ", os)
|
||||||
r.requestLogger.Printf("items: %+v\n",items)
|
r.requestLogger.Printf("items: %+v\n",items)
|
||||||
|
/*for _, it := range items {
|
||||||
|
r.requestLogger.Printf("it: %+v\n",string(it))
|
||||||
|
}*/
|
||||||
}
|
}
|
||||||
|
|
||||||
// Special handling
|
// Special handling
|
||||||
|
@ -860,11 +884,11 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
agent = {{.AllAgentMap.internetexplorer}}
|
agent = {{.AllAgentMap.internetexplorer}}
|
||||||
}
|
}
|
||||||
case {{.AllAgentMap.zgrab}}:
|
case {{.AllAgentMap.zgrab}}:
|
||||||
r.SuspiciousRequest(req,"Vulnerability Scanner")
|
r.SuspiciousRequest(req,"Vuln Scanner")
|
||||||
}
|
}
|
||||||
|
|
||||||
if agent == 0 {
|
if agent == 0 {
|
||||||
co.AgentViewCounter.Bump({{.AllAgentMap.unknown}})
|
//co.AgentViewCounter.Bump({{.AllAgentMap.unknown}})
|
||||||
if c.Dev.DebugMode {
|
if c.Dev.DebugMode {
|
||||||
var pre string
|
var pre string
|
||||||
for _, char := range req.UserAgent() {
|
for _, char := range req.UserAgent() {
|
||||||
|
@ -874,10 +898,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
} else {
|
} else {
|
||||||
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
|
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
|
||||||
}
|
}
|
||||||
} else {
|
}// else {
|
||||||
//co.AgentViewCounter.Bump(agentMapEnum[agent])
|
//co.AgentViewCounter.Bump(agentMapEnum[agent])
|
||||||
co.AgentViewCounter.Bump(agent)
|
co.AgentViewCounter.Bump(agent)
|
||||||
}
|
//}
|
||||||
co.OSViewCounter.Bump(os)
|
co.OSViewCounter.Bump(os)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -941,7 +965,31 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
//log.Println("req: ", req)
|
//log.Println("req: ", req)
|
||||||
|
|
||||||
// Disable Gzip when SSL is disabled for security reasons?
|
// Disable Gzip when SSL is disabled for security reasons?
|
||||||
if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
|
if prefix != "/ws" {
|
||||||
|
ae := req.Header.Get("Accept-Encoding")
|
||||||
|
/*if strings.Contains(ae, "br") {
|
||||||
|
h := w.Header()
|
||||||
|
h.Set("Content-Encoding", "br")
|
||||||
|
var ii = brPool.Get()
|
||||||
|
var igzw *brotli.Writer
|
||||||
|
if ii == nil {
|
||||||
|
igzw = brotli.NewWriter(w)
|
||||||
|
} else {
|
||||||
|
igzw = ii.(*brotli.Writer)
|
||||||
|
igzw.Reset(w)
|
||||||
|
}
|
||||||
|
gzw := c.BrResponseWriter{Writer: igzw, ResponseWriter: w}
|
||||||
|
defer func() {
|
||||||
|
//h := w.Header()
|
||||||
|
if h.Get("Content-Encoding") == "br" && h.Get("X-I") == "" {
|
||||||
|
//log.Print("push br close")
|
||||||
|
igzw := gzw.Writer.(*brotli.Writer)
|
||||||
|
igzw.Close()
|
||||||
|
brPool.Put(igzw)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
w = gzw
|
||||||
|
} else */if strings.Contains(ae, "gzip") {
|
||||||
h := w.Header()
|
h := w.Header()
|
||||||
h.Set("Content-Encoding", "gzip")
|
h.Set("Content-Encoding", "gzip")
|
||||||
var ii = gzipPool.Get()
|
var ii = gzipPool.Get()
|
||||||
|
@ -964,8 +1012,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
}()
|
}()
|
||||||
w = gzw
|
w = gzw
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
skip, ferr = hTbl.VhookSkippable("router_pre_route", w, req, user, prefix)
|
skip, ferr = c.H_router_pre_route_hook(hTbl, w, req, user, prefix)
|
||||||
if skip || ferr != nil {
|
if skip || ferr != nil {
|
||||||
r.handleError(ferr,w,req,user)
|
r.handleError(ferr,w,req,user)
|
||||||
return
|
return
|
||||||
|
@ -1004,11 +1053,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn)
|
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn)
|
||||||
return c.NotFound(w,req,nil)
|
return c.NotFound(w,req,nil)
|
||||||
}
|
}
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||||
if ok {
|
w = bzw.ResponseWriter
|
||||||
|
w.Header().Del("Content-Encoding")
|
||||||
|
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
h := w.Header()
|
w.Header().Del("Content-Encoding")
|
||||||
h.Del("Content-Encoding")
|
|
||||||
}
|
}
|
||||||
req.URL.Path += extraData
|
req.URL.Path += extraData
|
||||||
// TODO: Find a way to propagate errors up from this?
|
// TODO: Find a way to propagate errors up from this?
|
||||||
|
@ -1023,11 +1073,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
||||||
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.RobotsTxt"}}, cn)
|
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.RobotsTxt"}}, cn)
|
||||||
return routes.RobotsTxt(w,req)
|
return routes.RobotsTxt(w,req)
|
||||||
case "favicon.ico":
|
case "favicon.ico":
|
||||||
gzw, ok := w.(c.GzipResponseWriter)
|
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||||
if ok {
|
w = bzw.ResponseWriter
|
||||||
|
w.Header().Del("Content-Encoding")
|
||||||
|
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||||
w = gzw.ResponseWriter
|
w = gzw.ResponseWriter
|
||||||
h := w.Header()
|
w.Header().Del("Content-Encoding")
|
||||||
h.Del("Content-Encoding")
|
|
||||||
}
|
}
|
||||||
req.URL.Path = "/s/favicon.ico"
|
req.URL.Path = "/s/favicon.ico"
|
||||||
routes.StaticFile(w,req)
|
routes.StaticFile(w,req)
|
||||||
|
|
Loading…
Reference in New Issue