a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add cocolyze, babbar, surdotly, awesome_bot user agents 

reduce matching in the router slightly
use hookgen for router_after_filters
use hookgen for router_pre_route
shorten ua buffer variable name
skip more symbols in initial ua loop
shorten vulnscan string
avoid unneccessary fork in ua logic
This commit is contained in:
Azareal 2020-05-27 07:08:13 +10:00
parent 6be9a25e89
commit d0bc74f58f
3 changed files with 245 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
gen_router.go
View File

@ -20,6 +20,8 @@ import (
"github.com/Azareal/Gosora/uutils" "github.com/Azareal/Gosora/uutils"
"github.com/Azareal/Gosora/routes" "github.com/Azareal/Gosora/routes"
"github.com/Azareal/Gosora/routes/panel" "github.com/Azareal/Gosora/routes/panel"
//"github.com/andybalholm/brotli"
) )
var ErrNoRoute = errors.New("That route doesn't exist.") var ErrNoRoute = errors.New("That route doesn't exist.")
@ -618,18 +620,22 @@ var agentMapEnum = map[string]int{
"proximic": 44, "proximic": 44,
"megaindex": 45, "megaindex": 45,
"majestic": 46, "majestic": 46,
"netcraft": 47, "cocolyze": 47,
"blexbot": 48, "babbar": 48,
"burf": 49, "surdotly": 49,
"aspiegel": 50, "netcraft": 50,
"mail_ru": 51, "blexbot": 51,
"ccbot": 52, "burf": 52,
"zgrab": 53, "aspiegel": 53,
"cloudsystemnetworks": 54, "mail_ru": 54,
"curl": 55, "ccbot": 55,
"python": 56, "zgrab": 56,
"go": 57, "cloudsystemnetworks": 57,
"headlesschrome": 58, "curl": 58,
"python": 59,
"go": 60,
"headlesschrome": 61,
"awesome_bot": 62,
} }
var reverseAgentMapEnum = map[int]string{ var reverseAgentMapEnum = map[int]string{
0: "unknown", 0: "unknown",
@ -679,18 +685,22 @@ var reverseAgentMapEnum = map[int]string{
44: "proximic", 44: "proximic",
45: "megaindex", 45: "megaindex",
46: "majestic", 46: "majestic",
47: "netcraft", 47: "cocolyze",
48: "blexbot", 48: "babbar",
49: "burf", 49: "surdotly",
50: "aspiegel", 50: "netcraft",
51: "mail_ru", 51: "blexbot",
52: "ccbot", 52: "burf",
53: "zgrab", 53: "aspiegel",
54: "cloudsystemnetworks", 54: "mail_ru",
55: "curl", 55: "ccbot",
56: "python", 56: "zgrab",
57: "go", 57: "cloudsystemnetworks",
58: "headlesschrome", 58: "curl",
59: "python",
60: "go",
61: "headlesschrome",
62: "awesome_bot",
} }
var markToAgent = map[string]string{ var markToAgent = map[string]string{
"OPR": "opera", "OPR": "opera",
@ -741,6 +751,9 @@ var markToAgent = map[string]string{
"MegaIndex": "megaindex", "MegaIndex": "megaindex",
"MJ12bot": "majestic", "MJ12bot": "majestic",
"mj12bot": "majestic", "mj12bot": "majestic",
"Cocolyzebot": "cocolyze",
"Barkrowler": "babbar",
"SurdotlyBot": "surdotly",
"NetcraftSurveyAgent": "netcraft", "NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot", "BLEXBot": "blexbot",
"Burf": "burf", "Burf": "burf",
@ -753,6 +766,7 @@ var markToAgent = map[string]string{
"python": "python", "python": "python",
"Go": "go", "Go": "go",
"HeadlessChrome": "headlesschrome", "HeadlessChrome": "headlesschrome",
"awesome_bot": "awesome_bot",
} }
var markToID = map[string]int{ var markToID = map[string]int{
"OPR": 3, "OPR": 3,
@ -803,18 +817,22 @@ var markToID = map[string]int{
"MegaIndex": 45, "MegaIndex": 45,
"MJ12bot": 46, "MJ12bot": 46,
"mj12bot": 46, "mj12bot": 46,
"NetcraftSurveyAgent": 47, "Cocolyzebot": 47,
"BLEXBot": 48, "Barkrowler": 48,
"Burf": 49, "SurdotlyBot": 49,
"AspiegelBot": 50, "NetcraftSurveyAgent": 50,
"RU_Bot": 51, "BLEXBot": 51,
"CCBot": 52, "Burf": 52,
"zgrab": 53, "AspiegelBot": 53,
"Nimbostratus": 54, "RU_Bot": 54,
"curl": 55, "CCBot": 55,
"python": 56, "zgrab": 56,
"Go": 57, "Nimbostratus": 57,
"HeadlessChrome": 58, "curl": 58,
"python": 59,
"Go": 60,
"HeadlessChrome": 61,
"awesome_bot": 62,
} }
/*var agentRank = map[string]int{ /*var agentRank = map[string]int{
"opera":9, "opera":9,
@ -957,7 +975,9 @@ func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1" return h=="localhost" || h=="127.0.0.1" || h=="::1"
} }
//var brPool = sync.Pool{}
var gzipPool = sync.Pool{} var gzipPool = sync.Pool{}
//var uaBufPool = sync.Pool{}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
// TODO: SetDefaultPath // TODO: SetDefaultPath
@ -1040,7 +1060,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that // TODO: Flag any requests which has a dot with anything but a number after that
// TODO: Use HasSuffix to avoid over-scanning? // TODO: Use HasSuffix to avoid over-scanning?
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") { if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
r.SuspiciousRequest(req,"Bad snippet in path") r.SuspiciousRequest(req,"Bad snippet in path")
} }
@ -1053,7 +1073,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Use the same hook table as downstream // TODO: Use the same hook table as downstream
hTbl := c.GetHookTable() hTbl := c.GetHookTable()
skip, ferr := hTbl.VhookSkippable("router_after_filters", w, req, prefix) skip, ferr := c.H_router_after_filters_hook(hTbl, w, req, prefix)
if skip || ferr != nil { if skip || ferr != nil {
return return
} }
@ -1116,18 +1136,22 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} else { } else {
// WIP UA Parser // WIP UA Parser
//var ii = uaBufPool.Get()
var buf []byte
//if ii != nil {
// buf = ii.([]byte)
//}
var items []string var items []string
var buffer []byte
var os int var os int
for _, it := range uutils.StringToBytes(ua) { for _, it := range uutils.StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' { if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
// TODO: Store an index and slice that instead? // TODO: Store an index and slice that instead?
buffer = append(buffer, it) buf = append(buf, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buffer,[]byte("http")))*/ || it == ',' || it == '/' { } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buf,[]byte("http")))*/ || it == ',' || it == '/' {
if len(buffer) != 0 { if len(buf) != 0 {
if len(buffer) > 2 { if len(buf) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
switch(uutils.BytesToString(buffer)) { switch(uutils.BytesToString(buf)) {
case "Windows": case "Windows":
os = 1 os = 1
case "Linux": case "Linux":
@ -1138,23 +1162,24 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
os = 5 os = 5
case "Android": case "Android":
os = 4 os = 4
case "like","compatible": case "like","compatible","NT","X","KHTML":
// Skip these words // Skip these words
default: default:
items = append(items, string(buffer)) items = append(items, string(buf))
} }
} }
buffer = buffer[:0] buf = buf[:0]
} }
} else { } else {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buf: ", buffer) r.requestLogger.Print("UA Buf: ", buf)
r.requestLogger.Print("UA Buf String: ", string(buffer)) r.requestLogger.Print("UA Buf String: ", string(buf))
break break
} }
} }
//uaBufPool.Put(buf)
// Iterate over this in reverse as the real UA tends to be on the right side // Iterate over this in reverse as the real UA tends to be on the right side
for i := len(items) - 1; i >= 0; i-- { for i := len(items) - 1; i >= 0; i-- {
@ -1171,6 +1196,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.requestLogger.Print("parsed agent: ", agent) r.requestLogger.Print("parsed agent: ", agent)
r.requestLogger.Print("os: ", os) r.requestLogger.Print("os: ", os)
r.requestLogger.Printf("items: %+v\n",items) r.requestLogger.Printf("items: %+v\n",items)
/*for _, it := range items {
r.requestLogger.Printf("it: %+v\n",string(it))
}*/
} }
// Special handling // Special handling
@ -1188,12 +1216,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if strings.Contains(ua,"rv:11") { if strings.Contains(ua,"rv:11") {
agent = 6 agent = 6
} }
case 53: case 56:
r.SuspiciousRequest(req,"Vulnerability Scanner") r.SuspiciousRequest(req,"Vuln Scanner")
} }
if agent == 0 { if agent == 0 {
co.AgentViewCounter.Bump(0) //co.AgentViewCounter.Bump(0)
if c.Dev.DebugMode { if c.Dev.DebugMode {
var pre string var pre string
for _, char := range req.UserAgent() { for _, char := range req.UserAgent() {
@ -1203,10 +1231,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua)) r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
} }
} else { }// else {
//co.AgentViewCounter.Bump(agentMapEnum[agent]) //co.AgentViewCounter.Bump(agentMapEnum[agent])
co.AgentViewCounter.Bump(agent) co.AgentViewCounter.Bump(agent)
} //}
co.OSViewCounter.Bump(os) co.OSViewCounter.Bump(os)
} }
@ -1270,31 +1298,56 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//log.Println("req: ", req) //log.Println("req: ", req)
// Disable Gzip when SSL is disabled for security reasons? // Disable Gzip when SSL is disabled for security reasons?
if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { if prefix != "/ws" {
h := w.Header() ae := req.Header.Get("Accept-Encoding")
h.Set("Content-Encoding", "gzip") /*if strings.Contains(ae, "br") {
var ii = gzipPool.Get() h := w.Header()
var igzw *gzip.Writer h.Set("Content-Encoding", "br")
if ii == nil { var ii = brPool.Get()
igzw = gzip.NewWriter(w) var igzw *brotli.Writer
} else { if ii == nil {
igzw = ii.(*gzip.Writer) igzw = brotli.NewWriter(w)
igzw.Reset(w) } else {
} igzw = ii.(*brotli.Writer)
gzw := c.GzipResponseWriter{Writer: igzw, ResponseWriter: w} igzw.Reset(w)
defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "gzip" && h.Get("X-I") == "" {
//log.Print("push gzip close")
igzw := gzw.Writer.(*gzip.Writer)
igzw.Close()
gzipPool.Put(igzw)
} }
}() gzw := c.BrResponseWriter{Writer: igzw, ResponseWriter: w}
w = gzw defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "br" && h.Get("X-I") == "" {
//log.Print("push br close")
igzw := gzw.Writer.(*brotli.Writer)
igzw.Close()
brPool.Put(igzw)
}
}()
w = gzw
} else */if strings.Contains(ae, "gzip") {
h := w.Header()
h.Set("Content-Encoding", "gzip")
var ii = gzipPool.Get()
var igzw *gzip.Writer
if ii == nil {
igzw = gzip.NewWriter(w)
} else {
igzw = ii.(*gzip.Writer)
igzw.Reset(w)
}
gzw := c.GzipResponseWriter{Writer: igzw, ResponseWriter: w}
defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "gzip" && h.Get("X-I") == "" {
//log.Print("push gzip close")
igzw := gzw.Writer.(*gzip.Writer)
igzw.Close()
gzipPool.Put(igzw)
}
}()
w = gzw
}
} }
skip, ferr = hTbl.VhookSkippable("router_pre_route", w, req, user, prefix) skip, ferr = c.H_router_pre_route_hook(hTbl, w, req, user, prefix)
if skip || ferr != nil { if skip || ferr != nil {
r.handleError(ferr,w,req,user) r.handleError(ferr,w,req,user)
return return
@ -1363,8 +1416,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
return err return err
} }
gzw, ok := w.(c.GzipResponseWriter)
if ok { if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
w.Header().Del("Content-Encoding") w.Header().Del("Content-Encoding")
} }
@ -1929,8 +1982,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
return err return err
} }
gzw, ok := w.(c.GzipResponseWriter)
if ok { if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
w.Header().Del("Content-Encoding") w.Header().Del("Content-Encoding")
} }
@ -2821,11 +2874,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3(166, cn) co.RouteViewCounter.Bump3(166, cn)
return c.NotFound(w,req,nil) return c.NotFound(w,req,nil)
} }
gzw, ok := w.(c.GzipResponseWriter) /*if bzw, ok := w.(c.BrResponseWriter); ok {
if ok { w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
h := w.Header() w.Header().Del("Content-Encoding")
h.Del("Content-Encoding")
} }
req.URL.Path += extraData req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this? // TODO: Find a way to propagate errors up from this?
@ -2840,11 +2894,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3(168, cn) co.RouteViewCounter.Bump3(168, cn)
return routes.RobotsTxt(w,req) return routes.RobotsTxt(w,req)
case "favicon.ico": case "favicon.ico":
gzw, ok := w.(c.GzipResponseWriter) /*if bzw, ok := w.(c.BrResponseWriter); ok {
if ok { w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
h := w.Header() w.Header().Del("Content-Encoding")
h.Del("Content-Encoding")
} }
req.URL.Path = "/s/favicon.ico" req.URL.Path = "/s/favicon.ico"
routes.StaticFile(w,req) routes.StaticFile(w,req)

4
langs/english.json
View File

@ -230,6 +230,9 @@
"proximic":"Comscore", "proximic":"Comscore",
"megaindex":"MegaIndex", "megaindex":"MegaIndex",
"majestic":"MJ12bot", "majestic":"MJ12bot",
"cocolyze":"Cocolyze",
"babbar":"Babbar",
"surdotly":"Surdotly",
"netcraft":"Netcraft", "netcraft":"Netcraft",
"blexbot":"BLEXBot", "blexbot":"BLEXBot",
"burf":"Burf.co", "burf":"Burf.co",
@ -242,6 +245,7 @@
"python":"Python Bot", "python":"Python Bot",
"go":"Go Bot", "go":"Go Bot",
"headlesschrome":"Headless Chrome", "headlesschrome":"Headless Chrome",
"awesome_bot":"Awesome Bot",
"suspicious":"Suspicious", "suspicious":"Suspicious",
"unknown":"Unknown", "unknown":"Unknown",
"blank":"Blank", "blank":"Blank",

145
router_gen/main.go
View File

@ -278,6 +278,9 @@ func main() {
"proximic", "proximic",
"megaindex", "megaindex",
"majestic", "majestic",
"cocolyze",
"babbar",
"surdotly",
"netcraft", "netcraft",
"blexbot", "blexbot",
"burf", "burf",
@ -290,6 +293,7 @@ func main() {
"python", "python",
"go", "go",
"headlesschrome", "headlesschrome",
"awesome_bot",
} }
tmplVars.AllAgentMap = make(map[string]int) tmplVars.AllAgentMap = make(map[string]int)
@ -348,6 +352,9 @@ func main() {
"MegaIndex", "MegaIndex",
"MJ12bot", "MJ12bot",
"mj12bot", "mj12bot",
"Cocolyzebot",
"Barkrowler",
"SurdotlyBot",
"NetcraftSurveyAgent", "NetcraftSurveyAgent",
"BLEXBot", "BLEXBot",
"Burf", "Burf",
@ -360,6 +367,7 @@ func main() {
"python", "python",
"Go", "Go",
"HeadlessChrome", "HeadlessChrome",
"awesome_bot",
} }
tmplVars.AllAgentMarks = map[string]string{ tmplVars.AllAgentMarks = map[string]string{
@ -413,6 +421,9 @@ func main() {
"MegaIndex": "megaindex", "MegaIndex": "megaindex",
"MJ12bot": "majestic", "MJ12bot": "majestic",
"mj12bot": "majestic", "mj12bot": "majestic",
"Cocolyzebot": "cocolyze",
"Barkrowler": "babbar",
"SurdotlyBot": "surdotly",
"NetcraftSurveyAgent": "netcraft", "NetcraftSurveyAgent": "netcraft",
"BLEXBot": "blexbot", "BLEXBot": "blexbot",
"Burf": "burf", "Burf": "burf",
@ -425,6 +436,7 @@ func main() {
"python": "python", "python": "python",
"Go": "go", "Go": "go",
"HeadlessChrome": "headlesschrome", "HeadlessChrome": "headlesschrome",
"awesome_bot": "awesome_bot",
} }
tmplVars.AllAgentMarkIDs = make(map[string]int) tmplVars.AllAgentMarkIDs = make(map[string]int)
@ -454,6 +466,8 @@ import (
"github.com/Azareal/Gosora/uutils" "github.com/Azareal/Gosora/uutils"
"github.com/Azareal/Gosora/routes" "github.com/Azareal/Gosora/routes"
"github.com/Azareal/Gosora/routes/panel" "github.com/Azareal/Gosora/routes/panel"
//"github.com/andybalholm/brotli"
) )
var ErrNoRoute = errors.New("That route doesn't exist.") var ErrNoRoute = errors.New("That route doesn't exist.")
@ -628,7 +642,9 @@ func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1" return h=="localhost" || h=="127.0.0.1" || h=="::1"
} }
//var brPool = sync.Pool{}
var gzipPool = sync.Pool{} var gzipPool = sync.Pool{}
//var uaBufPool = sync.Pool{}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
// TODO: SetDefaultPath // TODO: SetDefaultPath
@ -711,7 +727,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that // TODO: Flag any requests which has a dot with anything but a number after that
// TODO: Use HasSuffix to avoid over-scanning? // TODO: Use HasSuffix to avoid over-scanning?
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") { if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
r.SuspiciousRequest(req,"Bad snippet in path") r.SuspiciousRequest(req,"Bad snippet in path")
} }
@ -724,7 +740,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Use the same hook table as downstream // TODO: Use the same hook table as downstream
hTbl := c.GetHookTable() hTbl := c.GetHookTable()
skip, ferr := hTbl.VhookSkippable("router_after_filters", w, req, prefix) skip, ferr := c.H_router_after_filters_hook(hTbl, w, req, prefix)
if skip || ferr != nil { if skip || ferr != nil {
return return
} }
@ -787,18 +803,22 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} else { } else {
// WIP UA Parser // WIP UA Parser
//var ii = uaBufPool.Get()
var buf []byte
//if ii != nil {
// buf = ii.([]byte)
//}
var items []string var items []string
var buffer []byte
var os int var os int
for _, it := range uutils.StringToBytes(ua) { for _, it := range uutils.StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' { if (it > 64 && it < 91) || (it > 96 && it < 123) || it == '_' {
// TODO: Store an index and slice that instead? // TODO: Store an index and slice that instead?
buffer = append(buffer, it) buf = append(buf, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buffer,[]byte("http")))*/ || it == ',' || it == '/' { } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buf,[]byte("http")))*/ || it == ',' || it == '/' {
if len(buffer) != 0 { if len(buf) != 0 {
if len(buffer) > 2 { if len(buf) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
switch(uutils.BytesToString(buffer)) { switch(uutils.BytesToString(buf)) {
case "Windows": case "Windows":
os = {{.AllOSMap.windows}} os = {{.AllOSMap.windows}}
case "Linux": case "Linux":
@ -809,23 +829,24 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
os = {{.AllOSMap.iphone}} os = {{.AllOSMap.iphone}}
case "Android": case "Android":
os = {{.AllOSMap.android}} os = {{.AllOSMap.android}}
case "like","compatible": case "like","compatible","NT","X","KHTML":
// Skip these words // Skip these words
default: default:
items = append(items, string(buffer)) items = append(items, string(buf))
} }
} }
buffer = buffer[:0] buf = buf[:0]
} }
} else { } else {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buf: ", buffer) r.requestLogger.Print("UA Buf: ", buf)
r.requestLogger.Print("UA Buf String: ", string(buffer)) r.requestLogger.Print("UA Buf String: ", string(buf))
break break
} }
} }
//uaBufPool.Put(buf)
// Iterate over this in reverse as the real UA tends to be on the right side // Iterate over this in reverse as the real UA tends to be on the right side
for i := len(items) - 1; i >= 0; i-- { for i := len(items) - 1; i >= 0; i-- {
@ -842,6 +863,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.requestLogger.Print("parsed agent: ", agent) r.requestLogger.Print("parsed agent: ", agent)
r.requestLogger.Print("os: ", os) r.requestLogger.Print("os: ", os)
r.requestLogger.Printf("items: %+v\n",items) r.requestLogger.Printf("items: %+v\n",items)
/*for _, it := range items {
r.requestLogger.Printf("it: %+v\n",string(it))
}*/
} }
// Special handling // Special handling
@ -860,11 +884,11 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
agent = {{.AllAgentMap.internetexplorer}} agent = {{.AllAgentMap.internetexplorer}}
} }
case {{.AllAgentMap.zgrab}}: case {{.AllAgentMap.zgrab}}:
r.SuspiciousRequest(req,"Vulnerability Scanner") r.SuspiciousRequest(req,"Vuln Scanner")
} }
if agent == 0 { if agent == 0 {
co.AgentViewCounter.Bump({{.AllAgentMap.unknown}}) //co.AgentViewCounter.Bump({{.AllAgentMap.unknown}})
if c.Dev.DebugMode { if c.Dev.DebugMode {
var pre string var pre string
for _, char := range req.UserAgent() { for _, char := range req.UserAgent() {
@ -874,10 +898,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua)) r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(ua))
} }
} else { }// else {
//co.AgentViewCounter.Bump(agentMapEnum[agent]) //co.AgentViewCounter.Bump(agentMapEnum[agent])
co.AgentViewCounter.Bump(agent) co.AgentViewCounter.Bump(agent)
} //}
co.OSViewCounter.Bump(os) co.OSViewCounter.Bump(os)
} }
@ -941,31 +965,56 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//log.Println("req: ", req) //log.Println("req: ", req)
// Disable Gzip when SSL is disabled for security reasons? // Disable Gzip when SSL is disabled for security reasons?
if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { if prefix != "/ws" {
h := w.Header() ae := req.Header.Get("Accept-Encoding")
h.Set("Content-Encoding", "gzip") /*if strings.Contains(ae, "br") {
var ii = gzipPool.Get() h := w.Header()
var igzw *gzip.Writer h.Set("Content-Encoding", "br")
if ii == nil { var ii = brPool.Get()
igzw = gzip.NewWriter(w) var igzw *brotli.Writer
} else { if ii == nil {
igzw = ii.(*gzip.Writer) igzw = brotli.NewWriter(w)
igzw.Reset(w) } else {
} igzw = ii.(*brotli.Writer)
gzw := c.GzipResponseWriter{Writer: igzw, ResponseWriter: w} igzw.Reset(w)
defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "gzip" && h.Get("X-I") == "" {
//log.Print("push gzip close")
igzw := gzw.Writer.(*gzip.Writer)
igzw.Close()
gzipPool.Put(igzw)
} }
}() gzw := c.BrResponseWriter{Writer: igzw, ResponseWriter: w}
w = gzw defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "br" && h.Get("X-I") == "" {
//log.Print("push br close")
igzw := gzw.Writer.(*brotli.Writer)
igzw.Close()
brPool.Put(igzw)
}
}()
w = gzw
} else */if strings.Contains(ae, "gzip") {
h := w.Header()
h.Set("Content-Encoding", "gzip")
var ii = gzipPool.Get()
var igzw *gzip.Writer
if ii == nil {
igzw = gzip.NewWriter(w)
} else {
igzw = ii.(*gzip.Writer)
igzw.Reset(w)
}
gzw := c.GzipResponseWriter{Writer: igzw, ResponseWriter: w}
defer func() {
//h := w.Header()
if h.Get("Content-Encoding") == "gzip" && h.Get("X-I") == "" {
//log.Print("push gzip close")
igzw := gzw.Writer.(*gzip.Writer)
igzw.Close()
gzipPool.Put(igzw)
}
}()
w = gzw
}
} }
skip, ferr = hTbl.VhookSkippable("router_pre_route", w, req, user, prefix) skip, ferr = c.H_router_pre_route_hook(hTbl, w, req, user, prefix)
if skip || ferr != nil { if skip || ferr != nil {
r.handleError(ferr,w,req,user) r.handleError(ferr,w,req,user)
return return
@ -1004,11 +1053,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn)
return c.NotFound(w,req,nil) return c.NotFound(w,req,nil)
} }
gzw, ok := w.(c.GzipResponseWriter) /*if bzw, ok := w.(c.BrResponseWriter); ok {
if ok { w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
h := w.Header() w.Header().Del("Content-Encoding")
h.Del("Content-Encoding")
} }
req.URL.Path += extraData req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this? // TODO: Find a way to propagate errors up from this?
@ -1023,11 +1073,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.RobotsTxt"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.RobotsTxt"}}, cn)
return routes.RobotsTxt(w,req) return routes.RobotsTxt(w,req)
case "favicon.ico": case "favicon.ico":
gzw, ok := w.(c.GzipResponseWriter) /*if bzw, ok := w.(c.BrResponseWriter); ok {
if ok { w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter w = gzw.ResponseWriter
h := w.Header() w.Header().Del("Content-Encoding")
h.Del("Content-Encoding")
} }
req.URL.Path = "/s/favicon.ico" req.URL.Path = "/s/favicon.ico"
routes.StaticFile(w,req) routes.StaticFile(w,req)