From 20a35fc21101e95b6288e3e44312745c0074f3ab Mon Sep 17 00:00:00 2001 From: Azareal Date: Tue, 23 Mar 2021 19:57:50 +1000 Subject: [PATCH] rotate the suspicious req logs reduce the number of bytes used by each req dump and don't log raw query when it is blank reduce the number of double suspicious req logs for the same req reduce the number of strings contains scans in the router split the constant parts of gen_router.go into router.go reduce boilerplate in a few spots --- gen_router.go | 215 +++------------------------------- router.go | 286 +++++++++++++++++++++++++++++++++++++++++++++ router_gen/main.go | 235 +++++-------------------------------- 3 files changed, 333 insertions(+), 403 deletions(-) diff --git a/gen_router.go b/gen_router.go index 9313e2d3..4557e0ff 100644 --- a/gen_router.go +++ b/gen_router.go @@ -3,15 +3,12 @@ package main import ( - "log" "strings" //"bytes" "strconv" "compress/gzip" - "sync" "sync/atomic" "errors" - "os" "net/http" "time" @@ -910,24 +907,6 @@ func init() { } } -type WriterIntercept struct { - http.ResponseWriter -} - -func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { - return &WriterIntercept{w} -} - -var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) -func (wi *WriterIntercept) WriteHeader(code int) { - if code == 200 { - h := wi.ResponseWriter.Header() - h.Set("Cache-Control", wiMaxAge) - h.Set("Vary", "Accept-Encoding") - } - wi.ResponseWriter.WriteHeader(code) -} - // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS type HTTPSRedirect struct {} @@ -938,116 +917,6 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Redirect(w, req, dest, http.StatusTemporaryRedirect) } -type GenRouter struct { - UploadHandler func(http.ResponseWriter, *http.Request) - extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError - requestLogger *log.Logger - suspReqLogger *log.Logger - - sync.RWMutex -} - -func NewGenRouter(uploads http.Handler) (*GenRouter, error) { - f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, r *http.Request) { - writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) - }, - extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), - requestLogger: log.New(f, "", log.LstdFlags), - suspReqLogger: log.New(f2, "", log.LstdFlags), - }, nil -} - -func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { - if err.Handled() { - return - } - if err.Type() == "system" { - c.InternalErrorJSQ(err, w, req, err.JSON()) - return - } - c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) -} - -func (r *GenRouter) Handle(_ string, _ http.Handler) { -} - -func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { - r.Lock() - defer r.Unlock() - r.extraRoutes[pattern] = h -} - -func (r *GenRouter) RemoveFunc(pattern string) error { - r.Lock() - defer r.Unlock() - _, ok := r.extraRoutes[pattern] - if !ok { - return ErrNoRoute - } - delete(r.extraRoutes, pattern) - return nil -} - -func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) { - var sb strings.Builder - r.ddumpRequest(req,pre,log,&sb) -} - -// TODO: Some of these sanitisations may be redundant -var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3 -var dumpReqLen2 = len("\nHead : ") + 2 -func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) { - nfield := func(label, val string) { - sb.WriteString(label) - sb.WriteString(val) - } - field := func(label, val string) { - nfield(label,c.SanitiseSingleLine(val)) - } - ua := req.UserAgent() - sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) - sb.WriteString(pre) - field("\nUA: ",ua) - field("\nMethod: ",req.Method) - for key, value := range req.Header { - // Avoid logging this for security reasons - if key == "Cookie" { - continue - } - for _, vvalue := range value { - sb.WriteString("\nHead ") - sb.WriteString(c.SanitiseSingleLine(key)) - sb.WriteString(": ") - sb.WriteString(c.SanitiseSingleLine(vvalue)) - } - } - field("\nHost: ",req.Host) - field("\nURL.Path: ",req.URL.Path) - field("\nURL.RawQuery: ",req.URL.RawQuery) - ref := req.Referer() - if ref != "" { - field("\nRef: ",req.Referer()) - } - nfield("\nIP: ",req.RemoteAddr) - sb.WriteString("\n") - - log.Print(sb.String()) -} - -func (r *GenRouter) DumpRequest(req *http.Request, pre string) { - r.dumpRequest(req,pre,r.requestLogger) -} - func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { var sb strings.Builder if pre != "" { @@ -1055,32 +924,10 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { } else { pre = "Suspicious Request" } - r.ddumpRequest(req,pre,r.suspReqLogger,&sb) + r.ddumpRequest(req,pre,r.suspLog,&sb) co.AgentViewCounter.Bump(43) } -func (r *GenRouter) unknownUA(req *http.Request) { - if c.Dev.DebugMode { - var presb strings.Builder - presb.WriteString("Unknown UA: ") - for _, ch := range req.UserAgent() { - presb.WriteString(strconv.Itoa(int(ch))) - presb.WriteRune(' ') - } - r.ddumpRequest(req, "", r.requestLogger, &presb) - } else { - r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) - } -} - -func isLocalHost(h string) bool { - return h=="localhost" || h=="127.0.0.1" || h=="::1" -} - -//var brPool = sync.Pool{} -var gzipPool = sync.Pool{} -//var uaBufPool = sync.Pool{} - // TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: SetDefaultPath // TODO: GetDefaultPath @@ -1126,11 +973,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { // TODO: Abstract the redirect logic? w.Header().Set("Connection", "close") - var s string + var s, p string if c.Config.SslSchema { s = "s" } - var p string if c.Site.PortInt != 80 && c.Site.PortInt != 443 { p = ":"+c.Site.Port } @@ -1148,23 +994,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { malformedRequest(3) return } - if c.Dev.FullReqLog { - r.DumpRequest(req,"") - } - - // TODO: Cover more suspicious strings and at a lower layer than this - for _, ch := range req.URL.Path { //char - if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { - r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path") - break - } - } - lp := strings.ToLower(req.URL.Path) - // TODO: Flag any requests which has a dot with anything but a number after that - // TODO: Use HasSuffix to avoid over-scanning? - if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action - r.SuspiciousRequest(req,"Bad snippet in path") - } + r.suspScan(req) // Indirect the default route onto a different one if req.URL.Path == "/" { @@ -1213,7 +1043,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } if c.Dev.SuperDebug { - r.requestLogger.Print("before PreRoute") + r.reqLogger.Print("before PreRoute") } /*if c.Dev.QuicPort != 0 { @@ -1276,8 +1106,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Test this items = items[:0] r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") - r.requestLogger.Print("UA Buf: ", buf) - r.requestLogger.Print("UA Buf String: ", string(buf)) + r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf)) break } } @@ -1295,11 +1124,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if c.Dev.SuperDebug { - r.requestLogger.Print("parsed agent: ", agent) - r.requestLogger.Print("os: ", os) - r.requestLogger.Printf("items: %+v\n",items) + r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os) + r.reqLogger.Printf("items: %+v\n",items) /*for _, it := range items { - r.requestLogger.Printf("it: %+v\n",string(it)) + r.reqLogger.Printf("it: %+v\n",string(it)) }*/ } @@ -1390,7 +1218,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { user := &ucpy user.LastAgent = agent if c.Dev.SuperDebug { - r.requestLogger.Print( + r.reqLogger.Print( "after PreRoute\n" + "routeMapEnum: ", routeMapEnum) } @@ -2767,12 +2595,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3(150, cn) } case "/profile": - err = c.NoSessionMismatch(w,req,user) + err = c.MemberOnly(w,req,user) if err != nil { return err } - err = c.MemberOnly(w,req,user) + err = c.NoSessionMismatch(w,req,user) if err != nil { return err } @@ -2926,8 +2754,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * case "favicon.ico": w = r.responseWriter(w) req.URL.Path = "/s/favicon.ico" - routes.StaticFile(w,req) co.RouteViewCounter.Bump3(173, cn) + routes.StaticFile(w,req) return nil case "opensearch.xml": co.RouteViewCounter.Bump3(172, cn) @@ -2953,7 +2781,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3(174, cn) lp := strings.ToLower(req.URL.Path) - if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { + if strings.Contains(lp,"w") { + if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { + r.SuspiciousRequest(req,"Bad Route") + return c.MicroNotFound(w,req) + } + } + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } @@ -2968,14 +2802,3 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } return err } - -func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { - /*if bzw, ok := w.(c.BrResponseWriter); ok { - w = bzw.ResponseWriter - w.Header().Del("Content-Encoding") - } else */if gzw, ok := w.(c.GzipResponseWriter); ok { - w = gzw.ResponseWriter - w.Header().Del("Content-Encoding") - } - return w -} diff --git a/router.go b/router.go index e69de29b..0d7909bb 100644 --- a/router.go +++ b/router.go @@ -0,0 +1,286 @@ +// Now home to the parts of gen_router.go which aren't expected to change from generation to generation +package main + +import ( + "log" + "net/http" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + + c "github.com/Azareal/Gosora/common" +) + +type WriterIntercept struct { + http.ResponseWriter +} + +func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { + return &WriterIntercept{w} +} + +var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) + +func (wi *WriterIntercept) WriteHeader(code int) { + if code == 200 { + h := wi.ResponseWriter.Header() + h.Set("Cache-Control", wiMaxAge) + h.Set("Vary", "Accept-Encoding") + } + wi.ResponseWriter.WriteHeader(code) +} + +type GenRouter struct { + UploadHandler func(http.ResponseWriter, *http.Request) + extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError + + reqLogger *log.Logger + //suspReqLogger *log.Logger + + reqLog2 *RouterLog + suspLog *RouterLog + + sync.RWMutex +} + +type RouterLogLog struct { + File *os.File + Log *log.Logger +} +type RouterLog struct { + FileVal atomic.Value + LogVal atomic.Value + + sync.RWMutex +} + +func (r *GenRouter) DailyTick() error { + r.suspLog.Lock() + defer r.suspLog.Unlock() + + f := r.suspLog.FileVal.Load().(*os.File) + stat, err := f.Stat() + if err != nil { + return nil + } + if stat.Size() < int64(c.Megabyte) { + return nil + } + if err = f.Close(); err != nil { + return err + } + + stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) + f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return err + } + l := log.New(f, "", log.LstdFlags) + r.suspLog.FileVal.Store(f) + r.suspLog.LogVal.Store(l) + + return nil +} + +func NewGenRouter(uploads http.Handler) (*GenRouter, error) { + stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) + createLog := func(name, stimestr string) (*RouterLog, error) { + f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return nil, err + } + l := log.New(f, "", log.LstdFlags) + var aVal atomic.Value + var aVal2 atomic.Value + aVal.Store(f) + aVal2.Store(l) + return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil + } + reqLog, err := createLog("reqs", stimestr) + if err != nil { + return nil, err + } + suspReqLog, err := createLog("reqs-susp", stimestr) + if err != nil { + return nil, err + } + f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return nil, err + } + reqMiscLog := log.New(f3, "", log.LstdFlags) + + return &GenRouter{ + UploadHandler: func(w http.ResponseWriter, r *http.Request) { + writ := NewWriterIntercept(w) + http.StripPrefix("/uploads/", uploads).ServeHTTP(writ, r) + }, + extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), + + reqLogger: reqMiscLog, + reqLog2: reqLog, + suspLog: suspReqLog, + }, nil +} + +func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { + if err.Handled() { + return + } + if err.Type() == "system" { + c.InternalErrorJSQ(err, w, req, err.JSON()) + return + } + c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) +} + +func (r *GenRouter) Handle(_ string, _ http.Handler) { +} + +func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { + r.Lock() + defer r.Unlock() + r.extraRoutes[pattern] = h +} + +func (r *GenRouter) RemoveFunc(pattern string) error { + r.Lock() + defer r.Unlock() + _, ok := r.extraRoutes[pattern] + if !ok { + return ErrNoRoute + } + delete(r.extraRoutes, pattern) + return nil +} + +func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) { + var sb strings.Builder + r.ddumpRequest(req, pre, log, &sb) +} + +// TODO: Some of these sanitisations may be redundant +var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7 +var dumpReqLen2 = len("\nHead : ") + 2 + +func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) { + nfield := func(label, val string) { + sb.WriteString(label) + sb.WriteString(val) + } + field := func(label, val string) { + nfield(label, c.SanitiseSingleLine(val)) + } + ua := req.UserAgent() + + sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) + sb.WriteString(pre) + sb.WriteString("\n") + sb.WriteString(c.SanitiseSingleLine(req.Method)) + sb.WriteRune(' ') + sb.WriteString(c.SanitiseSingleLine(req.URL.Path)) + field("\nUA: ", ua) + + for key, val := range req.Header { + // Avoid logging this for security reasons + if key == "Cookie" { + continue + } + for _, vvalue := range val { + sb.WriteString("\nHead ") + sb.WriteString(c.SanitiseSingleLine(key)) + sb.WriteString(": ") + sb.WriteString(c.SanitiseSingleLine(vvalue)) + } + } + field("\nHost: ", req.Host) + if rawQuery := req.URL.RawQuery; rawQuery != "" { + field("\nURL.RawQuery: ", rawQuery) + } + if ref := req.Referer(); ref != "" { + field("\nRef: ", ref) + } + nfield("\nIP: ", req.RemoteAddr) + sb.WriteString("\n") + + l.RLock() + l.LogVal.Load().(*log.Logger).Print(sb.String()) + l.RUnlock() +} + +func (r *GenRouter) DumpRequest(req *http.Request, pre string) { + r.dumpRequest(req, pre, r.reqLog2) +} + +func (r *GenRouter) unknownUA(req *http.Request) { + if c.Dev.DebugMode { + var presb strings.Builder + presb.WriteString("Unknown UA: ") + for _, ch := range req.UserAgent() { + presb.WriteString(strconv.Itoa(int(ch))) + presb.WriteRune(' ') + } + r.ddumpRequest(req, "", r.reqLog2, &presb) + } else { + r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) + } +} + +func (r *GenRouter) susp1(req *http.Request) bool { + if !strings.Contains(req.URL.Path, ".") { + return false + } + if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ { + return true + } + lp := strings.ToLower(req.URL.Path) + // TODO: Flag any requests which has a dot with anything but a number after that + // TODO: Use HasSuffix to avoid over-scanning? + return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action +} + +func (r *GenRouter) suspScan(req *http.Request) { + // TODO: Cover more suspicious strings and at a lower layer than this + var ch rune + var susp bool + for _, ch = range req.URL.Path { //char + if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { + susp = true + break + } + } + + // Avoid logging the same request multiple times + susp2 := r.susp1(req) + if susp && susp2 { + r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path") + } else if susp { + r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path") + } else if susp2 { + r.SuspiciousRequest(req, "Bad snippet in path") + } else if c.Dev.FullReqLog { + r.DumpRequest(req, "") + } +} + +func isLocalHost(h string) bool { + return h == "localhost" || h == "127.0.0.1" || h == "::1" +} + +//var brPool = sync.Pool{} +var gzipPool = sync.Pool{} + +//var uaBufPool = sync.Pool{} + +func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { + /*if bzw, ok := w.(c.BrResponseWriter); ok { + w = bzw.ResponseWriter + w.Header().Del("Content-Encoding") + } else */if gzw, ok := w.(c.GzipResponseWriter); ok { + w = gzw.ResponseWriter + w.Header().Del("Content-Encoding") + } + return w +} diff --git a/router_gen/main.go b/router_gen/main.go index 33f95e51..8cac26eb 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -419,15 +419,12 @@ func main() { package main import ( - "log" "strings" //"bytes" "strconv" "compress/gzip" - "sync" "sync/atomic" "errors" - "os" "net/http" "time" @@ -505,24 +502,6 @@ func init() { } } -type WriterIntercept struct { - http.ResponseWriter -} - -func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { - return &WriterIntercept{w} -} - -var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) -func (wi *WriterIntercept) WriteHeader(code int) { - if code == 200 { - h := wi.ResponseWriter.Header() - h.Set("Cache-Control", wiMaxAge) - h.Set("Vary", "Accept-Encoding") - } - wi.ResponseWriter.WriteHeader(code) -} - // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS type HTTPSRedirect struct {} @@ -533,149 +512,17 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Redirect(w, req, dest, http.StatusTemporaryRedirect) } -type GenRouter struct { - UploadHandler func(http.ResponseWriter, *http.Request) - extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError - requestLogger *log.Logger - suspReqLogger *log.Logger - - sync.RWMutex -} - -func NewGenRouter(uploads http.Handler) (*GenRouter, error) { - f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, r *http.Request) { - writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) - }, - extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), - requestLogger: log.New(f, "", log.LstdFlags), - suspReqLogger: log.New(f2, "", log.LstdFlags), - }, nil -} - -func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { - if err.Handled() { - return - } - if err.Type() == "system" { - c.InternalErrorJSQ(err, w, req, err.JSON()) - return - } - c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) -} - -func (r *GenRouter) Handle(_ string, _ http.Handler) { -} - -func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { - r.Lock() - defer r.Unlock() - r.extraRoutes[pattern] = h -} - -func (r *GenRouter) RemoveFunc(pattern string) error { - r.Lock() - defer r.Unlock() - _, ok := r.extraRoutes[pattern] - if !ok { - return ErrNoRoute - } - delete(r.extraRoutes, pattern) - return nil -} - -func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) { - var sb strings.Builder - r.ddumpRequest(req,pre,log,&sb) -} - -// TODO: Some of these sanitisations may be redundant -var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3 -var dumpReqLen2 = len("\nHead : ") + 2 -func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) { - nfield := func(label, val string) { - sb.WriteString(label) - sb.WriteString(val) - } - field := func(label, val string) { - nfield(label,c.SanitiseSingleLine(val)) - } - ua := req.UserAgent() - sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) - sb.WriteString(pre) - field("\nUA: ",ua) - field("\nMethod: ",req.Method) - for key, value := range req.Header { - // Avoid logging this for security reasons - if key == "Cookie" { - continue - } - for _, vvalue := range value { - sb.WriteString("\nHead ") - sb.WriteString(c.SanitiseSingleLine(key)) - sb.WriteString(": ") - sb.WriteString(c.SanitiseSingleLine(vvalue)) - } - } - field("\nHost: ",req.Host) - field("\nURL.Path: ",req.URL.Path) - field("\nURL.RawQuery: ",req.URL.RawQuery) - ref := req.Referer() - if ref != "" { - field("\nRef: ",req.Referer()) - } - nfield("\nIP: ",req.RemoteAddr) - sb.WriteString("\n") - - log.Print(sb.String()) -} - -func (r *GenRouter) DumpRequest(req *http.Request, pre string) { - r.dumpRequest(req,pre,r.requestLogger) -} - func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { var sb strings.Builder if pre != "" { - sb.WriteString("Suspicious Request") + sb.WriteString("Suspicious Request\n") } else { pre = "Suspicious Request" } - r.ddumpRequest(req,pre,r.suspReqLogger,&sb) + r.ddumpRequest(req,pre,r.suspLog,&sb) co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) } -func (r *GenRouter) unknownUA(req *http.Request) { - if c.Dev.DebugMode { - var presb strings.Builder - presb.WriteString("Unknown UA: ") - for _, ch := range req.UserAgent() { - presb.WriteString(strconv.Itoa(int(ch))) - presb.WriteRune(' ') - } - r.ddumpRequest(req, "", r.requestLogger, &presb) - } else { - r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) - } -} - -func isLocalHost(h string) bool { - return h=="localhost" || h=="127.0.0.1" || h=="::1" -} - -//var brPool = sync.Pool{} -var gzipPool = sync.Pool{} -//var uaBufPool = sync.Pool{} - // TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: SetDefaultPath // TODO: GetDefaultPath @@ -721,11 +568,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { // TODO: Abstract the redirect logic? w.Header().Set("Connection", "close") - var s string + var s, p string if c.Config.SslSchema { s = "s" } - var p string if c.Site.PortInt != 80 && c.Site.PortInt != 443 { p = ":"+c.Site.Port } @@ -743,23 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { malformedRequest(3) return } - if c.Dev.FullReqLog { - r.DumpRequest(req,"") - } - - // TODO: Cover more suspicious strings and at a lower layer than this - for _, ch := range req.URL.Path { //char - if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { - r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path") - break - } - } - lp := strings.ToLower(req.URL.Path) - // TODO: Flag any requests which has a dot with anything but a number after that - // TODO: Use HasSuffix to avoid over-scanning? - if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action - r.SuspiciousRequest(req,"Bad snippet in path") - } + r.suspScan(req) // Indirect the default route onto a different one if req.URL.Path == "/" { @@ -808,7 +638,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } if c.Dev.SuperDebug { - r.requestLogger.Print("before PreRoute") + r.reqLogger.Print("before PreRoute") } /*if c.Dev.QuicPort != 0 { @@ -871,8 +701,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Test this items = items[:0] r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") - r.requestLogger.Print("UA Buf: ", buf) - r.requestLogger.Print("UA Buf String: ", string(buf)) + r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf)) break } } @@ -890,11 +719,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if c.Dev.SuperDebug { - r.requestLogger.Print("parsed agent: ", agent) - r.requestLogger.Print("os: ", os) - r.requestLogger.Printf("items: %+v\n",items) + r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os) + r.reqLogger.Printf("items: %+v\n",items) /*for _, it := range items { - r.requestLogger.Printf("it: %+v\n",string(it)) + r.reqLogger.Printf("it: %+v\n",string(it)) }*/ } @@ -985,7 +813,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { user := &ucpy user.LastAgent = agent if c.Dev.SuperDebug { - r.requestLogger.Print( + r.reqLogger.Print( "after PreRoute\n" + "routeMapEnum: ", routeMapEnum) } @@ -1096,8 +924,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * case "favicon.ico": w = r.responseWriter(w) req.URL.Path = "/s/favicon.ico" - routes.StaticFile(w,req) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn) + routes.StaticFile(w,req) return nil case "opensearch.xml": co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn) @@ -1123,7 +951,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) lp := strings.ToLower(req.URL.Path) - if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { + if strings.Contains(lp,"w") { + if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { + r.SuspiciousRequest(req,"Bad Route") + return c.MicroNotFound(w,req) + } + } + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } @@ -1138,17 +972,6 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } return err } - -func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { - /*if bzw, ok := w.(c.BrResponseWriter); ok { - w = bzw.ResponseWriter - w.Header().Del("Content-Encoding") - } else */if gzw, ok := w.(c.GzipResponseWriter); ok { - w = gzw.ResponseWriter - w.Header().Del("Content-Encoding") - } - return w -} ` tmpl := template.Must(template.New("router").Parse(fileData)) var b bytes.Buffer @@ -1161,20 +984,18 @@ func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { } func writeFile(name, content string) { - f, err := os.Create(name) - if err != nil { - log.Fatal(err) + f, e := os.Create(name) + if e != nil { + log.Fatal(e) } - _, err = f.WriteString(content) - if err != nil { - log.Fatal(err) + _, e = f.WriteString(content) + if e != nil { + log.Fatal(e) } - err = f.Sync() - if err != nil { - log.Fatal(err) + if e = f.Sync(); e != nil { + log.Fatal(e) } - err = f.Close() - if err != nil { - log.Fatal(err) + if e = f.Close(); e != nil { + log.Fatal(e) } }