diff --git a/gen_router.go b/gen_router.go index 9313e2d3..4557e0ff 100644 --- a/gen_router.go +++ b/gen_router.go @@ -3,15 +3,12 @@ package main import ( - "log" "strings" //"bytes" "strconv" "compress/gzip" - "sync" "sync/atomic" "errors" - "os" "net/http" "time" @@ -910,24 +907,6 @@ func init() { } } -type WriterIntercept struct { - http.ResponseWriter -} - -func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { - return &WriterIntercept{w} -} - -var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) -func (wi *WriterIntercept) WriteHeader(code int) { - if code == 200 { - h := wi.ResponseWriter.Header() - h.Set("Cache-Control", wiMaxAge) - h.Set("Vary", "Accept-Encoding") - } - wi.ResponseWriter.WriteHeader(code) -} - // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS type HTTPSRedirect struct {} @@ -938,116 +917,6 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Redirect(w, req, dest, http.StatusTemporaryRedirect) } -type GenRouter struct { - UploadHandler func(http.ResponseWriter, *http.Request) - extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError - requestLogger *log.Logger - suspReqLogger *log.Logger - - sync.RWMutex -} - -func NewGenRouter(uploads http.Handler) (*GenRouter, error) { - f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, r *http.Request) { - writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) - }, - extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), - requestLogger: log.New(f, "", log.LstdFlags), - suspReqLogger: log.New(f2, "", log.LstdFlags), - }, nil -} - -func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { - if err.Handled() { - return - } - if err.Type() == "system" { - c.InternalErrorJSQ(err, w, req, err.JSON()) - return - } - c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) -} - -func (r *GenRouter) Handle(_ string, _ http.Handler) { -} - -func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { - r.Lock() - defer r.Unlock() - r.extraRoutes[pattern] = h -} - -func (r *GenRouter) RemoveFunc(pattern string) error { - r.Lock() - defer r.Unlock() - _, ok := r.extraRoutes[pattern] - if !ok { - return ErrNoRoute - } - delete(r.extraRoutes, pattern) - return nil -} - -func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) { - var sb strings.Builder - r.ddumpRequest(req,pre,log,&sb) -} - -// TODO: Some of these sanitisations may be redundant -var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3 -var dumpReqLen2 = len("\nHead : ") + 2 -func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) { - nfield := func(label, val string) { - sb.WriteString(label) - sb.WriteString(val) - } - field := func(label, val string) { - nfield(label,c.SanitiseSingleLine(val)) - } - ua := req.UserAgent() - sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) - sb.WriteString(pre) - field("\nUA: ",ua) - field("\nMethod: ",req.Method) - for key, value := range req.Header { - // Avoid logging this for security reasons - if key == "Cookie" { - continue - } - for _, vvalue := range value { - sb.WriteString("\nHead ") - sb.WriteString(c.SanitiseSingleLine(key)) - sb.WriteString(": ") - sb.WriteString(c.SanitiseSingleLine(vvalue)) - } - } - field("\nHost: ",req.Host) - field("\nURL.Path: ",req.URL.Path) - field("\nURL.RawQuery: ",req.URL.RawQuery) - ref := req.Referer() - if ref != "" { - field("\nRef: ",req.Referer()) - } - nfield("\nIP: ",req.RemoteAddr) - sb.WriteString("\n") - - log.Print(sb.String()) -} - -func (r *GenRouter) DumpRequest(req *http.Request, pre string) { - r.dumpRequest(req,pre,r.requestLogger) -} - func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { var sb strings.Builder if pre != "" { @@ -1055,32 +924,10 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { } else { pre = "Suspicious Request" } - r.ddumpRequest(req,pre,r.suspReqLogger,&sb) + r.ddumpRequest(req,pre,r.suspLog,&sb) co.AgentViewCounter.Bump(43) } -func (r *GenRouter) unknownUA(req *http.Request) { - if c.Dev.DebugMode { - var presb strings.Builder - presb.WriteString("Unknown UA: ") - for _, ch := range req.UserAgent() { - presb.WriteString(strconv.Itoa(int(ch))) - presb.WriteRune(' ') - } - r.ddumpRequest(req, "", r.requestLogger, &presb) - } else { - r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) - } -} - -func isLocalHost(h string) bool { - return h=="localhost" || h=="127.0.0.1" || h=="::1" -} - -//var brPool = sync.Pool{} -var gzipPool = sync.Pool{} -//var uaBufPool = sync.Pool{} - // TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: SetDefaultPath // TODO: GetDefaultPath @@ -1126,11 +973,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { // TODO: Abstract the redirect logic? w.Header().Set("Connection", "close") - var s string + var s, p string if c.Config.SslSchema { s = "s" } - var p string if c.Site.PortInt != 80 && c.Site.PortInt != 443 { p = ":"+c.Site.Port } @@ -1148,23 +994,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { malformedRequest(3) return } - if c.Dev.FullReqLog { - r.DumpRequest(req,"") - } - - // TODO: Cover more suspicious strings and at a lower layer than this - for _, ch := range req.URL.Path { //char - if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { - r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path") - break - } - } - lp := strings.ToLower(req.URL.Path) - // TODO: Flag any requests which has a dot with anything but a number after that - // TODO: Use HasSuffix to avoid over-scanning? - if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action - r.SuspiciousRequest(req,"Bad snippet in path") - } + r.suspScan(req) // Indirect the default route onto a different one if req.URL.Path == "/" { @@ -1213,7 +1043,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } if c.Dev.SuperDebug { - r.requestLogger.Print("before PreRoute") + r.reqLogger.Print("before PreRoute") } /*if c.Dev.QuicPort != 0 { @@ -1276,8 +1106,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Test this items = items[:0] r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") - r.requestLogger.Print("UA Buf: ", buf) - r.requestLogger.Print("UA Buf String: ", string(buf)) + r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf)) break } } @@ -1295,11 +1124,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if c.Dev.SuperDebug { - r.requestLogger.Print("parsed agent: ", agent) - r.requestLogger.Print("os: ", os) - r.requestLogger.Printf("items: %+v\n",items) + r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os) + r.reqLogger.Printf("items: %+v\n",items) /*for _, it := range items { - r.requestLogger.Printf("it: %+v\n",string(it)) + r.reqLogger.Printf("it: %+v\n",string(it)) }*/ } @@ -1390,7 +1218,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { user := &ucpy user.LastAgent = agent if c.Dev.SuperDebug { - r.requestLogger.Print( + r.reqLogger.Print( "after PreRoute\n" + "routeMapEnum: ", routeMapEnum) } @@ -2767,12 +2595,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3(150, cn) } case "/profile": - err = c.NoSessionMismatch(w,req,user) + err = c.MemberOnly(w,req,user) if err != nil { return err } - err = c.MemberOnly(w,req,user) + err = c.NoSessionMismatch(w,req,user) if err != nil { return err } @@ -2926,8 +2754,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * case "favicon.ico": w = r.responseWriter(w) req.URL.Path = "/s/favicon.ico" - routes.StaticFile(w,req) co.RouteViewCounter.Bump3(173, cn) + routes.StaticFile(w,req) return nil case "opensearch.xml": co.RouteViewCounter.Bump3(172, cn) @@ -2953,7 +2781,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3(174, cn) lp := strings.ToLower(req.URL.Path) - if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { + if strings.Contains(lp,"w") { + if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { + r.SuspiciousRequest(req,"Bad Route") + return c.MicroNotFound(w,req) + } + } + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } @@ -2968,14 +2802,3 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } return err } - -func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { - /*if bzw, ok := w.(c.BrResponseWriter); ok { - w = bzw.ResponseWriter - w.Header().Del("Content-Encoding") - } else */if gzw, ok := w.(c.GzipResponseWriter); ok { - w = gzw.ResponseWriter - w.Header().Del("Content-Encoding") - } - return w -} diff --git a/router.go b/router.go index e69de29b..0d7909bb 100644 --- a/router.go +++ b/router.go @@ -0,0 +1,286 @@ +// Now home to the parts of gen_router.go which aren't expected to change from generation to generation +package main + +import ( + "log" + "net/http" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + + c "github.com/Azareal/Gosora/common" +) + +type WriterIntercept struct { + http.ResponseWriter +} + +func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { + return &WriterIntercept{w} +} + +var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) + +func (wi *WriterIntercept) WriteHeader(code int) { + if code == 200 { + h := wi.ResponseWriter.Header() + h.Set("Cache-Control", wiMaxAge) + h.Set("Vary", "Accept-Encoding") + } + wi.ResponseWriter.WriteHeader(code) +} + +type GenRouter struct { + UploadHandler func(http.ResponseWriter, *http.Request) + extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError + + reqLogger *log.Logger + //suspReqLogger *log.Logger + + reqLog2 *RouterLog + suspLog *RouterLog + + sync.RWMutex +} + +type RouterLogLog struct { + File *os.File + Log *log.Logger +} +type RouterLog struct { + FileVal atomic.Value + LogVal atomic.Value + + sync.RWMutex +} + +func (r *GenRouter) DailyTick() error { + r.suspLog.Lock() + defer r.suspLog.Unlock() + + f := r.suspLog.FileVal.Load().(*os.File) + stat, err := f.Stat() + if err != nil { + return nil + } + if stat.Size() < int64(c.Megabyte) { + return nil + } + if err = f.Close(); err != nil { + return err + } + + stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) + f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return err + } + l := log.New(f, "", log.LstdFlags) + r.suspLog.FileVal.Store(f) + r.suspLog.LogVal.Store(l) + + return nil +} + +func NewGenRouter(uploads http.Handler) (*GenRouter, error) { + stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) + createLog := func(name, stimestr string) (*RouterLog, error) { + f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return nil, err + } + l := log.New(f, "", log.LstdFlags) + var aVal atomic.Value + var aVal2 atomic.Value + aVal.Store(f) + aVal2.Store(l) + return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil + } + reqLog, err := createLog("reqs", stimestr) + if err != nil { + return nil, err + } + suspReqLog, err := createLog("reqs-susp", stimestr) + if err != nil { + return nil, err + } + f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return nil, err + } + reqMiscLog := log.New(f3, "", log.LstdFlags) + + return &GenRouter{ + UploadHandler: func(w http.ResponseWriter, r *http.Request) { + writ := NewWriterIntercept(w) + http.StripPrefix("/uploads/", uploads).ServeHTTP(writ, r) + }, + extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), + + reqLogger: reqMiscLog, + reqLog2: reqLog, + suspLog: suspReqLog, + }, nil +} + +func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { + if err.Handled() { + return + } + if err.Type() == "system" { + c.InternalErrorJSQ(err, w, req, err.JSON()) + return + } + c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) +} + +func (r *GenRouter) Handle(_ string, _ http.Handler) { +} + +func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { + r.Lock() + defer r.Unlock() + r.extraRoutes[pattern] = h +} + +func (r *GenRouter) RemoveFunc(pattern string) error { + r.Lock() + defer r.Unlock() + _, ok := r.extraRoutes[pattern] + if !ok { + return ErrNoRoute + } + delete(r.extraRoutes, pattern) + return nil +} + +func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) { + var sb strings.Builder + r.ddumpRequest(req, pre, log, &sb) +} + +// TODO: Some of these sanitisations may be redundant +var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7 +var dumpReqLen2 = len("\nHead : ") + 2 + +func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) { + nfield := func(label, val string) { + sb.WriteString(label) + sb.WriteString(val) + } + field := func(label, val string) { + nfield(label, c.SanitiseSingleLine(val)) + } + ua := req.UserAgent() + + sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) + sb.WriteString(pre) + sb.WriteString("\n") + sb.WriteString(c.SanitiseSingleLine(req.Method)) + sb.WriteRune(' ') + sb.WriteString(c.SanitiseSingleLine(req.URL.Path)) + field("\nUA: ", ua) + + for key, val := range req.Header { + // Avoid logging this for security reasons + if key == "Cookie" { + continue + } + for _, vvalue := range val { + sb.WriteString("\nHead ") + sb.WriteString(c.SanitiseSingleLine(key)) + sb.WriteString(": ") + sb.WriteString(c.SanitiseSingleLine(vvalue)) + } + } + field("\nHost: ", req.Host) + if rawQuery := req.URL.RawQuery; rawQuery != "" { + field("\nURL.RawQuery: ", rawQuery) + } + if ref := req.Referer(); ref != "" { + field("\nRef: ", ref) + } + nfield("\nIP: ", req.RemoteAddr) + sb.WriteString("\n") + + l.RLock() + l.LogVal.Load().(*log.Logger).Print(sb.String()) + l.RUnlock() +} + +func (r *GenRouter) DumpRequest(req *http.Request, pre string) { + r.dumpRequest(req, pre, r.reqLog2) +} + +func (r *GenRouter) unknownUA(req *http.Request) { + if c.Dev.DebugMode { + var presb strings.Builder + presb.WriteString("Unknown UA: ") + for _, ch := range req.UserAgent() { + presb.WriteString(strconv.Itoa(int(ch))) + presb.WriteRune(' ') + } + r.ddumpRequest(req, "", r.reqLog2, &presb) + } else { + r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) + } +} + +func (r *GenRouter) susp1(req *http.Request) bool { + if !strings.Contains(req.URL.Path, ".") { + return false + } + if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ { + return true + } + lp := strings.ToLower(req.URL.Path) + // TODO: Flag any requests which has a dot with anything but a number after that + // TODO: Use HasSuffix to avoid over-scanning? + return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action +} + +func (r *GenRouter) suspScan(req *http.Request) { + // TODO: Cover more suspicious strings and at a lower layer than this + var ch rune + var susp bool + for _, ch = range req.URL.Path { //char + if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { + susp = true + break + } + } + + // Avoid logging the same request multiple times + susp2 := r.susp1(req) + if susp && susp2 { + r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path") + } else if susp { + r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path") + } else if susp2 { + r.SuspiciousRequest(req, "Bad snippet in path") + } else if c.Dev.FullReqLog { + r.DumpRequest(req, "") + } +} + +func isLocalHost(h string) bool { + return h == "localhost" || h == "127.0.0.1" || h == "::1" +} + +//var brPool = sync.Pool{} +var gzipPool = sync.Pool{} + +//var uaBufPool = sync.Pool{} + +func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { + /*if bzw, ok := w.(c.BrResponseWriter); ok { + w = bzw.ResponseWriter + w.Header().Del("Content-Encoding") + } else */if gzw, ok := w.(c.GzipResponseWriter); ok { + w = gzw.ResponseWriter + w.Header().Del("Content-Encoding") + } + return w +} diff --git a/router_gen/main.go b/router_gen/main.go index 33f95e51..8cac26eb 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -419,15 +419,12 @@ func main() { package main import ( - "log" "strings" //"bytes" "strconv" "compress/gzip" - "sync" "sync/atomic" "errors" - "os" "net/http" "time" @@ -505,24 +502,6 @@ func init() { } } -type WriterIntercept struct { - http.ResponseWriter -} - -func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { - return &WriterIntercept{w} -} - -var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) -func (wi *WriterIntercept) WriteHeader(code int) { - if code == 200 { - h := wi.ResponseWriter.Header() - h.Set("Cache-Control", wiMaxAge) - h.Set("Vary", "Accept-Encoding") - } - wi.ResponseWriter.WriteHeader(code) -} - // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS type HTTPSRedirect struct {} @@ -533,149 +512,17 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Redirect(w, req, dest, http.StatusTemporaryRedirect) } -type GenRouter struct { - UploadHandler func(http.ResponseWriter, *http.Request) - extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError - requestLogger *log.Logger - suspReqLogger *log.Logger - - sync.RWMutex -} - -func NewGenRouter(uploads http.Handler) (*GenRouter, error) { - f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) - if err != nil { - return nil, err - } - return &GenRouter{ - UploadHandler: func(w http.ResponseWriter, r *http.Request) { - writ := NewWriterIntercept(w) - http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r) - }, - extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), - requestLogger: log.New(f, "", log.LstdFlags), - suspReqLogger: log.New(f2, "", log.LstdFlags), - }, nil -} - -func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { - if err.Handled() { - return - } - if err.Type() == "system" { - c.InternalErrorJSQ(err, w, req, err.JSON()) - return - } - c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) -} - -func (r *GenRouter) Handle(_ string, _ http.Handler) { -} - -func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { - r.Lock() - defer r.Unlock() - r.extraRoutes[pattern] = h -} - -func (r *GenRouter) RemoveFunc(pattern string) error { - r.Lock() - defer r.Unlock() - _, ok := r.extraRoutes[pattern] - if !ok { - return ErrNoRoute - } - delete(r.extraRoutes, pattern) - return nil -} - -func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) { - var sb strings.Builder - r.ddumpRequest(req,pre,log,&sb) -} - -// TODO: Some of these sanitisations may be redundant -var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3 -var dumpReqLen2 = len("\nHead : ") + 2 -func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) { - nfield := func(label, val string) { - sb.WriteString(label) - sb.WriteString(val) - } - field := func(label, val string) { - nfield(label,c.SanitiseSingleLine(val)) - } - ua := req.UserAgent() - sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) - sb.WriteString(pre) - field("\nUA: ",ua) - field("\nMethod: ",req.Method) - for key, value := range req.Header { - // Avoid logging this for security reasons - if key == "Cookie" { - continue - } - for _, vvalue := range value { - sb.WriteString("\nHead ") - sb.WriteString(c.SanitiseSingleLine(key)) - sb.WriteString(": ") - sb.WriteString(c.SanitiseSingleLine(vvalue)) - } - } - field("\nHost: ",req.Host) - field("\nURL.Path: ",req.URL.Path) - field("\nURL.RawQuery: ",req.URL.RawQuery) - ref := req.Referer() - if ref != "" { - field("\nRef: ",req.Referer()) - } - nfield("\nIP: ",req.RemoteAddr) - sb.WriteString("\n") - - log.Print(sb.String()) -} - -func (r *GenRouter) DumpRequest(req *http.Request, pre string) { - r.dumpRequest(req,pre,r.requestLogger) -} - func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { var sb strings.Builder if pre != "" { - sb.WriteString("Suspicious Request") + sb.WriteString("Suspicious Request\n") } else { pre = "Suspicious Request" } - r.ddumpRequest(req,pre,r.suspReqLogger,&sb) + r.ddumpRequest(req,pre,r.suspLog,&sb) co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) } -func (r *GenRouter) unknownUA(req *http.Request) { - if c.Dev.DebugMode { - var presb strings.Builder - presb.WriteString("Unknown UA: ") - for _, ch := range req.UserAgent() { - presb.WriteString(strconv.Itoa(int(ch))) - presb.WriteRune(' ') - } - r.ddumpRequest(req, "", r.requestLogger, &presb) - } else { - r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) - } -} - -func isLocalHost(h string) bool { - return h=="localhost" || h=="127.0.0.1" || h=="::1" -} - -//var brPool = sync.Pool{} -var gzipPool = sync.Pool{} -//var uaBufPool = sync.Pool{} - // TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: SetDefaultPath // TODO: GetDefaultPath @@ -721,11 +568,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { // TODO: Abstract the redirect logic? w.Header().Set("Connection", "close") - var s string + var s, p string if c.Config.SslSchema { s = "s" } - var p string if c.Site.PortInt != 80 && c.Site.PortInt != 443 { p = ":"+c.Site.Port } @@ -743,23 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { malformedRequest(3) return } - if c.Dev.FullReqLog { - r.DumpRequest(req,"") - } - - // TODO: Cover more suspicious strings and at a lower layer than this - for _, ch := range req.URL.Path { //char - if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { - r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path") - break - } - } - lp := strings.ToLower(req.URL.Path) - // TODO: Flag any requests which has a dot with anything but a number after that - // TODO: Use HasSuffix to avoid over-scanning? - if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action - r.SuspiciousRequest(req,"Bad snippet in path") - } + r.suspScan(req) // Indirect the default route onto a different one if req.URL.Path == "/" { @@ -808,7 +638,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } if c.Dev.SuperDebug { - r.requestLogger.Print("before PreRoute") + r.reqLogger.Print("before PreRoute") } /*if c.Dev.QuicPort != 0 { @@ -871,8 +701,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Test this items = items[:0] r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") - r.requestLogger.Print("UA Buf: ", buf) - r.requestLogger.Print("UA Buf String: ", string(buf)) + r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf)) break } } @@ -890,11 +719,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if c.Dev.SuperDebug { - r.requestLogger.Print("parsed agent: ", agent) - r.requestLogger.Print("os: ", os) - r.requestLogger.Printf("items: %+v\n",items) + r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os) + r.reqLogger.Printf("items: %+v\n",items) /*for _, it := range items { - r.requestLogger.Printf("it: %+v\n",string(it)) + r.reqLogger.Printf("it: %+v\n",string(it)) }*/ } @@ -985,7 +813,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { user := &ucpy user.LastAgent = agent if c.Dev.SuperDebug { - r.requestLogger.Print( + r.reqLogger.Print( "after PreRoute\n" + "routeMapEnum: ", routeMapEnum) } @@ -1096,8 +924,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * case "favicon.ico": w = r.responseWriter(w) req.URL.Path = "/s/favicon.ico" - routes.StaticFile(w,req) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn) + routes.StaticFile(w,req) return nil case "opensearch.xml": co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn) @@ -1123,7 +951,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) lp := strings.ToLower(req.URL.Path) - if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { + if strings.Contains(lp,"w") { + if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { + r.SuspiciousRequest(req,"Bad Route") + return c.MicroNotFound(w,req) + } + } + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } @@ -1138,17 +972,6 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } return err } - -func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { - /*if bzw, ok := w.(c.BrResponseWriter); ok { - w = bzw.ResponseWriter - w.Header().Del("Content-Encoding") - } else */if gzw, ok := w.(c.GzipResponseWriter); ok { - w = gzw.ResponseWriter - w.Header().Del("Content-Encoding") - } - return w -} ` tmpl := template.Must(template.New("router").Parse(fileData)) var b bytes.Buffer @@ -1161,20 +984,18 @@ func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { } func writeFile(name, content string) { - f, err := os.Create(name) - if err != nil { - log.Fatal(err) + f, e := os.Create(name) + if e != nil { + log.Fatal(e) } - _, err = f.WriteString(content) - if err != nil { - log.Fatal(err) + _, e = f.WriteString(content) + if e != nil { + log.Fatal(e) } - err = f.Sync() - if err != nil { - log.Fatal(err) + if e = f.Sync(); e != nil { + log.Fatal(e) } - err = f.Close() - if err != nil { - log.Fatal(err) + if e = f.Close(); e != nil { + log.Fatal(e) } }