// Now home to the parts of gen_router.go which aren't expected to change from generation to generation package main import ( "log" "net/http" "os" "strconv" "strings" "sync" "sync/atomic" "time" c "git.tuxpa.in/a/gosora/common" co "git.tuxpa.in/a/gosora/common/counters" ) // TODO: Stop spilling these into the package scope? func init() { _ = time.Now() co.SetRouteMapEnum(routeMapEnum) co.SetReverseRouteMapEnum(reverseRouteMapEnum) co.SetAgentMapEnum(agentMapEnum) co.SetReverseAgentMapEnum(reverseAgentMapEnum) co.SetOSMapEnum(osMapEnum) co.SetReverseOSMapEnum(reverseOSMapEnum) g := func(n string) int { a, ok := agentMapEnum[n] if !ok { panic("name not found in agentMapEnum") } return a } c.Chrome = g("chrome") c.Firefox = g("firefox") c.SimpleBots = []int{ g("semrush"), g("ahrefs"), g("python"), //g("go"), g("curl"), } } type WriterIntercept struct { http.ResponseWriter } func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept { return &WriterIntercept{w} } var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day)) func (wi *WriterIntercept) WriteHeader(code int) { if code == 200 { h := wi.ResponseWriter.Header() h.Set("Cache-Control", wiMaxAge) h.Set("Vary", "Accept-Encoding") } wi.ResponseWriter.WriteHeader(code) } type GenRouter struct { UploadHandler func(http.ResponseWriter, *http.Request) extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError reqLogger *log.Logger reqLog2 *RouterLog suspLog *RouterLog sync.RWMutex } type RouterLogLog struct { File *os.File Log *log.Logger } type RouterLog struct { FileVal atomic.Value LogVal atomic.Value sync.RWMutex } func (r *GenRouter) DailyTick() error { currentTime := time.Now() rotateLog := func(l *RouterLog, name string) error { l.Lock() defer l.Unlock() f := l.FileVal.Load().(*os.File) stat, e := f.Stat() if e != nil { return nil } if (stat.Size() < int64(c.Megabyte)) && (currentTime.Sub(c.StartTime).Hours() >= (24 * 7)) { return nil } if e = f.Close(); e != nil { return e } stimestr := strconv.FormatInt(currentTime.Unix(), 10) f, e = os.OpenFile(c.Config.LogDir+name+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if e != nil { return e } lval := log.New(f, "", log.LstdFlags) l.FileVal.Store(f) l.LogVal.Store(lval) return nil } if !c.Config.DisableSuspLog { err := rotateLog(r.suspLog, "reqs-susp-") if err != nil { return err } } return rotateLog(r.reqLog2, "reqs-") } type RouterConfig struct { Uploads http.Handler DisableTick bool } func NewGenRouter(cfg *RouterConfig) (*GenRouter, error) { stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) createLog := func(name, stimestr string) (*RouterLog, error) { f, err := os.OpenFile(c.Config.LogDir+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { return nil, err } l := log.New(f, "", log.LstdFlags) var aVal atomic.Value var aVal2 atomic.Value aVal.Store(f) aVal2.Store(l) return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil } reqLog, err := createLog("reqs", stimestr) if err != nil { return nil, err } var suspReqLog *RouterLog if !c.Config.DisableSuspLog { suspReqLog, err = createLog("reqs-susp", stimestr) if err != nil { return nil, err } } f3, err := os.OpenFile(c.Config.LogDir+"reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { return nil, err } reqMiscLog := log.New(f3, "", log.LstdFlags) ro := &GenRouter{ UploadHandler: func(w http.ResponseWriter, r *http.Request) { writ := NewWriterIntercept(w) http.StripPrefix("/uploads/", cfg.Uploads).ServeHTTP(writ, r) }, extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError), reqLogger: reqMiscLog, reqLog2: reqLog, suspLog: suspReqLog, } if !cfg.DisableTick { c.Tasks.Day.Add(ro.DailyTick) } return ro, nil } func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) { if err.Handled() { return } if err.Type() == "system" { c.InternalErrorJSQ(err, w, req, err.JSON()) return } c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON()) } func (r *GenRouter) Handle(_ string, _ http.Handler) { } func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) { r.Lock() defer r.Unlock() r.extraRoutes[pattern] = h } func (r *GenRouter) RemoveFunc(pattern string) error { r.Lock() defer r.Unlock() _, ok := r.extraRoutes[pattern] if !ok { return ErrNoRoute } delete(r.extraRoutes, pattern) return nil } func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) { var sb strings.Builder r.ddumpRequest(req, pre, log, &sb) } // TODO: Some of these sanitisations may be redundant var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7 var dumpReqLen2 = len("\nHead : ") + 2 func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) { nfield := func(label, val string) { sb.WriteString(label) sb.WriteString(val) } field := func(label, val string) { nfield(label, c.SanitiseSingleLine(val)) } ua := req.UserAgent() sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header))) sb.WriteString(pre) sb.WriteString("\n") sb.WriteString(c.SanitiseSingleLine(req.Method)) sb.WriteRune(' ') sb.WriteString(c.SanitiseSingleLine(req.URL.Path)) field("\nUA: ", ua) for key, val := range req.Header { // Avoid logging this for security reasons if key == "Cookie" { continue } for _, vvalue := range val { sb.WriteString("\nHead ") sb.WriteString(c.SanitiseSingleLine(key)) sb.WriteString(": ") sb.WriteString(c.SanitiseSingleLine(vvalue)) } } field("\nHost: ", req.Host) if rawQuery := req.URL.RawQuery; rawQuery != "" { field("\nURL.RawQuery: ", rawQuery) } if ref := req.Referer(); ref != "" { field("\nRef: ", ref) } nfield("\nIP: ", req.RemoteAddr) sb.WriteString("\n") str := sb.String() l.RLock() l.LogVal.Load().(*log.Logger).Print(str) l.RUnlock() } func (r *GenRouter) DumpRequest(req *http.Request, pre string) { r.dumpRequest(req, pre, r.reqLog2) } func (r *GenRouter) unknownUA(req *http.Request) { if c.Dev.DebugMode { var presb strings.Builder presb.WriteString("Unknown UA: ") for _, ch := range req.UserAgent() { presb.WriteString(strconv.Itoa(int(ch))) presb.WriteRune(' ') } r.ddumpRequest(req, "", r.reqLog2, &presb) } else { r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent())) } } func (r *GenRouter) susp1(req *http.Request) bool { if !strings.Contains(req.URL.Path, ".") { return false } if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ { return true } lp := strings.ToLower(req.URL.Path) // TODO: Flag any requests which has a dot with anything but a number after that // TODO: Use HasSuffix to avoid over-scanning? return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action } func (r *GenRouter) suspScan(req *http.Request) { if c.Config.DisableSuspLog { if c.Dev.FullReqLog { r.DumpRequest(req, "") } return } // TODO: Cover more suspicious strings and at a lower layer than this var ch rune var susp bool for _, ch = range req.URL.Path { //char if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) { susp = true break } } // Avoid logging the same request multiple times susp2 := r.susp1(req) if susp && susp2 { r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path") } else if susp { r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path") } else if susp2 { r.SuspiciousRequest(req, "Bad snippet in path") } else if c.Dev.FullReqLog { r.DumpRequest(req, "") } } func isLocalHost(h string) bool { return h == "localhost" || h == "127.0.0.1" || h == "::1" } //var brPool = sync.Pool{} var gzipPool = sync.Pool{} //var uaBufPool = sync.Pool{} func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter { /*if bzw, ok := w.(c.BrResponseWriter); ok { w = bzw.ResponseWriter w.Header().Del("Content-Encoding") } else */if gzw, ok := w.(c.GzipResponseWriter); ok { w = gzw.ResponseWriter w.Header().Del("Content-Encoding") } return w }