rotate the suspicious req logs
reduce the number of bytes used by each req dump and don't log raw query when it is blank reduce the number of double suspicious req logs for the same req reduce the number of strings contains scans in the router split the constant parts of gen_router.go into router.go reduce boilerplate in a few spots
This commit is contained in:
Azareal
parent
db6bccb7ec
commit
20a35fc211
215
gen_router.go
215
gen_router.go
|
|
@ -3,15 +3,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
//"bytes"
|
||||
"strconv"
|
||||
"compress/gzip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"errors"
|
||||
"os"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
|
|
@ -910,24 +907,6 @@ func init() {
|
|||
}
|
||||
}
|
||||
|
||||
type WriterIntercept struct {
|
||||
http.ResponseWriter
|
||||
}
|
||||
|
||||
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
|
||||
return &WriterIntercept{w}
|
||||
}
|
||||
|
||||
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
|
||||
func (wi *WriterIntercept) WriteHeader(code int) {
|
||||
if code == 200 {
|
||||
h := wi.ResponseWriter.Header()
|
||||
h.Set("Cache-Control", wiMaxAge)
|
||||
h.Set("Vary", "Accept-Encoding")
|
||||
}
|
||||
wi.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS
|
||||
type HTTPSRedirect struct {}
|
||||
|
||||
|
|
@ -938,116 +917,6 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
http.Redirect(w, req, dest, http.StatusTemporaryRedirect)
|
||||
}
|
||||
|
||||
type GenRouter struct {
|
||||
UploadHandler func(http.ResponseWriter, *http.Request)
|
||||
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
|
||||
requestLogger *log.Logger
|
||||
suspReqLogger *log.Logger
|
||||
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
|
||||
f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GenRouter{
|
||||
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
|
||||
writ := NewWriterIntercept(w)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
|
||||
},
|
||||
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
|
||||
requestLogger: log.New(f, "", log.LstdFlags),
|
||||
suspReqLogger: log.New(f2, "", log.LstdFlags),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
|
||||
if err.Handled() {
|
||||
return
|
||||
}
|
||||
if err.Type() == "system" {
|
||||
c.InternalErrorJSQ(err, w, req, err.JSON())
|
||||
return
|
||||
}
|
||||
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
|
||||
}
|
||||
|
||||
func (r *GenRouter) Handle(_ string, _ http.Handler) {
|
||||
}
|
||||
|
||||
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.extraRoutes[pattern] = h
|
||||
}
|
||||
|
||||
func (r *GenRouter) RemoveFunc(pattern string) error {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
_, ok := r.extraRoutes[pattern]
|
||||
if !ok {
|
||||
return ErrNoRoute
|
||||
}
|
||||
delete(r.extraRoutes, pattern)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) {
|
||||
var sb strings.Builder
|
||||
r.ddumpRequest(req,pre,log,&sb)
|
||||
}
|
||||
|
||||
// TODO: Some of these sanitisations may be redundant
|
||||
var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3
|
||||
var dumpReqLen2 = len("\nHead : ") + 2
|
||||
func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) {
|
||||
nfield := func(label, val string) {
|
||||
sb.WriteString(label)
|
||||
sb.WriteString(val)
|
||||
}
|
||||
field := func(label, val string) {
|
||||
nfield(label,c.SanitiseSingleLine(val))
|
||||
}
|
||||
ua := req.UserAgent()
|
||||
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
|
||||
sb.WriteString(pre)
|
||||
field("\nUA: ",ua)
|
||||
field("\nMethod: ",req.Method)
|
||||
for key, value := range req.Header {
|
||||
// Avoid logging this for security reasons
|
||||
if key == "Cookie" {
|
||||
continue
|
||||
}
|
||||
for _, vvalue := range value {
|
||||
sb.WriteString("\nHead ")
|
||||
sb.WriteString(c.SanitiseSingleLine(key))
|
||||
sb.WriteString(": ")
|
||||
sb.WriteString(c.SanitiseSingleLine(vvalue))
|
||||
}
|
||||
}
|
||||
field("\nHost: ",req.Host)
|
||||
field("\nURL.Path: ",req.URL.Path)
|
||||
field("\nURL.RawQuery: ",req.URL.RawQuery)
|
||||
ref := req.Referer()
|
||||
if ref != "" {
|
||||
field("\nRef: ",req.Referer())
|
||||
}
|
||||
nfield("\nIP: ",req.RemoteAddr)
|
||||
sb.WriteString("\n")
|
||||
|
||||
log.Print(sb.String())
|
||||
}
|
||||
|
||||
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
|
||||
r.dumpRequest(req,pre,r.requestLogger)
|
||||
}
|
||||
|
||||
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
|
||||
var sb strings.Builder
|
||||
if pre != "" {
|
||||
|
|
@ -1055,32 +924,10 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
|
|||
} else {
|
||||
pre = "Suspicious Request"
|
||||
}
|
||||
r.ddumpRequest(req,pre,r.suspReqLogger,&sb)
|
||||
r.ddumpRequest(req,pre,r.suspLog,&sb)
|
||||
co.AgentViewCounter.Bump(43)
|
||||
}
|
||||
|
||||
func (r *GenRouter) unknownUA(req *http.Request) {
|
||||
if c.Dev.DebugMode {
|
||||
var presb strings.Builder
|
||||
presb.WriteString("Unknown UA: ")
|
||||
for _, ch := range req.UserAgent() {
|
||||
presb.WriteString(strconv.Itoa(int(ch)))
|
||||
presb.WriteRune(' ')
|
||||
}
|
||||
r.ddumpRequest(req, "", r.requestLogger, &presb)
|
||||
} else {
|
||||
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
|
||||
}
|
||||
}
|
||||
|
||||
func isLocalHost(h string) bool {
|
||||
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
||||
}
|
||||
|
||||
//var brPool = sync.Pool{}
|
||||
var gzipPool = sync.Pool{}
|
||||
//var uaBufPool = sync.Pool{}
|
||||
|
||||
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
||||
// TODO: SetDefaultPath
|
||||
// TODO: GetDefaultPath
|
||||
|
|
@ -1126,11 +973,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) {
|
||||
// TODO: Abstract the redirect logic?
|
||||
w.Header().Set("Connection", "close")
|
||||
var s string
|
||||
var s, p string
|
||||
if c.Config.SslSchema {
|
||||
s = "s"
|
||||
}
|
||||
var p string
|
||||
if c.Site.PortInt != 80 && c.Site.PortInt != 443 {
|
||||
p = ":"+c.Site.Port
|
||||
}
|
||||
|
|
@ -1148,23 +994,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
malformedRequest(3)
|
||||
return
|
||||
}
|
||||
if c.Dev.FullReqLog {
|
||||
r.DumpRequest(req,"")
|
||||
}
|
||||
|
||||
// TODO: Cover more suspicious strings and at a lower layer than this
|
||||
for _, ch := range req.URL.Path { //char
|
||||
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
|
||||
r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path")
|
||||
break
|
||||
}
|
||||
}
|
||||
lp := strings.ToLower(req.URL.Path)
|
||||
// TODO: Flag any requests which has a dot with anything but a number after that
|
||||
// TODO: Use HasSuffix to avoid over-scanning?
|
||||
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
|
||||
r.SuspiciousRequest(req,"Bad snippet in path")
|
||||
}
|
||||
r.suspScan(req)
|
||||
|
||||
// Indirect the default route onto a different one
|
||||
if req.URL.Path == "/" {
|
||||
|
|
@ -1213,7 +1043,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
return
|
||||
}
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print("before PreRoute")
|
||||
r.reqLogger.Print("before PreRoute")
|
||||
}
|
||||
|
||||
/*if c.Dev.QuicPort != 0 {
|
||||
|
|
@ -1276,8 +1106,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
// TODO: Test this
|
||||
items = items[:0]
|
||||
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
||||
r.requestLogger.Print("UA Buf: ", buf)
|
||||
r.requestLogger.Print("UA Buf String: ", string(buf))
|
||||
r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -1295,11 +1124,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
}
|
||||
}
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print("parsed agent: ", agent)
|
||||
r.requestLogger.Print("os: ", os)
|
||||
r.requestLogger.Printf("items: %+v\n",items)
|
||||
r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os)
|
||||
r.reqLogger.Printf("items: %+v\n",items)
|
||||
/*for _, it := range items {
|
||||
r.requestLogger.Printf("it: %+v\n",string(it))
|
||||
r.reqLogger.Printf("it: %+v\n",string(it))
|
||||
}*/
|
||||
}
|
||||
|
||||
|
|
@ -1390,7 +1218,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
user := &ucpy
|
||||
user.LastAgent = agent
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print(
|
||||
r.reqLogger.Print(
|
||||
"after PreRoute\n" +
|
||||
"routeMapEnum: ", routeMapEnum)
|
||||
}
|
||||
|
|
@ -2767,12 +2595,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
co.RouteViewCounter.Bump3(150, cn)
|
||||
}
|
||||
case "/profile":
|
||||
err = c.NoSessionMismatch(w,req,user)
|
||||
err = c.MemberOnly(w,req,user)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.MemberOnly(w,req,user)
|
||||
err = c.NoSessionMismatch(w,req,user)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -2926,8 +2754,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
case "favicon.ico":
|
||||
w = r.responseWriter(w)
|
||||
req.URL.Path = "/s/favicon.ico"
|
||||
routes.StaticFile(w,req)
|
||||
co.RouteViewCounter.Bump3(173, cn)
|
||||
routes.StaticFile(w,req)
|
||||
return nil
|
||||
case "opensearch.xml":
|
||||
co.RouteViewCounter.Bump3(172, cn)
|
||||
|
|
@ -2953,7 +2781,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
co.RouteViewCounter.Bump3(174, cn)
|
||||
|
||||
lp := strings.ToLower(req.URL.Path)
|
||||
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
|
||||
if strings.Contains(lp,"w") {
|
||||
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
|
||||
r.SuspiciousRequest(req,"Bad Route")
|
||||
return c.MicroNotFound(w,req)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
|
||||
r.SuspiciousRequest(req,"Bad Route")
|
||||
return c.MicroNotFound(w,req)
|
||||
}
|
||||
|
|
@ -2968,14 +2802,3 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
|
||||
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||
w = bzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||
w = gzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
|
|
|||
286
router.go
286
router.go
|
|
@ -0,0 +1,286 @@
|
|||
// Now home to the parts of gen_router.go which aren't expected to change from generation to generation
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
c "github.com/Azareal/Gosora/common"
|
||||
)
|
||||
|
||||
type WriterIntercept struct {
|
||||
http.ResponseWriter
|
||||
}
|
||||
|
||||
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
|
||||
return &WriterIntercept{w}
|
||||
}
|
||||
|
||||
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
|
||||
|
||||
func (wi *WriterIntercept) WriteHeader(code int) {
|
||||
if code == 200 {
|
||||
h := wi.ResponseWriter.Header()
|
||||
h.Set("Cache-Control", wiMaxAge)
|
||||
h.Set("Vary", "Accept-Encoding")
|
||||
}
|
||||
wi.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
type GenRouter struct {
|
||||
UploadHandler func(http.ResponseWriter, *http.Request)
|
||||
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
|
||||
|
||||
reqLogger *log.Logger
|
||||
//suspReqLogger *log.Logger
|
||||
|
||||
reqLog2 *RouterLog
|
||||
suspLog *RouterLog
|
||||
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
type RouterLogLog struct {
|
||||
File *os.File
|
||||
Log *log.Logger
|
||||
}
|
||||
type RouterLog struct {
|
||||
FileVal atomic.Value
|
||||
LogVal atomic.Value
|
||||
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
func (r *GenRouter) DailyTick() error {
|
||||
r.suspLog.Lock()
|
||||
defer r.suspLog.Unlock()
|
||||
|
||||
f := r.suspLog.FileVal.Load().(*os.File)
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if stat.Size() < int64(c.Megabyte) {
|
||||
return nil
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
|
||||
f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l := log.New(f, "", log.LstdFlags)
|
||||
r.suspLog.FileVal.Store(f)
|
||||
r.suspLog.LogVal.Store(l)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
|
||||
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
|
||||
createLog := func(name, stimestr string) (*RouterLog, error) {
|
||||
f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l := log.New(f, "", log.LstdFlags)
|
||||
var aVal atomic.Value
|
||||
var aVal2 atomic.Value
|
||||
aVal.Store(f)
|
||||
aVal2.Store(l)
|
||||
return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil
|
||||
}
|
||||
reqLog, err := createLog("reqs", stimestr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suspReqLog, err := createLog("reqs-susp", stimestr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqMiscLog := log.New(f3, "", log.LstdFlags)
|
||||
|
||||
return &GenRouter{
|
||||
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
|
||||
writ := NewWriterIntercept(w)
|
||||
http.StripPrefix("/uploads/", uploads).ServeHTTP(writ, r)
|
||||
},
|
||||
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
|
||||
|
||||
reqLogger: reqMiscLog,
|
||||
reqLog2: reqLog,
|
||||
suspLog: suspReqLog,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
|
||||
if err.Handled() {
|
||||
return
|
||||
}
|
||||
if err.Type() == "system" {
|
||||
c.InternalErrorJSQ(err, w, req, err.JSON())
|
||||
return
|
||||
}
|
||||
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
|
||||
}
|
||||
|
||||
func (r *GenRouter) Handle(_ string, _ http.Handler) {
|
||||
}
|
||||
|
||||
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.extraRoutes[pattern] = h
|
||||
}
|
||||
|
||||
func (r *GenRouter) RemoveFunc(pattern string) error {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
_, ok := r.extraRoutes[pattern]
|
||||
if !ok {
|
||||
return ErrNoRoute
|
||||
}
|
||||
delete(r.extraRoutes, pattern)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) {
|
||||
var sb strings.Builder
|
||||
r.ddumpRequest(req, pre, log, &sb)
|
||||
}
|
||||
|
||||
// TODO: Some of these sanitisations may be redundant
|
||||
var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7
|
||||
var dumpReqLen2 = len("\nHead : ") + 2
|
||||
|
||||
func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) {
|
||||
nfield := func(label, val string) {
|
||||
sb.WriteString(label)
|
||||
sb.WriteString(val)
|
||||
}
|
||||
field := func(label, val string) {
|
||||
nfield(label, c.SanitiseSingleLine(val))
|
||||
}
|
||||
ua := req.UserAgent()
|
||||
|
||||
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
|
||||
sb.WriteString(pre)
|
||||
sb.WriteString("\n")
|
||||
sb.WriteString(c.SanitiseSingleLine(req.Method))
|
||||
sb.WriteRune(' ')
|
||||
sb.WriteString(c.SanitiseSingleLine(req.URL.Path))
|
||||
field("\nUA: ", ua)
|
||||
|
||||
for key, val := range req.Header {
|
||||
// Avoid logging this for security reasons
|
||||
if key == "Cookie" {
|
||||
continue
|
||||
}
|
||||
for _, vvalue := range val {
|
||||
sb.WriteString("\nHead ")
|
||||
sb.WriteString(c.SanitiseSingleLine(key))
|
||||
sb.WriteString(": ")
|
||||
sb.WriteString(c.SanitiseSingleLine(vvalue))
|
||||
}
|
||||
}
|
||||
field("\nHost: ", req.Host)
|
||||
if rawQuery := req.URL.RawQuery; rawQuery != "" {
|
||||
field("\nURL.RawQuery: ", rawQuery)
|
||||
}
|
||||
if ref := req.Referer(); ref != "" {
|
||||
field("\nRef: ", ref)
|
||||
}
|
||||
nfield("\nIP: ", req.RemoteAddr)
|
||||
sb.WriteString("\n")
|
||||
|
||||
l.RLock()
|
||||
l.LogVal.Load().(*log.Logger).Print(sb.String())
|
||||
l.RUnlock()
|
||||
}
|
||||
|
||||
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
|
||||
r.dumpRequest(req, pre, r.reqLog2)
|
||||
}
|
||||
|
||||
func (r *GenRouter) unknownUA(req *http.Request) {
|
||||
if c.Dev.DebugMode {
|
||||
var presb strings.Builder
|
||||
presb.WriteString("Unknown UA: ")
|
||||
for _, ch := range req.UserAgent() {
|
||||
presb.WriteString(strconv.Itoa(int(ch)))
|
||||
presb.WriteRune(' ')
|
||||
}
|
||||
r.ddumpRequest(req, "", r.reqLog2, &presb)
|
||||
} else {
|
||||
r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *GenRouter) susp1(req *http.Request) bool {
|
||||
if !strings.Contains(req.URL.Path, ".") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ {
|
||||
return true
|
||||
}
|
||||
lp := strings.ToLower(req.URL.Path)
|
||||
// TODO: Flag any requests which has a dot with anything but a number after that
|
||||
// TODO: Use HasSuffix to avoid over-scanning?
|
||||
return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action
|
||||
}
|
||||
|
||||
func (r *GenRouter) suspScan(req *http.Request) {
|
||||
// TODO: Cover more suspicious strings and at a lower layer than this
|
||||
var ch rune
|
||||
var susp bool
|
||||
for _, ch = range req.URL.Path { //char
|
||||
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
|
||||
susp = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Avoid logging the same request multiple times
|
||||
susp2 := r.susp1(req)
|
||||
if susp && susp2 {
|
||||
r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path")
|
||||
} else if susp {
|
||||
r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path")
|
||||
} else if susp2 {
|
||||
r.SuspiciousRequest(req, "Bad snippet in path")
|
||||
} else if c.Dev.FullReqLog {
|
||||
r.DumpRequest(req, "")
|
||||
}
|
||||
}
|
||||
|
||||
func isLocalHost(h string) bool {
|
||||
return h == "localhost" || h == "127.0.0.1" || h == "::1"
|
||||
}
|
||||
|
||||
//var brPool = sync.Pool{}
|
||||
var gzipPool = sync.Pool{}
|
||||
|
||||
//var uaBufPool = sync.Pool{}
|
||||
|
||||
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
|
||||
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||
w = bzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||
w = gzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
|
@ -419,15 +419,12 @@ func main() {
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
//"bytes"
|
||||
"strconv"
|
||||
"compress/gzip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"errors"
|
||||
"os"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
|
|
@ -505,24 +502,6 @@ func init() {
|
|||
}
|
||||
}
|
||||
|
||||
type WriterIntercept struct {
|
||||
http.ResponseWriter
|
||||
}
|
||||
|
||||
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
|
||||
return &WriterIntercept{w}
|
||||
}
|
||||
|
||||
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
|
||||
func (wi *WriterIntercept) WriteHeader(code int) {
|
||||
if code == 200 {
|
||||
h := wi.ResponseWriter.Header()
|
||||
h.Set("Cache-Control", wiMaxAge)
|
||||
h.Set("Vary", "Accept-Encoding")
|
||||
}
|
||||
wi.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS
|
||||
type HTTPSRedirect struct {}
|
||||
|
||||
|
|
@ -533,149 +512,17 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
http.Redirect(w, req, dest, http.StatusTemporaryRedirect)
|
||||
}
|
||||
|
||||
type GenRouter struct {
|
||||
UploadHandler func(http.ResponseWriter, *http.Request)
|
||||
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
|
||||
requestLogger *log.Logger
|
||||
suspReqLogger *log.Logger
|
||||
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
|
||||
f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GenRouter{
|
||||
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
|
||||
writ := NewWriterIntercept(w)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
|
||||
},
|
||||
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
|
||||
requestLogger: log.New(f, "", log.LstdFlags),
|
||||
suspReqLogger: log.New(f2, "", log.LstdFlags),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
|
||||
if err.Handled() {
|
||||
return
|
||||
}
|
||||
if err.Type() == "system" {
|
||||
c.InternalErrorJSQ(err, w, req, err.JSON())
|
||||
return
|
||||
}
|
||||
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
|
||||
}
|
||||
|
||||
func (r *GenRouter) Handle(_ string, _ http.Handler) {
|
||||
}
|
||||
|
||||
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.extraRoutes[pattern] = h
|
||||
}
|
||||
|
||||
func (r *GenRouter) RemoveFunc(pattern string) error {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
_, ok := r.extraRoutes[pattern]
|
||||
if !ok {
|
||||
return ErrNoRoute
|
||||
}
|
||||
delete(r.extraRoutes, pattern)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) {
|
||||
var sb strings.Builder
|
||||
r.ddumpRequest(req,pre,log,&sb)
|
||||
}
|
||||
|
||||
// TODO: Some of these sanitisations may be redundant
|
||||
var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3
|
||||
var dumpReqLen2 = len("\nHead : ") + 2
|
||||
func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) {
|
||||
nfield := func(label, val string) {
|
||||
sb.WriteString(label)
|
||||
sb.WriteString(val)
|
||||
}
|
||||
field := func(label, val string) {
|
||||
nfield(label,c.SanitiseSingleLine(val))
|
||||
}
|
||||
ua := req.UserAgent()
|
||||
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
|
||||
sb.WriteString(pre)
|
||||
field("\nUA: ",ua)
|
||||
field("\nMethod: ",req.Method)
|
||||
for key, value := range req.Header {
|
||||
// Avoid logging this for security reasons
|
||||
if key == "Cookie" {
|
||||
continue
|
||||
}
|
||||
for _, vvalue := range value {
|
||||
sb.WriteString("\nHead ")
|
||||
sb.WriteString(c.SanitiseSingleLine(key))
|
||||
sb.WriteString(": ")
|
||||
sb.WriteString(c.SanitiseSingleLine(vvalue))
|
||||
}
|
||||
}
|
||||
field("\nHost: ",req.Host)
|
||||
field("\nURL.Path: ",req.URL.Path)
|
||||
field("\nURL.RawQuery: ",req.URL.RawQuery)
|
||||
ref := req.Referer()
|
||||
if ref != "" {
|
||||
field("\nRef: ",req.Referer())
|
||||
}
|
||||
nfield("\nIP: ",req.RemoteAddr)
|
||||
sb.WriteString("\n")
|
||||
|
||||
log.Print(sb.String())
|
||||
}
|
||||
|
||||
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
|
||||
r.dumpRequest(req,pre,r.requestLogger)
|
||||
}
|
||||
|
||||
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
|
||||
var sb strings.Builder
|
||||
if pre != "" {
|
||||
sb.WriteString("Suspicious Request")
|
||||
sb.WriteString("Suspicious Request\n")
|
||||
} else {
|
||||
pre = "Suspicious Request"
|
||||
}
|
||||
r.ddumpRequest(req,pre,r.suspReqLogger,&sb)
|
||||
r.ddumpRequest(req,pre,r.suspLog,&sb)
|
||||
co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}})
|
||||
}
|
||||
|
||||
func (r *GenRouter) unknownUA(req *http.Request) {
|
||||
if c.Dev.DebugMode {
|
||||
var presb strings.Builder
|
||||
presb.WriteString("Unknown UA: ")
|
||||
for _, ch := range req.UserAgent() {
|
||||
presb.WriteString(strconv.Itoa(int(ch)))
|
||||
presb.WriteRune(' ')
|
||||
}
|
||||
r.ddumpRequest(req, "", r.requestLogger, &presb)
|
||||
} else {
|
||||
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
|
||||
}
|
||||
}
|
||||
|
||||
func isLocalHost(h string) bool {
|
||||
return h=="localhost" || h=="127.0.0.1" || h=="::1"
|
||||
}
|
||||
|
||||
//var brPool = sync.Pool{}
|
||||
var gzipPool = sync.Pool{}
|
||||
//var uaBufPool = sync.Pool{}
|
||||
|
||||
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
|
||||
// TODO: SetDefaultPath
|
||||
// TODO: GetDefaultPath
|
||||
|
|
@ -721,11 +568,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) {
|
||||
// TODO: Abstract the redirect logic?
|
||||
w.Header().Set("Connection", "close")
|
||||
var s string
|
||||
var s, p string
|
||||
if c.Config.SslSchema {
|
||||
s = "s"
|
||||
}
|
||||
var p string
|
||||
if c.Site.PortInt != 80 && c.Site.PortInt != 443 {
|
||||
p = ":"+c.Site.Port
|
||||
}
|
||||
|
|
@ -743,23 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
malformedRequest(3)
|
||||
return
|
||||
}
|
||||
if c.Dev.FullReqLog {
|
||||
r.DumpRequest(req,"")
|
||||
}
|
||||
|
||||
// TODO: Cover more suspicious strings and at a lower layer than this
|
||||
for _, ch := range req.URL.Path { //char
|
||||
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
|
||||
r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path")
|
||||
break
|
||||
}
|
||||
}
|
||||
lp := strings.ToLower(req.URL.Path)
|
||||
// TODO: Flag any requests which has a dot with anything but a number after that
|
||||
// TODO: Use HasSuffix to avoid over-scanning?
|
||||
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
|
||||
r.SuspiciousRequest(req,"Bad snippet in path")
|
||||
}
|
||||
r.suspScan(req)
|
||||
|
||||
// Indirect the default route onto a different one
|
||||
if req.URL.Path == "/" {
|
||||
|
|
@ -808,7 +638,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
return
|
||||
}
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print("before PreRoute")
|
||||
r.reqLogger.Print("before PreRoute")
|
||||
}
|
||||
|
||||
/*if c.Dev.QuicPort != 0 {
|
||||
|
|
@ -871,8 +701,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
// TODO: Test this
|
||||
items = items[:0]
|
||||
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
|
||||
r.requestLogger.Print("UA Buf: ", buf)
|
||||
r.requestLogger.Print("UA Buf String: ", string(buf))
|
||||
r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -890,11 +719,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
}
|
||||
}
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print("parsed agent: ", agent)
|
||||
r.requestLogger.Print("os: ", os)
|
||||
r.requestLogger.Printf("items: %+v\n",items)
|
||||
r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os)
|
||||
r.reqLogger.Printf("items: %+v\n",items)
|
||||
/*for _, it := range items {
|
||||
r.requestLogger.Printf("it: %+v\n",string(it))
|
||||
r.reqLogger.Printf("it: %+v\n",string(it))
|
||||
}*/
|
||||
}
|
||||
|
||||
|
|
@ -985,7 +813,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
user := &ucpy
|
||||
user.LastAgent = agent
|
||||
if c.Dev.SuperDebug {
|
||||
r.requestLogger.Print(
|
||||
r.reqLogger.Print(
|
||||
"after PreRoute\n" +
|
||||
"routeMapEnum: ", routeMapEnum)
|
||||
}
|
||||
|
|
@ -1096,8 +924,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
case "favicon.ico":
|
||||
w = r.responseWriter(w)
|
||||
req.URL.Path = "/s/favicon.ico"
|
||||
routes.StaticFile(w,req)
|
||||
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn)
|
||||
routes.StaticFile(w,req)
|
||||
return nil
|
||||
case "opensearch.xml":
|
||||
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn)
|
||||
|
|
@ -1123,7 +951,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn)
|
||||
|
||||
lp := strings.ToLower(req.URL.Path)
|
||||
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
|
||||
if strings.Contains(lp,"w") {
|
||||
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
|
||||
r.SuspiciousRequest(req,"Bad Route")
|
||||
return c.MicroNotFound(w,req)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
|
||||
r.SuspiciousRequest(req,"Bad Route")
|
||||
return c.MicroNotFound(w,req)
|
||||
}
|
||||
|
|
@ -1138,17 +972,6 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
|
||||
/*if bzw, ok := w.(c.BrResponseWriter); ok {
|
||||
w = bzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
|
||||
w = gzw.ResponseWriter
|
||||
w.Header().Del("Content-Encoding")
|
||||
}
|
||||
return w
|
||||
}
|
||||
`
|
||||
tmpl := template.Must(template.New("router").Parse(fileData))
|
||||
var b bytes.Buffer
|
||||
|
|
@ -1161,20 +984,18 @@ func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
|
|||
}
|
||||
|
||||
func writeFile(name, content string) {
|
||||
f, err := os.Create(name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
f, e := os.Create(name)
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
_, err = f.WriteString(content)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
_, e = f.WriteString(content)
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
err = f.Sync()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if e = f.Sync(); e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if e = f.Close(); e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue