rotate the suspicious req logs

reduce the number of bytes used by each req dump and don't log raw query when it is blank
reduce the number of double suspicious req logs for the same req
reduce the number of strings contains scans in the router
split the constant parts of gen_router.go into router.go
reduce boilerplate in a few spots
This commit is contained in:
Azareal 2021-03-23 19:57:50 +10:00
parent db6bccb7ec
commit 20a35fc211
3 changed files with 333 additions and 403 deletions

View File

@ -3,15 +3,12 @@
package main package main
import ( import (
"log"
"strings" "strings"
//"bytes" //"bytes"
"strconv" "strconv"
"compress/gzip" "compress/gzip"
"sync"
"sync/atomic" "sync/atomic"
"errors" "errors"
"os"
"net/http" "net/http"
"time" "time"
@ -910,24 +907,6 @@ func init() {
} }
} }
type WriterIntercept struct {
http.ResponseWriter
}
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
return &WriterIntercept{w}
}
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
func (wi *WriterIntercept) WriteHeader(code int) {
if code == 200 {
h := wi.ResponseWriter.Header()
h.Set("Cache-Control", wiMaxAge)
h.Set("Vary", "Accept-Encoding")
}
wi.ResponseWriter.WriteHeader(code)
}
// HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS
type HTTPSRedirect struct {} type HTTPSRedirect struct {}
@ -938,116 +917,6 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, dest, http.StatusTemporaryRedirect) http.Redirect(w, req, dest, http.StatusTemporaryRedirect)
} }
type GenRouter struct {
UploadHandler func(http.ResponseWriter, *http.Request)
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
requestLogger *log.Logger
suspReqLogger *log.Logger
sync.RWMutex
}
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
return &GenRouter{
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
},
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags),
suspReqLogger: log.New(f2, "", log.LstdFlags),
}, nil
}
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
if err.Handled() {
return
}
if err.Type() == "system" {
c.InternalErrorJSQ(err, w, req, err.JSON())
return
}
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
}
func (r *GenRouter) Handle(_ string, _ http.Handler) {
}
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
r.Lock()
defer r.Unlock()
r.extraRoutes[pattern] = h
}
func (r *GenRouter) RemoveFunc(pattern string) error {
r.Lock()
defer r.Unlock()
_, ok := r.extraRoutes[pattern]
if !ok {
return ErrNoRoute
}
delete(r.extraRoutes, pattern)
return nil
}
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) {
var sb strings.Builder
r.ddumpRequest(req,pre,log,&sb)
}
// TODO: Some of these sanitisations may be redundant
var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3
var dumpReqLen2 = len("\nHead : ") + 2
func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) {
nfield := func(label, val string) {
sb.WriteString(label)
sb.WriteString(val)
}
field := func(label, val string) {
nfield(label,c.SanitiseSingleLine(val))
}
ua := req.UserAgent()
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
sb.WriteString(pre)
field("\nUA: ",ua)
field("\nMethod: ",req.Method)
for key, value := range req.Header {
// Avoid logging this for security reasons
if key == "Cookie" {
continue
}
for _, vvalue := range value {
sb.WriteString("\nHead ")
sb.WriteString(c.SanitiseSingleLine(key))
sb.WriteString(": ")
sb.WriteString(c.SanitiseSingleLine(vvalue))
}
}
field("\nHost: ",req.Host)
field("\nURL.Path: ",req.URL.Path)
field("\nURL.RawQuery: ",req.URL.RawQuery)
ref := req.Referer()
if ref != "" {
field("\nRef: ",req.Referer())
}
nfield("\nIP: ",req.RemoteAddr)
sb.WriteString("\n")
log.Print(sb.String())
}
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
r.dumpRequest(req,pre,r.requestLogger)
}
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
var sb strings.Builder var sb strings.Builder
if pre != "" { if pre != "" {
@ -1055,32 +924,10 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
} else { } else {
pre = "Suspicious Request" pre = "Suspicious Request"
} }
r.ddumpRequest(req,pre,r.suspReqLogger,&sb) r.ddumpRequest(req,pre,r.suspLog,&sb)
co.AgentViewCounter.Bump(43) co.AgentViewCounter.Bump(43)
} }
func (r *GenRouter) unknownUA(req *http.Request) {
if c.Dev.DebugMode {
var presb strings.Builder
presb.WriteString("Unknown UA: ")
for _, ch := range req.UserAgent() {
presb.WriteString(strconv.Itoa(int(ch)))
presb.WriteRune(' ')
}
r.ddumpRequest(req, "", r.requestLogger, &presb)
} else {
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
}
}
func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1"
}
//var brPool = sync.Pool{}
var gzipPool = sync.Pool{}
//var uaBufPool = sync.Pool{}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
// TODO: SetDefaultPath // TODO: SetDefaultPath
// TODO: GetDefaultPath // TODO: GetDefaultPath
@ -1126,11 +973,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) {
// TODO: Abstract the redirect logic? // TODO: Abstract the redirect logic?
w.Header().Set("Connection", "close") w.Header().Set("Connection", "close")
var s string var s, p string
if c.Config.SslSchema { if c.Config.SslSchema {
s = "s" s = "s"
} }
var p string
if c.Site.PortInt != 80 && c.Site.PortInt != 443 { if c.Site.PortInt != 80 && c.Site.PortInt != 443 {
p = ":"+c.Site.Port p = ":"+c.Site.Port
} }
@ -1148,23 +994,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
malformedRequest(3) malformedRequest(3)
return return
} }
if c.Dev.FullReqLog { r.suspScan(req)
r.DumpRequest(req,"")
}
// TODO: Cover more suspicious strings and at a lower layer than this
for _, ch := range req.URL.Path { //char
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path")
break
}
}
lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that
// TODO: Use HasSuffix to avoid over-scanning?
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
r.SuspiciousRequest(req,"Bad snippet in path")
}
// Indirect the default route onto a different one // Indirect the default route onto a different one
if req.URL.Path == "/" { if req.URL.Path == "/" {
@ -1213,7 +1043,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print("before PreRoute") r.reqLogger.Print("before PreRoute")
} }
/*if c.Dev.QuicPort != 0 { /*if c.Dev.QuicPort != 0 {
@ -1276,8 +1106,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buf: ", buf) r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf))
r.requestLogger.Print("UA Buf String: ", string(buf))
break break
} }
} }
@ -1295,11 +1124,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} }
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print("parsed agent: ", agent) r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os)
r.requestLogger.Print("os: ", os) r.reqLogger.Printf("items: %+v\n",items)
r.requestLogger.Printf("items: %+v\n",items)
/*for _, it := range items { /*for _, it := range items {
r.requestLogger.Printf("it: %+v\n",string(it)) r.reqLogger.Printf("it: %+v\n",string(it))
}*/ }*/
} }
@ -1390,7 +1218,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
user := &ucpy user := &ucpy
user.LastAgent = agent user.LastAgent = agent
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print( r.reqLogger.Print(
"after PreRoute\n" + "after PreRoute\n" +
"routeMapEnum: ", routeMapEnum) "routeMapEnum: ", routeMapEnum)
} }
@ -2767,12 +2595,12 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3(150, cn) co.RouteViewCounter.Bump3(150, cn)
} }
case "/profile": case "/profile":
err = c.NoSessionMismatch(w,req,user) err = c.MemberOnly(w,req,user)
if err != nil { if err != nil {
return err return err
} }
err = c.MemberOnly(w,req,user) err = c.NoSessionMismatch(w,req,user)
if err != nil { if err != nil {
return err return err
} }
@ -2926,8 +2754,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
case "favicon.ico": case "favicon.ico":
w = r.responseWriter(w) w = r.responseWriter(w)
req.URL.Path = "/s/favicon.ico" req.URL.Path = "/s/favicon.ico"
routes.StaticFile(w,req)
co.RouteViewCounter.Bump3(173, cn) co.RouteViewCounter.Bump3(173, cn)
routes.StaticFile(w,req)
return nil return nil
case "opensearch.xml": case "opensearch.xml":
co.RouteViewCounter.Bump3(172, cn) co.RouteViewCounter.Bump3(172, cn)
@ -2953,7 +2781,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3(174, cn) co.RouteViewCounter.Bump3(174, cn)
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { if strings.Contains(lp,"w") {
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req)
}
}
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req) return c.MicroNotFound(w,req)
} }
@ -2968,14 +2802,3 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
} }
return err return err
} }
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
/*if bzw, ok := w.(c.BrResponseWriter); ok {
w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter
w.Header().Del("Content-Encoding")
}
return w
}

286
router.go
View File

@ -0,0 +1,286 @@
// Now home to the parts of gen_router.go which aren't expected to change from generation to generation
package main
import (
"log"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
c "github.com/Azareal/Gosora/common"
)
type WriterIntercept struct {
http.ResponseWriter
}
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
return &WriterIntercept{w}
}
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
func (wi *WriterIntercept) WriteHeader(code int) {
if code == 200 {
h := wi.ResponseWriter.Header()
h.Set("Cache-Control", wiMaxAge)
h.Set("Vary", "Accept-Encoding")
}
wi.ResponseWriter.WriteHeader(code)
}
type GenRouter struct {
UploadHandler func(http.ResponseWriter, *http.Request)
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
reqLogger *log.Logger
//suspReqLogger *log.Logger
reqLog2 *RouterLog
suspLog *RouterLog
sync.RWMutex
}
type RouterLogLog struct {
File *os.File
Log *log.Logger
}
type RouterLog struct {
FileVal atomic.Value
LogVal atomic.Value
sync.RWMutex
}
func (r *GenRouter) DailyTick() error {
r.suspLog.Lock()
defer r.suspLog.Unlock()
f := r.suspLog.FileVal.Load().(*os.File)
stat, err := f.Stat()
if err != nil {
return nil
}
if stat.Size() < int64(c.Megabyte) {
return nil
}
if err = f.Close(); err != nil {
return err
}
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return err
}
l := log.New(f, "", log.LstdFlags)
r.suspLog.FileVal.Store(f)
r.suspLog.LogVal.Store(l)
return nil
}
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
createLog := func(name, stimestr string) (*RouterLog, error) {
f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
l := log.New(f, "", log.LstdFlags)
var aVal atomic.Value
var aVal2 atomic.Value
aVal.Store(f)
aVal2.Store(l)
return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil
}
reqLog, err := createLog("reqs", stimestr)
if err != nil {
return nil, err
}
suspReqLog, err := createLog("reqs-susp", stimestr)
if err != nil {
return nil, err
}
f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
reqMiscLog := log.New(f3, "", log.LstdFlags)
return &GenRouter{
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/", uploads).ServeHTTP(writ, r)
},
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
reqLogger: reqMiscLog,
reqLog2: reqLog,
suspLog: suspReqLog,
}, nil
}
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
if err.Handled() {
return
}
if err.Type() == "system" {
c.InternalErrorJSQ(err, w, req, err.JSON())
return
}
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
}
func (r *GenRouter) Handle(_ string, _ http.Handler) {
}
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
r.Lock()
defer r.Unlock()
r.extraRoutes[pattern] = h
}
func (r *GenRouter) RemoveFunc(pattern string) error {
r.Lock()
defer r.Unlock()
_, ok := r.extraRoutes[pattern]
if !ok {
return ErrNoRoute
}
delete(r.extraRoutes, pattern)
return nil
}
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) {
var sb strings.Builder
r.ddumpRequest(req, pre, log, &sb)
}
// TODO: Some of these sanitisations may be redundant
var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7
var dumpReqLen2 = len("\nHead : ") + 2
func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) {
nfield := func(label, val string) {
sb.WriteString(label)
sb.WriteString(val)
}
field := func(label, val string) {
nfield(label, c.SanitiseSingleLine(val))
}
ua := req.UserAgent()
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
sb.WriteString(pre)
sb.WriteString("\n")
sb.WriteString(c.SanitiseSingleLine(req.Method))
sb.WriteRune(' ')
sb.WriteString(c.SanitiseSingleLine(req.URL.Path))
field("\nUA: ", ua)
for key, val := range req.Header {
// Avoid logging this for security reasons
if key == "Cookie" {
continue
}
for _, vvalue := range val {
sb.WriteString("\nHead ")
sb.WriteString(c.SanitiseSingleLine(key))
sb.WriteString(": ")
sb.WriteString(c.SanitiseSingleLine(vvalue))
}
}
field("\nHost: ", req.Host)
if rawQuery := req.URL.RawQuery; rawQuery != "" {
field("\nURL.RawQuery: ", rawQuery)
}
if ref := req.Referer(); ref != "" {
field("\nRef: ", ref)
}
nfield("\nIP: ", req.RemoteAddr)
sb.WriteString("\n")
l.RLock()
l.LogVal.Load().(*log.Logger).Print(sb.String())
l.RUnlock()
}
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
r.dumpRequest(req, pre, r.reqLog2)
}
func (r *GenRouter) unknownUA(req *http.Request) {
if c.Dev.DebugMode {
var presb strings.Builder
presb.WriteString("Unknown UA: ")
for _, ch := range req.UserAgent() {
presb.WriteString(strconv.Itoa(int(ch)))
presb.WriteRune(' ')
}
r.ddumpRequest(req, "", r.reqLog2, &presb)
} else {
r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
}
}
func (r *GenRouter) susp1(req *http.Request) bool {
if !strings.Contains(req.URL.Path, ".") {
return false
}
if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ {
return true
}
lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that
// TODO: Use HasSuffix to avoid over-scanning?
return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action
}
func (r *GenRouter) suspScan(req *http.Request) {
// TODO: Cover more suspicious strings and at a lower layer than this
var ch rune
var susp bool
for _, ch = range req.URL.Path { //char
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
susp = true
break
}
}
// Avoid logging the same request multiple times
susp2 := r.susp1(req)
if susp && susp2 {
r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path")
} else if susp {
r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path")
} else if susp2 {
r.SuspiciousRequest(req, "Bad snippet in path")
} else if c.Dev.FullReqLog {
r.DumpRequest(req, "")
}
}
func isLocalHost(h string) bool {
return h == "localhost" || h == "127.0.0.1" || h == "::1"
}
//var brPool = sync.Pool{}
var gzipPool = sync.Pool{}
//var uaBufPool = sync.Pool{}
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
/*if bzw, ok := w.(c.BrResponseWriter); ok {
w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter
w.Header().Del("Content-Encoding")
}
return w
}

View File

@ -419,15 +419,12 @@ func main() {
package main package main
import ( import (
"log"
"strings" "strings"
//"bytes" //"bytes"
"strconv" "strconv"
"compress/gzip" "compress/gzip"
"sync"
"sync/atomic" "sync/atomic"
"errors" "errors"
"os"
"net/http" "net/http"
"time" "time"
@ -505,24 +502,6 @@ func init() {
} }
} }
type WriterIntercept struct {
http.ResponseWriter
}
func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
return &WriterIntercept{w}
}
var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
func (wi *WriterIntercept) WriteHeader(code int) {
if code == 200 {
h := wi.ResponseWriter.Header()
h.Set("Cache-Control", wiMaxAge)
h.Set("Vary", "Accept-Encoding")
}
wi.ResponseWriter.WriteHeader(code)
}
// HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS
type HTTPSRedirect struct {} type HTTPSRedirect struct {}
@ -533,149 +512,17 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, dest, http.StatusTemporaryRedirect) http.Redirect(w, req, dest, http.StatusTemporaryRedirect)
} }
type GenRouter struct {
UploadHandler func(http.ResponseWriter, *http.Request)
extraRoutes map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
requestLogger *log.Logger
suspReqLogger *log.Logger
sync.RWMutex
}
func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
f, err := os.OpenFile("./logs/reqs-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
f2, err := os.OpenFile("./logs/reqs-susp-"+strconv.FormatInt(c.StartTime.Unix(),10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
return &GenRouter{
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
writ := NewWriterIntercept(w)
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
},
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
requestLogger: log.New(f, "", log.LstdFlags),
suspReqLogger: log.New(f2, "", log.LstdFlags),
}, nil
}
func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
if err.Handled() {
return
}
if err.Type() == "system" {
c.InternalErrorJSQ(err, w, req, err.JSON())
return
}
c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
}
func (r *GenRouter) Handle(_ string, _ http.Handler) {
}
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
r.Lock()
defer r.Unlock()
r.extraRoutes[pattern] = h
}
func (r *GenRouter) RemoveFunc(pattern string) error {
r.Lock()
defer r.Unlock()
_, ok := r.extraRoutes[pattern]
if !ok {
return ErrNoRoute
}
delete(r.extraRoutes, pattern)
return nil
}
func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *log.Logger) {
var sb strings.Builder
r.ddumpRequest(req,pre,log,&sb)
}
// TODO: Some of these sanitisations may be redundant
var dumpReqLen = len("\nUA: \nMethod: \nHost: \nURL.Path: \nURL.RawQuery: \nIP: \n") + 3
var dumpReqLen2 = len("\nHead : ") + 2
func (r *GenRouter) ddumpRequest(req *http.Request, pre string,log *log.Logger, sb *strings.Builder) {
nfield := func(label, val string) {
sb.WriteString(label)
sb.WriteString(val)
}
field := func(label, val string) {
nfield(label,c.SanitiseSingleLine(val))
}
ua := req.UserAgent()
sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
sb.WriteString(pre)
field("\nUA: ",ua)
field("\nMethod: ",req.Method)
for key, value := range req.Header {
// Avoid logging this for security reasons
if key == "Cookie" {
continue
}
for _, vvalue := range value {
sb.WriteString("\nHead ")
sb.WriteString(c.SanitiseSingleLine(key))
sb.WriteString(": ")
sb.WriteString(c.SanitiseSingleLine(vvalue))
}
}
field("\nHost: ",req.Host)
field("\nURL.Path: ",req.URL.Path)
field("\nURL.RawQuery: ",req.URL.RawQuery)
ref := req.Referer()
if ref != "" {
field("\nRef: ",req.Referer())
}
nfield("\nIP: ",req.RemoteAddr)
sb.WriteString("\n")
log.Print(sb.String())
}
func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
r.dumpRequest(req,pre,r.requestLogger)
}
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
var sb strings.Builder var sb strings.Builder
if pre != "" { if pre != "" {
sb.WriteString("Suspicious Request") sb.WriteString("Suspicious Request\n")
} else { } else {
pre = "Suspicious Request" pre = "Suspicious Request"
} }
r.ddumpRequest(req,pre,r.suspReqLogger,&sb) r.ddumpRequest(req,pre,r.suspLog,&sb)
co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}})
} }
func (r *GenRouter) unknownUA(req *http.Request) {
if c.Dev.DebugMode {
var presb strings.Builder
presb.WriteString("Unknown UA: ")
for _, ch := range req.UserAgent() {
presb.WriteString(strconv.Itoa(int(ch)))
presb.WriteRune(' ')
}
r.ddumpRequest(req, "", r.requestLogger, &presb)
} else {
r.requestLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
}
}
func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1"
}
//var brPool = sync.Pool{}
var gzipPool = sync.Pool{}
//var uaBufPool = sync.Pool{}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
// TODO: SetDefaultPath // TODO: SetDefaultPath
// TODO: GetDefaultPath // TODO: GetDefaultPath
@ -721,11 +568,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) {
// TODO: Abstract the redirect logic? // TODO: Abstract the redirect logic?
w.Header().Set("Connection", "close") w.Header().Set("Connection", "close")
var s string var s, p string
if c.Config.SslSchema { if c.Config.SslSchema {
s = "s" s = "s"
} }
var p string
if c.Site.PortInt != 80 && c.Site.PortInt != 443 { if c.Site.PortInt != 80 && c.Site.PortInt != 443 {
p = ":"+c.Site.Port p = ":"+c.Site.Port
} }
@ -743,23 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
malformedRequest(3) malformedRequest(3)
return return
} }
if c.Dev.FullReqLog { r.suspScan(req)
r.DumpRequest(req,"")
}
// TODO: Cover more suspicious strings and at a lower layer than this
for _, ch := range req.URL.Path { //char
if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
r.SuspiciousRequest(req,"Bad char '"+string(ch)+"' in path")
break
}
}
lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that
// TODO: Use HasSuffix to avoid over-scanning?
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".act") { //.action
r.SuspiciousRequest(req,"Bad snippet in path")
}
// Indirect the default route onto a different one // Indirect the default route onto a different one
if req.URL.Path == "/" { if req.URL.Path == "/" {
@ -808,7 +638,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print("before PreRoute") r.reqLogger.Print("before PreRoute")
} }
/*if c.Dev.QuicPort != 0 { /*if c.Dev.QuicPort != 0 {
@ -871,8 +701,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buf: ", buf) r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf))
r.requestLogger.Print("UA Buf String: ", string(buf))
break break
} }
} }
@ -890,11 +719,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} }
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print("parsed agent: ", agent) r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os)
r.requestLogger.Print("os: ", os) r.reqLogger.Printf("items: %+v\n",items)
r.requestLogger.Printf("items: %+v\n",items)
/*for _, it := range items { /*for _, it := range items {
r.requestLogger.Printf("it: %+v\n",string(it)) r.reqLogger.Printf("it: %+v\n",string(it))
}*/ }*/
} }
@ -985,7 +813,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
user := &ucpy user := &ucpy
user.LastAgent = agent user.LastAgent = agent
if c.Dev.SuperDebug { if c.Dev.SuperDebug {
r.requestLogger.Print( r.reqLogger.Print(
"after PreRoute\n" + "after PreRoute\n" +
"routeMapEnum: ", routeMapEnum) "routeMapEnum: ", routeMapEnum)
} }
@ -1096,8 +924,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
case "favicon.ico": case "favicon.ico":
w = r.responseWriter(w) w = r.responseWriter(w)
req.URL.Path = "/s/favicon.ico" req.URL.Path = "/s/favicon.ico"
routes.StaticFile(w,req)
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn)
routes.StaticFile(w,req)
return nil return nil
case "opensearch.xml": case "opensearch.xml":
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn)
@ -1123,7 +951,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn)
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { if strings.Contains(lp,"w") {
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req)
}
}
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req) return c.MicroNotFound(w,req)
} }
@ -1138,17 +972,6 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
} }
return err return err
} }
func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
/*if bzw, ok := w.(c.BrResponseWriter); ok {
w = bzw.ResponseWriter
w.Header().Del("Content-Encoding")
} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
w = gzw.ResponseWriter
w.Header().Del("Content-Encoding")
}
return w
}
` `
tmpl := template.Must(template.New("router").Parse(fileData)) tmpl := template.Must(template.New("router").Parse(fileData))
var b bytes.Buffer var b bytes.Buffer
@ -1161,20 +984,18 @@ func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
} }
func writeFile(name, content string) { func writeFile(name, content string) {
f, err := os.Create(name) f, e := os.Create(name)
if err != nil { if e != nil {
log.Fatal(err) log.Fatal(e)
} }
_, err = f.WriteString(content) _, e = f.WriteString(content)
if err != nil { if e != nil {
log.Fatal(err) log.Fatal(e)
} }
err = f.Sync() if e = f.Sync(); e != nil {
if err != nil { log.Fatal(e)
log.Fatal(err)
} }
err = f.Close() if e = f.Close(); e != nil {
if err != nil { log.Fatal(e)
log.Fatal(err)
} }
} }