a/gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

optimise lang counter

Browse Source 
optimise dumprequest
add more suspicious phrases
capture more router gen errors
This commit is contained in:
Azareal 2020-02-25 14:25:50 +10:00
parent 1ea023bb6d
commit 64335cf1ef
3 changed files with 64 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
common/counters/langs.go
View File

@ -2,6 +2,7 @@ package counters
import (
"database/sql"
"sync/atomic"
c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen"
@ -12,7 +13,7 @@ var LangViewCounter *DefaultLangViewCounter
var langCodes = []string{
"unknown",
"none",
"",
"af",
"ar",
"az",
@ -98,26 +99,22 @@ var langCodes = []string{
}
type DefaultLangViewCounter struct {
buckets []*RWMutexCounterBucket //[OSID]count
//buckets []*MutexCounterBucket //[OSID]count
buckets []int64 //[OSID]count
codesToIndices map[string]int
insert *sql.Stmt
}
func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) {
langBuckets := make([]*RWMutexCounterBucket, len(langCodes))
for bucketID, _ := range langBuckets {
langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
codesToIndices := make(map[string]int, len(langCodes))
for index, code := range langCodes {
codesToIndices[code] = index
}
co := &DefaultLangViewCounter{
buckets: langBuckets,
buckets: make([]int64, len(langCodes)),
codesToIndices: codesToIndices,
insert: acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
insert: acc.Insert("viewchunks_langs").Columns("count,createdAt,lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
c.AddScheduledFifteenMinuteTask(co.Tick)
@ -127,13 +124,8 @@ func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter,
}
func (co *DefaultLangViewCounter) Tick() error {
for id, bucket := range co.buckets {
var count int
bucket.RLock()
count = bucket.counter
bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
bucket.RUnlock()
for id := 0; id < len(co.buckets); id++ {
count := atomic.SwapInt64(&co.buckets[id], 0)
err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
if err != nil {
return errors.Wrap(errors.WithStack(err), "langview counter")
@ -142,11 +134,14 @@ func (co *DefaultLangViewCounter) Tick() error {
return nil
}
func (co *DefaultLangViewCounter) insertChunk(count int, id int) error {
func (co *DefaultLangViewCounter) insertChunk(count int64, id int) error {
if count == 0 {
return nil
}
langCode := langCodes[id]
if langCode == "" {
langCode = "none"
}
c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id)
_, err := co.insert.Exec(count, langCode)
return err
@ -166,9 +161,7 @@ func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
if len(co.buckets) <= id || id < 0 {
return validCode
}
co.buckets[id].Lock()
co.buckets[id].counter++
co.buckets[id].Unlock()
atomic.AddInt64(&co.buckets[id], 1)
return validCode
}

46
gen_router.go
View File

@ -742,10 +742,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht
func (r *GenRouter) Handle(_ string, _ http.Handler) {
}
func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
r.Lock()
defer r.Unlock()
r.extraRoutes[pattern] = handle
r.extraRoutes[pattern] = h
}
func (r *GenRouter) RemoveFunc(pattern string) error {
@ -770,11 +770,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
r.requestLogger.Print(prepend +
"\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" +
"Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads +
"req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"req.RemoteAddr: " + req.RemoteAddr + "\n")
"Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"RemoteAddr: " + req.RemoteAddr + "\n")
}
func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
@ -785,8 +785,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
co.AgentViewCounter.Bump(29)
}
func isLocalHost(host string) bool {
return host=="localhost" || host=="127.0.0.1" || host=="::1"
func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1"
}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
@ -943,10 +943,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string
var buffer []byte
var os string
for _, item := range StringToBytes(ua) {
if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item)
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' {
for _, it := range StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 {
if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
@ -972,7 +972,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else {
// TODO: Test this
items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA")
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buffer: ", buffer)
r.requestLogger.Print("UA Buffer String: ", string(buffer))
break
@ -1050,16 +1050,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
break
}
c.DebugDetail("llLang:", llLang)
if llLang == "" {
co.LangViewCounter.Bump("none")
} else {
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
} else {
co.LangViewCounter.Bump("none")
co.LangViewCounter.Bump("")
}
if !c.Config.RefNoTrack {
@ -1114,7 +1110,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//c.StoppedServer("Profile end")
}
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError {
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
var err c.RouteError
switch(prefix) {
case "/overview":
@ -2657,8 +2653,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
return handle(w,req,user)
}
lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route")
} else {
r.DumpRequest(req,"Bad Route")

57
router_gen/main.go
View File

@ -441,10 +441,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht
func (r *GenRouter) Handle(_ string, _ http.Handler) {
}
func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
r.Lock()
defer r.Unlock()
r.extraRoutes[pattern] = handle
r.extraRoutes[pattern] = h
}
func (r *GenRouter) RemoveFunc(pattern string) error {
@ -469,11 +469,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
r.requestLogger.Print(prepend +
"\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" +
"Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads +
"req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"req.RemoteAddr: " + req.RemoteAddr + "\n")
"Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"RemoteAddr: " + req.RemoteAddr + "\n")
}
func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
@ -484,8 +484,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}})
}
func isLocalHost(host string) bool {
return host=="localhost" || host=="127.0.0.1" || host=="::1"
func isLocalHost(h string) bool {
return h=="localhost" || h=="127.0.0.1" || h=="::1"
}
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global
@ -619,6 +619,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.requestLogger.Print("before PreRoute")
}
/*if c.Dev.QuicPort != 0 {
w.Header().Set("Alt-Svc", "quic=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=2592000; v=\"44,43,39\", h3-23=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h3-24=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h2=\":443\"; ma=3600")
}*/
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
@ -638,10 +642,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string
var buffer []byte
var os string
for _, item := range StringToBytes(ua) {
if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item)
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' {
for _, it := range StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 {
if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
@ -667,7 +671,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else {
// TODO: Test this
items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA")
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buffer: ", buffer)
r.requestLogger.Print("UA Buffer String: ", string(buffer))
break
@ -745,16 +749,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
break
}
c.DebugDetail("llLang:", llLang)
if llLang == "" {
co.LangViewCounter.Bump("none")
} else {
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
} else {
co.LangViewCounter.Bump("none")
co.LangViewCounter.Bump("")
}
if !c.Config.RefNoTrack {
@ -809,7 +809,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//c.StoppedServer("Profile end")
}
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError {
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
var err c.RouteError
switch(prefix) {` + out + `
/*case "/sitemaps": // TODO: Count these views
@ -869,8 +869,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
return handle(w,req,user)
}
lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route")
} else {
r.DumpRequest(req,"Bad Route")
@ -891,7 +891,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
log.Println("Successfully generated the router")
}
func writeFile(name string, content string) {
func writeFile(name, content string) {
f, err := os.Create(name)
if err != nil {
log.Fatal(err)
@ -900,6 +900,9 @@ func writeFile(name string, content string) {
if err != nil {
log.Fatal(err)
}
f.Sync()
err = f.Sync()
if err != nil {
log.Fatal(err)
}
f.Close()
}