experiment with more route perf tracking

This commit is contained in:
Azareal 2020-02-29 09:11:07 +10:00
parent c403b4a85a
commit 03862d1b09
9 changed files with 399 additions and 167 deletions

View File

@ -27,7 +27,7 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
co := &DefaultForumViewCounter{ co := &DefaultForumViewCounter{
oddMap: make(map[int]*RWMutexCounterBucket), oddMap: make(map[int]*RWMutexCounterBucket),
evenMap: make(map[int]*RWMutexCounterBucket), evenMap: make(map[int]*RWMutexCounterBucket),
insert: acc.Insert("viewchunks_forums").Columns("count, createdAt, forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(), insert: acc.Insert("viewchunks_forums").Columns("count,createdAt,forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
} }
c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
//c.AddScheduledSecondTask(co.Tick) //c.AddScheduledSecondTask(co.Tick)
@ -38,17 +38,17 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
func (co *DefaultForumViewCounter) Tick() error { func (co *DefaultForumViewCounter) Tick() error {
cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error { cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error {
l.RLock() l.RLock()
for forumID, forum := range m { for fid, f := range m {
l.RUnlock() l.RUnlock()
var count int var count int
forum.RLock() f.RLock()
count = forum.counter count = f.counter
forum.RUnlock() f.RUnlock()
// TODO: Only delete the bucket when it's zero to avoid hitting popular forums? // TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
l.Lock() l.Lock()
delete(m, forumID) delete(m, fid)
l.Unlock() l.Unlock()
err := co.insertChunk(count, forumID) err := co.insertChunk(count, fid)
if err != nil { if err != nil {
return errors.Wrap(errors.WithStack(err),"forum counter") return errors.Wrap(errors.WithStack(err),"forum counter")
} }
@ -64,7 +64,7 @@ func (co *DefaultForumViewCounter) Tick() error {
return cLoop(&co.evenLock,co.evenMap) return cLoop(&co.evenLock,co.evenMap)
} }
func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error { func (co *DefaultForumViewCounter) insertChunk(count, forum int) error {
if count == 0 { if count == 0 {
return nil return nil
} }
@ -73,34 +73,34 @@ func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
return err return err
} }
func (co *DefaultForumViewCounter) Bump(forumID int) { func (co *DefaultForumViewCounter) Bump(fid int) {
// Is the ID even? // Is the ID even?
if forumID%2 == 0 { if fid%2 == 0 {
co.evenLock.RLock() co.evenLock.RLock()
forum, ok := co.evenMap[forumID] f, ok := co.evenMap[fid]
co.evenLock.RUnlock() co.evenLock.RUnlock()
if ok { if ok {
forum.Lock() f.Lock()
forum.counter++ f.counter++
forum.Unlock() f.Unlock()
} else { } else {
co.evenLock.Lock() co.evenLock.Lock()
co.evenMap[forumID] = &RWMutexCounterBucket{counter: 1} co.evenMap[fid] = &RWMutexCounterBucket{counter: 1}
co.evenLock.Unlock() co.evenLock.Unlock()
} }
return return
} }
co.oddLock.RLock() co.oddLock.RLock()
forum, ok := co.oddMap[forumID] f, ok := co.oddMap[fid]
co.oddLock.RUnlock() co.oddLock.RUnlock()
if ok { if ok {
forum.Lock() f.Lock()
forum.counter++ f.counter++
forum.Unlock() f.Unlock()
} else { } else {
co.oddLock.Lock() co.oddLock.Lock()
co.oddMap[forumID] = &RWMutexCounterBucket{counter: 1} co.oddMap[fid] = &RWMutexCounterBucket{counter: 1}
co.oddLock.Unlock() co.oddLock.Unlock()
} }
} }

View File

@ -7,6 +7,7 @@ import (
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen" qgen "github.com/Azareal/Gosora/query_gen"
"github.com/Azareal/Gosora/uutils"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -150,6 +151,32 @@ func (co *DefaultRouteViewCounter) Bump2(route int, t time.Time) {
return return
} }
micro := int(time.Since(t).Microseconds()) micro := int(time.Since(t).Microseconds())
//co.PerfCounter.Push(since, true)
b.Lock()
b.counter++
if micro != b.avg {
if b.avg == 0 {
b.avg = micro
} else {
b.avg = (micro + b.avg) / 2
}
}
b.Unlock()
}
// TODO: Eliminate the lock?
func (co *DefaultRouteViewCounter) Bump3(route int, nano int64) {
if c.Config.DisableAnalytics {
return
}
// TODO: Test this check
b := co.buckets[route]
c.DebugDetail("buckets[", route, "]: ", b)
if len(co.buckets) <= route || route < 0 {
return
}
micro := int((uutils.Nanotime() - nano) / 1000)
//co.PerfCounter.Push(since, true)
b.Lock() b.Lock()
b.counter++ b.counter++
if micro != b.avg { if micro != b.avg {

File diff suppressed because it is too large Load Diff

View File

@ -83,6 +83,9 @@ func main() {
vcpy := route.Vars vcpy := route.Vars
route.Vars = []string{"h"} route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...) route.Vars = append(route.Vars, vcpy...)
} else if route.Name != "common.RouteWebsockets" {
//out += "\n\t\t\tsa := time.Now()"
out += "\n\t\t\tcn := uutils.Nanotime()"
} }
out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user" out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
for _, item := range route.Vars { for _, item := range route.Vars {
@ -91,8 +94,10 @@ func main() {
out += `)` out += `)`
if !route.Action && !route.NoHead { if !route.Action && !route.NoHead {
out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)" out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else { } else if route.Name != "common.RouteWebsockets" {
out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")" //out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
//out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", sa)"
out += "\n\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
} }
} }
@ -149,6 +154,8 @@ func main() {
vcpy := route.Vars vcpy := route.Vars
route.Vars = []string{"h"} route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...) route.Vars = append(route.Vars, vcpy...)
} else {
out += "\n\t\t\t\t\tcn := uutils.Nanotime()"
} }
out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user" out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
for _, item := range route.Vars { for _, item := range route.Vars {
@ -158,7 +165,8 @@ func main() {
if !route.Action && !route.NoHead && !group.NoHead { if !route.Action && !route.NoHead && !group.NoHead {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)" out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else { } else {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")" //out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
} }
} }
@ -341,9 +349,11 @@ import (
"errors" "errors"
"os" "os"
"net/http" "net/http"
"time"
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
co "github.com/Azareal/Gosora/common/counters" co "github.com/Azareal/Gosora/common/counters"
"github.com/Azareal/Gosora/uutils"
"github.com/Azareal/Gosora/routes" "github.com/Azareal/Gosora/routes"
"github.com/Azareal/Gosora/routes/panel" "github.com/Azareal/Gosora/routes/panel"
) )
@ -384,6 +394,7 @@ var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNa
// TODO: Stop spilling these into the package scope? // TODO: Stop spilling these into the package scope?
func init() { func init() {
_ = time.Now()
co.SetRouteMapEnum(routeMapEnum) co.SetRouteMapEnum(routeMapEnum)
co.SetReverseRouteMapEnum(reverseRouteMapEnum) co.SetReverseRouteMapEnum(reverseRouteMapEnum)
co.SetAgentMapEnum(agentMapEnum) co.SetAgentMapEnum(agentMapEnum)
@ -666,14 +677,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string var items []string
var buffer []byte var buffer []byte
var os int var os int
for _, it := range StringToBytes(ua) { for _, it := range uutils.StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) { if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, it) buffer = append(buffer, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' { } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 { if len(buffer) != 0 {
if len(buffer) > 2 { if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
switch(BytesToString(buffer)) { switch(uutils.BytesToString(buffer)) {
case "Windows": case "Windows":
os = {{.AllOSMap.windows}} os = {{.AllOSMap.windows}}
case "Linux": case "Linux":

View File

@ -21,7 +21,7 @@ var forumStmts ForumStmts
func init() { func init() {
c.DbInits.Add(func(acc *qgen.Accumulator) error { c.DbInits.Add(func(acc *qgen.Accumulator) error {
forumStmts = ForumStmts{ forumStmts = ForumStmts{
getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID = ?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(), getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(),
} }
return acc.FirstError() return acc.FirstError()
}) })

View File

@ -834,6 +834,9 @@ func AnalyticsRoutesPerf(w http.ResponseWriter, r *http.Request, user c.User) c.
if inEx(ovitem.name) { if inEx(ovitem.name) {
continue continue
} }
if strings.HasPrefix(ovitem.name,"panel.") {
continue
}
var viewList []int64 var viewList []int64
for _, value := range revLabelList { for _, value := range revLabelList {
viewList = append(viewList, ovitem.viewMap[value]) viewList = append(viewList, ovitem.viewMap[value])

View File

@ -1,9 +1,10 @@
package tmpl package tmpl
import ( import (
"reflect" //"reflect"
"runtime" //"runtime"
"unsafe" //"unsafe"
"github.com/Azareal/Gosora/uutils"
) )
var GetFrag = func(name string) [][]byte { var GetFrag = func(name string) [][]byte {
@ -14,6 +15,9 @@ type WriteString interface {
WriteString(s string) (n int, err error) WriteString(s string) (n int, err error)
} }
var StringToBytes = uutils.StringToBytes
/*
func StringToBytes(s string) (bytes []byte) { func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s)) str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes)) slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@ -23,3 +27,4 @@ func StringToBytes(s string) (bytes []byte) {
runtime.KeepAlive(&s) runtime.KeepAlive(&s)
return bytes return bytes
} }
*/

View File

@ -1,11 +1,14 @@
package main package main
import ( import (
"reflect" //"reflect"
"runtime" //"runtime"
"unsafe" //"unsafe"
"github.com/Azareal/Gosora/uutils"
) )
// TODO: Add a safe build mode for things like Google Appengine
var GetFrag = func(name string) [][]byte { var GetFrag = func(name string) [][]byte {
return nil return nil
} }
@ -14,6 +17,11 @@ type WriteString interface {
WriteString(s string) (n int, err error) WriteString(s string) (n int, err error)
} }
var StringToBytes = uutils.StringToBytes
var BytesToString = uutils.BytesToString
var Nanotime = uutils.Nanotime
/*
func StringToBytes(s string) (bytes []byte) { func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s)) str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes)) slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@ -32,3 +40,11 @@ func BytesToString(bytes []byte) (s string) {
runtime.KeepAlive(&bytes) runtime.KeepAlive(&bytes)
return s return s
} }
//go:noescape
//go:linkname nanotime runtime.nanotime
func nanotime() int64
func Nanotime() int64 {
return nanotime()
}*/

36
uutils/utils.go Normal file
View File

@ -0,0 +1,36 @@
package uutils
import (
"reflect"
"runtime"
"unsafe"
)
// TODO: Add a safe build mode for things like Google Appengine
func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
slice.Data = str.Data
slice.Len = str.Len
slice.Cap = str.Len
runtime.KeepAlive(&s)
return bytes
}
func BytesToString(bytes []byte) (s string) {
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
str.Data = slice.Data
str.Len = slice.Len
runtime.KeepAlive(&bytes)
return s
}
//go:noescape
//go:linkname nanotime runtime.nanotime
func nanotime() int64
func Nanotime() int64 {
return nanotime()
}