a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

experiment with more route perf tracking

This commit is contained in:
Azareal 2020-02-29 09:11:07 +10:00
parent c403b4a85a
commit 03862d1b09
9 changed files with 399 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
common/counters/forums.go
View File

@ -27,7 +27,7 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
co := &DefaultForumViewCounter{
oddMap: make(map[int]*RWMutexCounterBucket),
evenMap: make(map[int]*RWMutexCounterBucket),
insert: acc.Insert("viewchunks_forums").Columns("count, createdAt, forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
insert: acc.Insert("viewchunks_forums").Columns("count,createdAt,forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
//c.AddScheduledSecondTask(co.Tick)
@ -38,17 +38,17 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
func (co *DefaultForumViewCounter) Tick() error {
cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error {
l.RLock()
for forumID, forum := range m {
for fid, f := range m {
l.RUnlock()
var count int
forum.RLock()
count = forum.counter
forum.RUnlock()
f.RLock()
count = f.counter
f.RUnlock()
// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
l.Lock()
delete(m, forumID)
delete(m, fid)
l.Unlock()
err := co.insertChunk(count, forumID)
err := co.insertChunk(count, fid)
if err != nil {
return errors.Wrap(errors.WithStack(err),"forum counter")
}
@ -64,7 +64,7 @@ func (co *DefaultForumViewCounter) Tick() error {
return cLoop(&co.evenLock,co.evenMap)
}
func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
func (co *DefaultForumViewCounter) insertChunk(count, forum int) error {
if count == 0 {
return nil
}
@ -73,34 +73,34 @@ func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
return err
}
func (co *DefaultForumViewCounter) Bump(forumID int) {
func (co *DefaultForumViewCounter) Bump(fid int) {
// Is the ID even?
if forumID%2 == 0 {
if fid%2 == 0 {
co.evenLock.RLock()
forum, ok := co.evenMap[forumID]
f, ok := co.evenMap[fid]
co.evenLock.RUnlock()
if ok {
forum.Lock()
forum.counter++
forum.Unlock()
f.Lock()
f.counter++
f.Unlock()
} else {
co.evenLock.Lock()
co.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
co.evenMap[fid] = &RWMutexCounterBucket{counter: 1}
co.evenLock.Unlock()
}
return
}
co.oddLock.RLock()
forum, ok := co.oddMap[forumID]
f, ok := co.oddMap[fid]
co.oddLock.RUnlock()
if ok {
forum.Lock()
forum.counter++
forum.Unlock()
f.Lock()
f.counter++
f.Unlock()
} else {
co.oddLock.Lock()
co.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
co.oddMap[fid] = &RWMutexCounterBucket{counter: 1}
co.oddLock.Unlock()
}
}

27
common/counters/routes.go
View File

@ -7,6 +7,7 @@ import (
c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen"
"github.com/Azareal/Gosora/uutils"
"github.com/pkg/errors"
)
@ -150,6 +151,32 @@ func (co *DefaultRouteViewCounter) Bump2(route int, t time.Time) {
return
}
micro := int(time.Since(t).Microseconds())
//co.PerfCounter.Push(since, true)
b.Lock()
b.counter++
if micro != b.avg {
if b.avg == 0 {
b.avg = micro
} else {
b.avg = (micro + b.avg) / 2
}
}
b.Unlock()
}
// TODO: Eliminate the lock?
func (co *DefaultRouteViewCounter) Bump3(route int, nano int64) {
if c.Config.DisableAnalytics {
return
}
// TODO: Test this check
b := co.buckets[route]
c.DebugDetail("buckets[", route, "]: ", b)
if len(co.buckets) <= route || route < 0 {
return
}
micro := int((uutils.Nanotime() - nano) / 1000)
//co.PerfCounter.Push(since, true)
b.Lock()
b.counter++
if micro != b.avg {

404
gen_router.go
View File

File diff suppressed because it is too large Load Diff

21
router_gen/main.go
View File

@ -83,6 +83,9 @@ func main() {
vcpy := route.Vars
route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...)
} else if route.Name != "common.RouteWebsockets" {
//out += "\n\t\t\tsa := time.Now()"
out += "\n\t\t\tcn := uutils.Nanotime()"
}
out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
for _, item := range route.Vars {
@ -91,8 +94,10 @@ func main() {
out += `)`
if !route.Action && !route.NoHead {
out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else {
out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
} else if route.Name != "common.RouteWebsockets" {
//out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
//out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", sa)"
out += "\n\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
}
}
@ -149,6 +154,8 @@ func main() {
vcpy := route.Vars
route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...)
} else {
out += "\n\t\t\t\t\tcn := uutils.Nanotime()"
}
out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
for _, item := range route.Vars {
@ -158,7 +165,8 @@ func main() {
if !route.Action && !route.NoHead && !group.NoHead {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
//out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
}
}
@ -341,9 +349,11 @@ import (
"errors"
"os"
"net/http"
"time"
c "github.com/Azareal/Gosora/common"
co "github.com/Azareal/Gosora/common/counters"
"github.com/Azareal/Gosora/uutils"
"github.com/Azareal/Gosora/routes"
"github.com/Azareal/Gosora/routes/panel"
)
@ -384,6 +394,7 @@ var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNa
// TODO: Stop spilling these into the package scope?
func init() {
_ = time.Now()
co.SetRouteMapEnum(routeMapEnum)
co.SetReverseRouteMapEnum(reverseRouteMapEnum)
co.SetAgentMapEnum(agentMapEnum)
@ -666,14 +677,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string
var buffer []byte
var os int
for _, it := range StringToBytes(ua) {
for _, it := range uutils.StringToBytes(ua) {
if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, it)
} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 {
if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
switch(BytesToString(buffer)) {
switch(uutils.BytesToString(buffer)) {
case "Windows":
os = {{.AllOSMap.windows}}
case "Linux":

2
routes/forum.go
View File

@ -21,7 +21,7 @@ var forumStmts ForumStmts
func init() {
c.DbInits.Add(func(acc *qgen.Accumulator) error {
forumStmts = ForumStmts{
getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID = ?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(),
getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(),
}
return acc.FirstError()
})

3
routes/panel/analytics.go
View File

@ -834,6 +834,9 @@ func AnalyticsRoutesPerf(w http.ResponseWriter, r *http.Request, user c.User) c.
if inEx(ovitem.name) {
continue
}
if strings.HasPrefix(ovitem.name,"panel.") {
continue
}
var viewList []int64
for _, value := range revLabelList {
viewList = append(viewList, ovitem.viewMap[value])

11
tmpl_client/stub.go
View File

@ -1,9 +1,10 @@
package tmpl
import (
"reflect"
"runtime"
"unsafe"
//"reflect"
//"runtime"
//"unsafe"
"github.com/Azareal/Gosora/uutils"
)
var GetFrag = func(name string) [][]byte {
@ -14,6 +15,9 @@ type WriteString interface {
WriteString(s string) (n int, err error)
}
var StringToBytes = uutils.StringToBytes
/*
func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@ -23,3 +27,4 @@ func StringToBytes(s string) (bytes []byte) {
runtime.KeepAlive(&s)
return bytes
}
*/

22
tmpl_stub.go
View File

@ -1,11 +1,14 @@
package main
import (
"reflect"
"runtime"
"unsafe"
//"reflect"
//"runtime"
//"unsafe"
"github.com/Azareal/Gosora/uutils"
)
// TODO: Add a safe build mode for things like Google Appengine
var GetFrag = func(name string) [][]byte {
return nil
}
@ -14,6 +17,11 @@ type WriteString interface {
WriteString(s string) (n int, err error)
}
var StringToBytes = uutils.StringToBytes
var BytesToString = uutils.BytesToString
var Nanotime = uutils.Nanotime
/*
func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@ -32,3 +40,11 @@ func BytesToString(bytes []byte) (s string) {
runtime.KeepAlive(&bytes)
return s
}
//go:noescape
//go:linkname nanotime runtime.nanotime
func nanotime() int64
func Nanotime() int64 {
return nanotime()
}*/

36
uutils/utils.go Normal file
View File

@ -0,0 +1,36 @@
package uutils
import (
"reflect"
"runtime"
"unsafe"
)
// TODO: Add a safe build mode for things like Google Appengine
func StringToBytes(s string) (bytes []byte) {
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
slice.Data = str.Data
slice.Len = str.Len
slice.Cap = str.Len
runtime.KeepAlive(&s)
return bytes
}
func BytesToString(bytes []byte) (s string) {
slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
str := (*reflect.StringHeader)(unsafe.Pointer(&s))
str.Data = slice.Data
str.Len = slice.Len
runtime.KeepAlive(&bytes)
return s
}
//go:noescape
//go:linkname nanotime runtime.nanotime
func nanotime() int64
func Nanotime() int64 {
return nanotime()
}