a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

optimise agent and system counters

This commit is contained in:
Azareal 2020-02-25 18:12:54 +10:00
parent 64335cf1ef
commit 04fdf4c318
2 changed files with 20 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
common/counters/agents.go
View File

@ -2,6 +2,7 @@ package counters
import ( import (
"database/sql" "database/sql"
"sync/atomic"
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen" qgen "github.com/Azareal/Gosora/query_gen"
@ -11,18 +12,14 @@ import (
var AgentViewCounter *DefaultAgentViewCounter var AgentViewCounter *DefaultAgentViewCounter
type DefaultAgentViewCounter struct { type DefaultAgentViewCounter struct {
agentBuckets []*RWMutexCounterBucket //[AgentID]count buckets []int64 //[AgentID]count
insert *sql.Stmt insert *sql.Stmt
} }
func NewDefaultAgentViewCounter(acc *qgen.Accumulator) (*DefaultAgentViewCounter, error) { func NewDefaultAgentViewCounter(acc *qgen.Accumulator) (*DefaultAgentViewCounter, error) {
var agentBuckets = make([]*RWMutexCounterBucket, len(agentMapEnum))
for bucketID, _ := range agentBuckets {
agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
co := &DefaultAgentViewCounter{ co := &DefaultAgentViewCounter{
agentBuckets: agentBuckets, buckets: make([]int64, len(agentMapEnum)),
insert: acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(), insert: acc.Insert("viewchunks_agents").Columns("count,createdAt,browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
} }
c.AddScheduledFifteenMinuteTask(co.Tick) c.AddScheduledFifteenMinuteTask(co.Tick)
//c.AddScheduledSecondTask(co.Tick) //c.AddScheduledSecondTask(co.Tick)
@ -31,14 +28,9 @@ func NewDefaultAgentViewCounter(acc *qgen.Accumulator) (*DefaultAgentViewCounter
} }
func (co *DefaultAgentViewCounter) Tick() error { func (co *DefaultAgentViewCounter) Tick() error {
for agentID, agentBucket := range co.agentBuckets { for id, _ := range co.buckets {
var count int count := atomic.SwapInt64(&co.buckets[id], 0)
agentBucket.RLock() err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
count = agentBucket.counter
agentBucket.counter = 0
agentBucket.RUnlock()
err := co.insertChunk(count, agentID) // TODO: Bulk insert for speed?
if err != nil { if err != nil {
return errors.Wrap(errors.WithStack(err), "agent counter") return errors.Wrap(errors.WithStack(err), "agent counter")
} }
@ -46,7 +38,7 @@ func (co *DefaultAgentViewCounter) Tick() error {
return nil return nil
} }
func (co *DefaultAgentViewCounter) insertChunk(count int, agent int) error { func (co *DefaultAgentViewCounter) insertChunk(count int64, agent int) error {
if count == 0 { if count == 0 {
return nil return nil
} }
@ -58,11 +50,9 @@ func (co *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
func (co *DefaultAgentViewCounter) Bump(agent int) { func (co *DefaultAgentViewCounter) Bump(agent int) {
// TODO: Test this check // TODO: Test this check
c.DebugDetail("co.agentBuckets[", agent, "]: ", co.agentBuckets[agent]) c.DebugDetail("co.buckets[", agent, "]: ", co.buckets[agent])
if len(co.agentBuckets) <= agent || agent < 0 { if len(co.buckets) <= agent || agent < 0 {
return return
} }
co.agentBuckets[agent].Lock() atomic.AddInt64(&co.buckets[agent], 1)
co.agentBuckets[agent].counter++
co.agentBuckets[agent].Unlock()
} }

26
common/counters/systems.go
View File

@ -2,6 +2,7 @@ package counters
import ( import (
"database/sql" "database/sql"
"sync/atomic"
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen" qgen "github.com/Azareal/Gosora/query_gen"
@ -11,18 +12,14 @@ import (
var OSViewCounter *DefaultOSViewCounter var OSViewCounter *DefaultOSViewCounter
type DefaultOSViewCounter struct { type DefaultOSViewCounter struct {
buckets []*RWMutexCounterBucket //[OSID]count buckets []int64 //[OSID]count
insert *sql.Stmt insert *sql.Stmt
} }
func NewDefaultOSViewCounter(acc *qgen.Accumulator) (*DefaultOSViewCounter, error) { func NewDefaultOSViewCounter(acc *qgen.Accumulator) (*DefaultOSViewCounter, error) {
var osBuckets = make([]*RWMutexCounterBucket, len(osMapEnum))
for bucketID, _ := range osBuckets {
osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
co := &DefaultOSViewCounter{ co := &DefaultOSViewCounter{
buckets: osBuckets, buckets: make([]int64, len(osMapEnum)),
insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(), insert: acc.Insert("viewchunks_systems").Columns("count,createdAt,system").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
} }
c.AddScheduledFifteenMinuteTask(co.Tick) c.AddScheduledFifteenMinuteTask(co.Tick)
//c.AddScheduledSecondTask(co.Tick) //c.AddScheduledSecondTask(co.Tick)
@ -31,13 +28,8 @@ func NewDefaultOSViewCounter(acc *qgen.Accumulator) (*DefaultOSViewCounter, erro
} }
func (co *DefaultOSViewCounter) Tick() error { func (co *DefaultOSViewCounter) Tick() error {
for id, bucket := range co.buckets { for id, _ := range co.buckets {
var count int count := atomic.SwapInt64(&co.buckets[id], 0)
bucket.RLock()
count = bucket.counter
bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
bucket.RUnlock()
err := co.insertChunk(count, id) // TODO: Bulk insert for speed? err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
if err != nil { if err != nil {
return errors.Wrap(errors.WithStack(err), "system counter") return errors.Wrap(errors.WithStack(err), "system counter")
@ -46,7 +38,7 @@ func (co *DefaultOSViewCounter) Tick() error {
return nil return nil
} }
func (co *DefaultOSViewCounter) insertChunk(count int, os int) error { func (co *DefaultOSViewCounter) insertChunk(count int64, os int) error {
if count == 0 { if count == 0 {
return nil return nil
} }
@ -62,7 +54,5 @@ func (co *DefaultOSViewCounter) Bump(id int) {
if len(co.buckets) <= id || id < 0 { if len(co.buckets) <= id || id < 0 {
return return
} }
co.buckets[id].Lock() atomic.AddInt64(&co.buckets[id], 1)
co.buckets[id].counter++
co.buckets[id].Unlock()
} }