simplify the agents, forums, memory and referrer counters

move DirSize into it's own file

common/counters/agents.go

@@ -4,7 +4,8 @@ import (
 	"database/sql"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var AgentViewCounter *DefaultAgentViewCounter
@@ -19,49 +20,49 @@ func NewDefaultAgentViewCounter(acc *qgen.Accumulator) (*DefaultAgentViewCounter
 	for bucketID, _ := range agentBuckets {
 		agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
 	}
-	counter := &DefaultAgentViewCounter{
+	co := &DefaultAgentViewCounter{
 		agentBuckets: agentBuckets,
 		insert:       acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
-	c.AddScheduledFifteenMinuteTask(counter.Tick)
+	c.AddScheduledFifteenMinuteTask(co.Tick)
-	//c.AddScheduledSecondTask(counter.Tick)
+	//c.AddScheduledSecondTask(co.Tick)
-	c.AddShutdownTask(counter.Tick)
+	c.AddShutdownTask(co.Tick)
-	return counter, acc.FirstError()
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultAgentViewCounter) Tick() error {
+func (co *DefaultAgentViewCounter) Tick() error {
-	for agentID, agentBucket := range counter.agentBuckets {
+	for agentID, agentBucket := range co.agentBuckets {
 		var count int
 		agentBucket.RLock()
 		count = agentBucket.counter
 		agentBucket.counter = 0
 		agentBucket.RUnlock()
 
-		err := counter.insertChunk(count, agentID) // TODO: Bulk insert for speed?
+		err := co.insertChunk(count, agentID) // TODO: Bulk insert for speed?
 		if err != nil {
-			return err
+			return errors.Wrap(errors.WithStack(err), "agent counter")
 		}
 	}
 	return nil
 }
 
-func (counter *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
+func (co *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
 	if count == 0 {
 		return nil
 	}
-	var agentName = reverseAgentMapEnum[agent]
+	agentName := reverseAgentMapEnum[agent]
-	c.DebugLogf("Inserting a viewchunk with a count of %d for agent %s (%d)", count, agentName, agent)
+	c.DebugLogf("Inserting a vchunk with a count of %d for agent %s (%d)", count, agentName, agent)
-	_, err := counter.insert.Exec(count, agentName)
+	_, err := co.insert.Exec(count, agentName)
 	return err
 }
 
-func (counter *DefaultAgentViewCounter) Bump(agent int) {
+func (co *DefaultAgentViewCounter) Bump(agent int) {
 	// TODO: Test this check
-	c.DebugDetail("counter.agentBuckets[", agent, "]: ", counter.agentBuckets[agent])
+	c.DebugDetail("co.agentBuckets[", agent, "]: ", co.agentBuckets[agent])
-	if len(counter.agentBuckets) <= agent || agent < 0 {
+	if len(co.agentBuckets) <= agent || agent < 0 {
 		return
 	}
-	counter.agentBuckets[agent].Lock()
+	co.agentBuckets[agent].Lock()
-	counter.agentBuckets[agent].counter++
+	co.agentBuckets[agent].counter++
-	counter.agentBuckets[agent].Unlock()
+	co.agentBuckets[agent].Unlock()
 }

common/counters/forums.go

@@ -6,6 +6,7 @@ import (
 
 	c "github.com/Azareal/Gosora/common"
 	"github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var ForumViewCounter *DefaultForumViewCounter
@@ -23,95 +24,84 @@ type DefaultForumViewCounter struct {
 
 func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
 	acc := qgen.NewAcc()
-	counter := &DefaultForumViewCounter{
+	co := &DefaultForumViewCounter{
 		oddMap:  make(map[int]*RWMutexCounterBucket),
 		evenMap: make(map[int]*RWMutexCounterBucket),
 		insert:  acc.Insert("viewchunks_forums").Columns("count, createdAt, forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
-	c.AddScheduledFifteenMinuteTask(counter.Tick) // There could be a lot of routes, so we don't want to be running this every second
+	c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
-	//c.AddScheduledSecondTask(counter.Tick)
+	//c.AddScheduledSecondTask(co.Tick)
-	c.AddShutdownTask(counter.Tick)
+	c.AddShutdownTask(co.Tick)
-	return counter, acc.FirstError()
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultForumViewCounter) Tick() error {
+func (co *DefaultForumViewCounter) Tick() error {
-	counter.oddLock.RLock()
+	cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error {
-	oddMap := counter.oddMap
+		l.RLock()
-	counter.oddLock.RUnlock()
+		for forumID, forum := range m {
-	for forumID, forum := range oddMap {
+			l.RUnlock()
-		var count int
+			var count int
-		forum.RLock()
+			forum.RLock()
-		count = forum.counter
+			count = forum.counter
-		forum.RUnlock()
+			forum.RUnlock()
-		// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
+			// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
-		counter.oddLock.Lock()
+			l.Lock()
-		delete(counter.oddMap, forumID)
+			delete(m, forumID)
-		counter.oddLock.Unlock()
+			l.Unlock()
-		err := counter.insertChunk(count, forumID)
+			err := co.insertChunk(count, forumID)
-		if err != nil {
+			if err != nil {
-			return err
+				return errors.Wrap(errors.WithStack(err),"forum counter")
+			}
+			l.RLock()
 		}
+		l.RUnlock()
+		return nil
 	}
-
+	err := cLoop(&co.oddLock,co.oddMap)
-	counter.evenLock.RLock()
+	if err != nil {
-	evenMap := counter.evenMap
+		return err
-	counter.evenLock.RUnlock()
-	for forumID, forum := range evenMap {
-		var count int
-		forum.RLock()
-		count = forum.counter
-		forum.RUnlock()
-		// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
-		counter.evenLock.Lock()
-		delete(counter.evenMap, forumID)
-		counter.evenLock.Unlock()
-		err := counter.insertChunk(count, forumID)
-		if err != nil {
-			return err
-		}
 	}
-
+	return cLoop(&co.evenLock,co.evenMap)
-	return nil
 }
 
-func (counter *DefaultForumViewCounter) insertChunk(count int, forum int) error {
+func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
 	if count == 0 {
 		return nil
 	}
-	c.DebugLogf("Inserting a viewchunk with a count of %d for forum %d", count, forum)
+	c.DebugLogf("Inserting a vchunk with a count of %d for forum %d", count, forum)
-	_, err := counter.insert.Exec(count, forum)
+	_, err := co.insert.Exec(count, forum)
 	return err
 }
 
-func (counter *DefaultForumViewCounter) Bump(forumID int) {
+func (co *DefaultForumViewCounter) Bump(forumID int) {
 	// Is the ID even?
 	if forumID%2 == 0 {
-		counter.evenLock.RLock()
+		co.evenLock.RLock()
-		forum, ok := counter.evenMap[forumID]
+		forum, ok := co.evenMap[forumID]
-		counter.evenLock.RUnlock()
+		co.evenLock.RUnlock()
 		if ok {
 			forum.Lock()
 			forum.counter++
 			forum.Unlock()
 		} else {
-			counter.evenLock.Lock()
+			co.evenLock.Lock()
-			counter.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
+			co.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
-			counter.evenLock.Unlock()
+			co.evenLock.Unlock()
 		}
 		return
 	}
 
-	counter.oddLock.RLock()
+	co.oddLock.RLock()
-	forum, ok := counter.oddMap[forumID]
+	forum, ok := co.oddMap[forumID]
-	counter.oddLock.RUnlock()
+	co.oddLock.RUnlock()
 	if ok {
 		forum.Lock()
 		forum.counter++
 		forum.Unlock()
 	} else {
-		counter.oddLock.Lock()
+		co.oddLock.Lock()
-		counter.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
+		co.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
-		counter.oddLock.Unlock()
+		co.oddLock.Unlock()
 	}
 }
 

common/counters/langs.go

@@ -1,8 +1,12 @@
 package counters
 
-import "database/sql"
+import (
-import c "github.com/Azareal/Gosora/common"
+	"database/sql"
-import "github.com/Azareal/Gosora/query_gen"
+
+	c "github.com/Azareal/Gosora/common"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
+)
 
 var LangViewCounter *DefaultLangViewCounter
 
@@ -101,56 +105,56 @@ type DefaultLangViewCounter struct {
 }
 
 func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) {
-	var langBuckets = make([]*RWMutexCounterBucket, len(langCodes))
+	langBuckets := make([]*RWMutexCounterBucket, len(langCodes))
 	for bucketID, _ := range langBuckets {
 		langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
 	}
-	var codesToIndices = make(map[string]int)
+	codesToIndices := make(map[string]int, len(langCodes))
 	for index, code := range langCodes {
 		codesToIndices[code] = index
 	}
 
-	counter := &DefaultLangViewCounter{
+	co := &DefaultLangViewCounter{
 		buckets:        langBuckets,
 		codesToIndices: codesToIndices,
 		insert:         acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
 
-	c.AddScheduledFifteenMinuteTask(counter.Tick)
+	c.AddScheduledFifteenMinuteTask(co.Tick)
-	//c.AddScheduledSecondTask(counter.Tick)
+	//c.AddScheduledSecondTask(co.Tick)
-	c.AddShutdownTask(counter.Tick)
+	c.AddShutdownTask(co.Tick)
-	return counter, acc.FirstError()
+	return co, acc.FirstError()
 }
 
-func (counter *DefaultLangViewCounter) Tick() error {
+func (co *DefaultLangViewCounter) Tick() error {
-	for id, bucket := range counter.buckets {
+	for id, bucket := range co.buckets {
 		var count int
 		bucket.RLock()
 		count = bucket.counter
 		bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
 		bucket.RUnlock()
 
-		err := counter.insertChunk(count, id) // TODO: Bulk insert for speed?
+		err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
 		if err != nil {
-			return err
+			return errors.Wrap(errors.WithStack(err), "langview counter")
 		}
 	}
 	return nil
 }
 
-func (counter *DefaultLangViewCounter) insertChunk(count int, id int) error {
+func (co *DefaultLangViewCounter) insertChunk(count int, id int) error {
 	if count == 0 {
 		return nil
 	}
-	var langCode = langCodes[id]
+	langCode := langCodes[id]
-	c.DebugLogf("Inserting a viewchunk with a count of %d for lang %s (%d)", count, langCode, id)
+	c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id)
-	_, err := counter.insert.Exec(count, langCode)
+	_, err := co.insert.Exec(count, langCode)
 	return err
 }
 
-func (counter *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
+func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
 	validCode = true
-	id, ok := counter.codesToIndices[langCode]
+	id, ok := co.codesToIndices[langCode]
 	if !ok {
 		// TODO: Tell the caller that the code's invalid
 		id = 0 // Unknown
@@ -158,13 +162,13 @@ func (counter *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
 	}
 
 	// TODO: Test this check
-	c.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id])
+	c.DebugDetail("co.buckets[", id, "]: ", co.buckets[id])
-	if len(counter.buckets) <= id || id < 0 {
+	if len(co.buckets) <= id || id < 0 {
 		return validCode
 	}
-	counter.buckets[id].Lock()
+	co.buckets[id].Lock()
-	counter.buckets[id].counter++
+	co.buckets[id].counter++
-	counter.buckets[id].Unlock()
+	co.buckets[id].Unlock()
 
 	return validCode
 }

common/counters/memory.go

@@ -1,13 +1,14 @@
 package counters
 
 import (
-	"time"
-	"sync"
-	"runtime"
 	"database/sql"
+	"runtime"
+	"sync"
+	"time"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var MemoryCounter *DefaultMemoryCounter
@@ -15,19 +16,19 @@ var MemoryCounter *DefaultMemoryCounter
 type DefaultMemoryCounter struct {
 	insert *sql.Stmt
 
-	totMem uint64
+	totMem     uint64
-	totCount uint64
+	totCount   uint64
-	stackMem uint64
+	stackMem   uint64
 	stackCount uint64
-	heapMem uint64
+	heapMem    uint64
-	heapCount uint64
+	heapCount  uint64
 
 	sync.Mutex
 }
 
 func NewMemoryCounter(acc *qgen.Accumulator) (*DefaultMemoryCounter, error) {
 	co := &DefaultMemoryCounter{
-		insert:        acc.Insert("memchunks").Columns("count, stack, heap, createdAt").Fields("?,?,?,UTC_TIMESTAMP()").Prepare(),
+		insert: acc.Insert("memchunks").Columns("count, stack, heap, createdAt").Fields("?,?,?,UTC_TIMESTAMP()").Prepare(),
 	}
 	c.AddScheduledFifteenMinuteTask(co.Tick)
 	//c.AddScheduledSecondTask(co.Tick)
@@ -81,5 +82,8 @@ func (co *DefaultMemoryCounter) Tick() (err error) {
 
 	c.DebugLogf("Inserting a memchunk with a value of %d - %d - %d", avgMem, avgStack, avgHeap)
 	_, err = co.insert.Exec(avgMem, avgStack, avgHeap)
-	return err
+	if err != nil {
+		return errors.Wrap(errors.WithStack(err), "mem counter")
+	}
+	return nil
 }

common/counters/referrers.go

@@ -5,8 +5,9 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/Azareal/Gosora/common"
+	c "github.com/Azareal/Gosora/common"
 	"github.com/Azareal/Gosora/query_gen"
+	"github.com/pkg/errors"
 )
 
 var ReferrerTracker *DefaultReferrerTracker
@@ -36,9 +37,9 @@ func NewDefaultReferrerTracker() (*DefaultReferrerTracker, error) {
 		even:   make(map[string]*ReferrerItem),
 		insert: acc.Insert("viewchunks_referrers").Columns("count, createdAt, domain").Fields("?,UTC_TIMESTAMP(),?").Prepare(), // TODO: Do something more efficient than doing a query for each referrer
 	}
-	common.AddScheduledFifteenMinuteTask(refTracker.Tick)
+	c.AddScheduledFifteenMinuteTask(refTracker.Tick)
-	//common.AddScheduledSecondTask(refTracker.Tick)
+	//c.AddScheduledSecondTask(refTracker.Tick)
-	common.AddShutdownTask(refTracker.Tick)
+	c.AddShutdownTask(refTracker.Tick)
 	return refTracker, acc.FirstError()
 }
 
@@ -50,50 +51,41 @@ func (ref *DefaultReferrerTracker) Tick() (err error) {
 		if count != 0 {
 			err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
 			if err != nil {
-				return err
+				return errors.Wrap(errors.WithStack(err),"ref counter")
 			}
 		}
-
 		delete(referrersToDelete, referrer)
 	}
 
 	//  Run the queries and schedule zero view refs for deletion from memory
-	ref.oddLock.Lock()
+	refLoop := func(l *sync.RWMutex, m map[string]*ReferrerItem) error {
-	for referrer, counter := range ref.odd {
+		l.Lock()
-		if counter.Count == 0 {
+		defer l.Unlock()
-			referrersToDelete[referrer] = counter
+		for referrer, counter := range m {
-			delete(ref.odd, referrer)
+			if counter.Count == 0 {
-		}
+				referrersToDelete[referrer] = counter
-		count := atomic.SwapInt64(&counter.Count, 0)
+				delete(m, referrer)
-		err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
+			}
-		if err != nil {
+			count := atomic.SwapInt64(&counter.Count, 0)
-			return err
+			err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
+			if err != nil {
+				return errors.Wrap(errors.WithStack(err),"ref counter")
+			}
 		}
+		return nil
 	}
-	ref.oddLock.Unlock()
+	err = refLoop(&ref.oddLock,ref.odd)
-
+	if err != nil {
-	ref.evenLock.Lock()
+		return err
-	for referrer, counter := range ref.even {
-		if counter.Count == 0 {
-			referrersToDelete[referrer] = counter
-			delete(ref.even, referrer)
-		}
-		count := atomic.SwapInt64(&counter.Count, 0)
-		err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
-		if err != nil {
-			return err
-		}
 	}
-	ref.evenLock.Unlock()
+	return refLoop(&ref.evenLock,ref.even)
-
-	return nil
 }
 
 func (ref *DefaultReferrerTracker) insertChunk(referrer string, count int64) error {
 	if count == 0 {
 		return nil
 	}
-	common.DebugDetailf("Inserting a viewchunk with a count of %d for referrer %s", count, referrer)
+	c.DebugDetailf("Inserting a vchunk with a count of %d for referrer %s", count, referrer)
 	_, err := ref.insert.Exec(count, referrer)
 	return err
 }

common/disk.go

@@ -0,0 +1,20 @@
+package common
+
+import (
+	"path/filepath"
+	"os"
+)
+
+func DirSize(path string) (int, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, file os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !file.IsDir() {
+			size += file.Size()
+		}
+		return err
+	})
+	return int(size), err
+}

common/utils.go

@@ -10,7 +10,6 @@ import (
 	"crypto/rand"
 	"encoding/base32"
 	"encoding/base64"
-	"path/filepath"
 	"errors"
 	"fmt"
 	"html"
@@ -537,17 +536,3 @@ func BuildSlug(slug string, id int) string {
 	}
 	return slug + "." + strconv.Itoa(id)
 }
-
-func DirSize(path string) (int, error) {
-	var size int64
-	err := filepath.Walk(path, func(_ string, file os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !file.IsDir() {
-			size += file.Size()
-		}
-		return err
-	})
-	return int(size), err
-}