From 225a2cc8a16651dd8aaf68f425e191e423085787 Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 19 Sep 2018 16:09:03 +1000 Subject: [PATCH] 0.1.0 tag, checkpoint release. Upped the user cache capacity and topic cache capacity. More resource management work. WIP. Added Hyperdrive to the experimental folder. It doesn't really work right now, but I'd like to track it's progress. Eliminated a line in global.js --- common/null_user_cache.go | 4 +++- common/user_cache.go | 24 ++++++++++++++++++++-- config/config_example.json | 4 ++-- experimental/plugin_hyperdrive.go | 34 +++++++++++++++++++++++++++++++ install/install.go | 4 ++-- main.go | 14 +++++++++---- public/global.js | 4 +--- 7 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 experimental/plugin_hyperdrive.go diff --git a/common/null_user_cache.go b/common/null_user_cache.go index 04cc9203..73279d50 100644 --- a/common/null_user_cache.go +++ b/common/null_user_cache.go @@ -10,7 +10,9 @@ func NewNullUserCache() *NullUserCache { } // nolint -func (mus *NullUserCache) DeallocOverflow() {} +func (mus *NullUserCache) DeallocOverflow(evictPriority bool) (evicted int) { + return 0 +} func (mus *NullUserCache) Get(id int) (*User, error) { return nil, ErrNoRows } diff --git a/common/user_cache.go b/common/user_cache.go index 8678908c..29e27ba5 100644 --- a/common/user_cache.go +++ b/common/user_cache.go @@ -7,7 +7,7 @@ import ( // UserCache is an interface which spits out users from a fast cache rather than the database, whether from memory or from an application like Redis. Users may not be present in the cache but may be in the database type UserCache interface { - DeallocOverflow() // May cause thread contention, looks for items to evict + DeallocOverflow(evictPriority bool) (evicted int) // May cause thread contention, looks for items to evict Get(id int) (*User, error) GetUnsafe(id int) (*User, error) BulkGet(ids []int) (list []*User) @@ -40,7 +40,7 @@ func NewMemoryUserCache(capacity int) *MemoryUserCache { } // TODO: Avoid deallocating topic list users -func (mus *MemoryUserCache) DeallocOverflow() { +func (mus *MemoryUserCache) DeallocOverflow(evictPriority bool) (evicted int) { var toEvict = make([]int, 10) var evIndex = 0 mus.RLock() @@ -58,6 +58,25 @@ func (mus *MemoryUserCache) DeallocOverflow() { } mus.RUnlock() + // Clear some of the less active users now with a bit more aggressiveness + if evIndex == 0 && evictPriority { + toEvict = make([]int, 20) + mus.RLock() + for _, user := range mus.items { + if user.Score < 100 && !user.IsMod { + if EnableWebsockets && WsHub.HasUser(user.ID) { + continue + } + toEvict[evIndex] = user.ID + evIndex++ + if evIndex == 20 { + break + } + } + } + mus.RUnlock() + } + // Remove zero IDs from the evictable list, so we don't waste precious cycles locked for those var lastZero = -1 for i, uid := range toEvict { @@ -70,6 +89,7 @@ func (mus *MemoryUserCache) DeallocOverflow() { } mus.BulkRemove(toEvict) + return len(toEvict) } // Get fetches a user by ID. Returns ErrNoRows if not present. diff --git a/config/config_example.json b/config/config_example.json index 36fe9b56..18ba2240 100644 --- a/config/config_example.json +++ b/config/config_example.json @@ -20,8 +20,8 @@ "MaxRequestSizeStr":"5MB", "UserCache":"static", "TopicCache":"static", - "UserCacheCapacity":120, - "TopicCacheCapacity":200, + "UserCacheCapacity":150, + "TopicCacheCapacity":250, "DefaultPath":"/topics/", "DefaultGroup":3, "ActivationGroup":5, diff --git a/experimental/plugin_hyperdrive.go b/experimental/plugin_hyperdrive.go new file mode 100644 index 00000000..20bd33a6 --- /dev/null +++ b/experimental/plugin_hyperdrive.go @@ -0,0 +1,34 @@ +// Highly experimental plugin for caching rendered pages for guests +package main + +import ( + "sync/atomic" + + "./common" +) + +var hyperPageCache *HyperPageCache + +func init() { + common.Plugins.Add(&common.Plugin{UName: "hyperdrive", Name: "Hyperdrive", Author: "Azareal", Init: initHyperdrive, Deactivate: deactivateHyperdrive}) +} + +func initHyperdrive() error { + hyperPageCache = newHyperPageCache() + common.Plugins["hyperdrive"].AddHook("somewhere", deactivateHyperdrive) + return nil +} + +func deactivateHyperdrive() { + hyperPageCache = nil +} + +type HyperPageCache struct { + topicList atomic.Value +} + +func newHyperPageCache() *HyperPageCache { + pageCache := new(HyperPageCache) + pageCache.topicList.Store([]byte("")) + return pageCache +} diff --git a/install/install.go b/install/install.go index 17448fc0..50419b22 100644 --- a/install/install.go +++ b/install/install.go @@ -118,8 +118,8 @@ func main() { "MaxRequestSizeStr":"5MB", "UserCache":"static", "TopicCache":"static", - "UserCacheCapacity":120, - "TopicCacheCapacity":200, + "UserCacheCapacity":150, + "TopicCacheCapacity":250, "DefaultPath":"/topics/", "DefaultGroup":3, "ActivationGroup":5, diff --git a/main.go b/main.go index 45d2a159..ad6e2a1d 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( "github.com/pkg/errors" ) -var version = common.Version{Major: 0, Minor: 1, Patch: 0, Tag: "dev"} +var version = common.Version{Major: 0, Minor: 1, Patch: 0} var router *GenRouter var logWriter = io.MultiWriter(os.Stderr) @@ -373,13 +373,15 @@ func main() { go tickLoop(thumbChan, halfSecondTicker, secondTicker, fifteenMinuteTicker, hourTicker) // Resource Management Goroutine - go func() { + /*go func() { ucache := common.Users.GetCache() tcache := common.Topics.GetCache() if ucache == nil && tcache == nil { return } + var lastEvictedCount int + var couldNotDealloc bool for { select { case <-secondTicker.C: @@ -387,13 +389,17 @@ func main() { if ucache != nil { ucap := ucache.GetCapacity() if ucache.Length() <= ucap || common.Users.GlobalCount() <= ucap { + countNotDealloc = false continue } - //ucache.DeallocOverflow() + lastEvictedCount = ucache.DeallocOverflow(countNotDealloc) + countNotDealloc = (lastEvictedCount == 0) + } else { + countNotDealloc = false } } } - }() + }()*/ log.Print("Initialising the router") router, err = NewGenRouter(http.FileServer(http.Dir("./uploads"))) diff --git a/public/global.js b/public/global.js index bbc1b738..653a9e5f 100644 --- a/public/global.js +++ b/public/global.js @@ -121,13 +121,11 @@ function loadAlerts(menuAlerts) { updateAlertList(menuAlerts) }, error: (magic,theStatus,error) => { - let errtxt + let errtxt = "Unable to get the alerts"; try { var data = JSON.parse(magic.responseText); if("errmsg" in data) errtxt = data.errmsg; - else errtxt = "Unable to get the alerts"; } catch(err) { - errtxt = "Unable to get the alerts"; console.log(magic.responseText); console.log(err); }