From b04d77d7b6082b9c23de7e653d8e0f6c6a5d69e0 Mon Sep 17 00:00:00 2001 From: Azareal Date: Mon, 9 Mar 2020 13:51:44 +1000 Subject: [PATCH] optimise topic lists by caching common qcounts for getList stmts recalc forum topic counts reduce thaw period to 3 cache top 8 forums instead of 5 optimise GetListByForum with zero topics add Each method to ForumStore --- common/forum_store.go | 80 +++++++++++++++++------------ common/recalc.go | 19 +++++++ common/thaw.go | 2 +- common/topic_list.go | 108 ++++++++++++++++++++++++++++++++++----- query_gen/querygen.go | 18 +++---- templates/paginator.html | 4 +- tickloop.go | 6 +++ 7 files changed, 178 insertions(+), 59 deletions(-) diff --git a/common/forum_store.go b/common/forum_store.go index f6bac5fc..df943863 100644 --- a/common/forum_store.go +++ b/common/forum_store.go @@ -28,6 +28,7 @@ var ErrNoDeleteReports = errors.New("You cannot delete the Reports forum") // ForumStore is an interface for accessing the forums and the metadata stored on them type ForumStore interface { LoadForums() error + Each(h func(*Forum) error) error DirtyGet(id int) *Forum Get(id int) (*Forum, error) BypassGet(id int) (*Forum, error) @@ -35,10 +36,10 @@ type ForumStore interface { Reload(id int) error // ? - Should we move this to ForumCache? It might require us to do some unnecessary casting though //Update(Forum) error Delete(id int) error - AddTopic(tid int, uid int, fid int) error + AddTopic(tid, uid, fid int) error RemoveTopic(fid int) error RemoveTopics(fid, count int) error - UpdateLastTopic(tid int, uid int, fid int) error + UpdateLastTopic(tid, uid, fid int) error Exists(id int) bool GetAll() ([]*Forum, error) GetAllIDs() ([]int, error) @@ -46,7 +47,7 @@ type ForumStore interface { GetAllVisibleIDs() ([]int, error) //GetChildren(parentID int, parentType string) ([]*Forum,error) //GetFirstChild(parentID int, parentType string) (*Forum,error) - Create(name string, desc string, active bool, preset string) (int, error) + Create(name, desc string, active bool, preset string) (int, error) UpdateOrder(updateMap map[int]int) error Count() int @@ -82,16 +83,16 @@ func NewMemoryForumStore() (*MemoryForumStore, error) { f := "forums" // TODO: Do a proper delete return &MemoryForumStore{ - get: acc.Select(f).Columns("name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Where("fid = ?").Prepare(), + get: acc.Select(f).Columns("name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Where("fid=?").Prepare(), getAll: acc.Select(f).Columns("fid, name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Orderby("order ASC, fid ASC").Prepare(), - delete: acc.Update(f).Set("name='',active=0").Where("fid = ?").Prepare(), + delete: acc.Update(f).Set("name='',active=0").Where("fid=?").Prepare(), create: acc.Insert(f).Columns("name, desc, tmpl, active, preset").Fields("?,?,'',?,?").Prepare(), count: acc.Count(f).Where("name != ''").Prepare(), - updateCache: acc.Update(f).Set("lastTopicID = ?, lastReplyerID = ?").Where("fid = ?").Prepare(), - addTopics: acc.Update(f).Set("topicCount=topicCount+?").Where("fid = ?").Prepare(), - removeTopics: acc.Update(f).Set("topicCount=topicCount-?").Where("fid = ?").Prepare(), - lastTopic: acc.Select("topics").Columns("tid").Where("parentID = ?").Orderby("lastReplyAt DESC, createdAt DESC").Limit("1").Prepare(), - updateOrder: acc.Update(f).Set("order = ?").Where("fid = ?").Prepare(), + updateCache: acc.Update(f).Set("lastTopicID=?, lastReplyerID=?").Where("fid=?").Prepare(), + addTopics: acc.Update(f).Set("topicCount=topicCount+?").Where("fid=?").Prepare(), + removeTopics: acc.Update(f).Set("topicCount=topicCount-?").Where("fid=?").Prepare(), + lastTopic: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("lastReplyAt DESC, createdAt DESC").Limit("1").Prepare(), + updateOrder: acc.Update(f).Set("order=?").Where("fid=?").Prepare(), }, acc.FirstError() } @@ -99,10 +100,10 @@ func NewMemoryForumStore() (*MemoryForumStore, error) { // TODO: Add support for subforums func (s *MemoryForumStore) LoadForums() error { var forumView []*Forum - addForum := func(forum *Forum) { - s.forums.Store(forum.ID, forum) - if forum.Active && forum.Name != "" && forum.ParentType == "" { - forumView = append(forumView, forum) + addForum := func(f *Forum) { + s.forums.Store(f.ID, f) + if f.Active && f.Name != "" && f.ParentType == "" { + forumView = append(forumView, f) } } @@ -140,11 +141,11 @@ func (s *MemoryForumStore) LoadForums() error { // ? - Will this be hit a lot by plugin_guilds? func (s *MemoryForumStore) rebuildView() { var forumView []*Forum - s.forums.Range(func(_ interface{}, value interface{}) bool { - forum := value.(*Forum) + s.forums.Range(func(_, val interface{}) bool { + f := val.(*Forum) // ? - ParentType blank means that it doesn't have a parent - if forum.Active && forum.Name != "" && forum.ParentType == "" { - forumView = append(forumView, forum) + if f.Active && f.Name != "" && f.ParentType == "" { + forumView = append(forumView, f) } return true }) @@ -153,6 +154,17 @@ func (s *MemoryForumStore) rebuildView() { TopicListThaw.Thaw() } +func (s *MemoryForumStore) Each(h func(*Forum) error) (err error) { + s.forums.Range(func(_, val interface{}) bool { + err = h(val.(*Forum)) + if err != nil { + return false + } + return true + }) + return err +} + func (s *MemoryForumStore) DirtyGet(id int) *Forum { fint, ok := s.forums.Load(id) if !ok || fint.(*Forum).Name == "" { @@ -208,11 +220,11 @@ func (s *MemoryForumStore) BypassGet(id int) (*Forum, error) { func (s *MemoryForumStore) BulkGetCopy(ids []int) (forums []Forum, err error) { forums = make([]Forum, len(ids)) for i, id := range ids { - forum, err := s.Get(id) + f, err := s.Get(id) if err != nil { return nil, err } - forums[i] = forum.Copy() + forums[i] = f.Copy() } return forums, nil } @@ -226,16 +238,16 @@ func (s *MemoryForumStore) Reload(id int) error { return nil } -func (s *MemoryForumStore) CacheSet(forum *Forum) error { - s.forums.Store(forum.ID, forum) +func (s *MemoryForumStore) CacheSet(f *Forum) error { + s.forums.Store(f.ID, f) s.rebuildView() return nil } // ! Has a randomised order func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) { - s.forums.Range(func(_ interface{}, value interface{}) bool { - forumView = append(forumView, value.(*Forum)) + s.forums.Range(func(_, val interface{}) bool { + forumView = append(forumView, val.(*Forum)) return true }) sort.Sort(SortForum(forumView)) @@ -244,8 +256,8 @@ func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) { // ? - Can we optimise the sorting? func (s *MemoryForumStore) GetAllIDs() (ids []int, err error) { - s.forums.Range(func(_ interface{}, value interface{}) bool { - ids = append(ids, value.(*Forum).ID) + s.forums.Range(func(_, val interface{}) bool { + ids = append(ids, val.(*Forum).ID) return true }) sort.Ints(ids) @@ -299,7 +311,7 @@ func (s *MemoryForumStore) Delete(id int) error { return err } -func (s *MemoryForumStore) AddTopic(tid int, uid int, fid int) error { +func (s *MemoryForumStore) AddTopic(tid, uid, fid int) error { _, err := s.updateCache.Exec(tid, uid, fid) if err != nil { return err @@ -351,7 +363,7 @@ func (s *MemoryForumStore) RemoveTopic(fid int) error { } return s.RefreshTopic(fid) } -func (s *MemoryForumStore) RemoveTopics(fid int, count int) error { +func (s *MemoryForumStore) RemoveTopics(fid, count int) error { _, err := s.removeTopics.Exec(count, fid) if err != nil { return err @@ -361,7 +373,7 @@ func (s *MemoryForumStore) RemoveTopics(fid int, count int) error { // DEPRECATED. forum.Update() will be the way to do this in the future, once it's completed // TODO: Have a pointer to the last topic rather than storing it on the forum itself -func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error { +func (s *MemoryForumStore) UpdateLastTopic(tid, uid, fid int) error { _, err := s.updateCache.Exec(tid, uid, fid) if err != nil { return err @@ -370,7 +382,7 @@ func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error { return s.Reload(fid) } -func (s *MemoryForumStore) Create(name string, desc string, active bool, preset string) (int, error) { +func (s *MemoryForumStore) Create(name, desc string, active bool, preset string) (int, error) { if name == "" { return 0, ErrBlankName } @@ -410,12 +422,12 @@ func (s *MemoryForumStore) UpdateOrder(updateMap map[int]int) error { // ! Might be slightly inaccurate, if the sync.Map is constantly shifting and churning, but it'll stabilise eventually. Also, slow. Don't use this on every request x.x // Length returns the number of forums in the memory cache -func (s *MemoryForumStore) Length() (length int) { - s.forums.Range(func(_ interface{}, value interface{}) bool { - length++ +func (s *MemoryForumStore) Length() (len int) { + s.forums.Range(func(_, _ interface{}) bool { + len++ return true }) - return length + return len } // TODO: Get the total count of forums in the forum store rather than doing a heavy query for this? diff --git a/common/recalc.go b/common/recalc.go index 43c65d3e..af6de5b7 100644 --- a/common/recalc.go +++ b/common/recalc.go @@ -12,6 +12,7 @@ var Recalc RecalcInt type RecalcInt interface { Replies() (count int, err error) + Forums() (count int, err error) Subscriptions() (count int, err error) ActivityStream() (count int, err error) Users() error @@ -22,6 +23,8 @@ type DefaultRecalc struct { getActivitySubscriptions *sql.Stmt getActivityStream *sql.Stmt getAttachments *sql.Stmt + getTopicCount *sql.Stmt + resetTopicCount *sql.Stmt } func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) { @@ -29,6 +32,10 @@ func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) { getActivitySubscriptions: acc.Select("activity_subscriptions").Columns("targetID,targetType").Prepare(), getActivityStream: acc.Select("activity_stream").Columns("asid,event,elementID,elementType,extra").Prepare(), getAttachments: acc.Select("attachments").Columns("attachID,originID,originTable").Prepare(), + getTopicCount: acc.Count("topics").Where("parentID=?").Prepare(), + //resetTopicCount: acc.SimpleUpdateSelect("forums", "topicCount = tc", "topics", "count(*) as tc", "parentID=?", "", ""), + // TODO: Avoid using RawPrepare + resetTopicCount: acc.RawPrepare("UPDATE forums, (SELECT COUNT(*) as tc FROM topics WHERE parentID=?) AS src SET forums.topicCount=src.tc WHERE forums.fid=?"), }, acc.FirstError() } @@ -50,6 +57,18 @@ func (s *DefaultRecalc) Replies() (count int, err error) { return count, err } +func (s *DefaultRecalc) Forums() (count int, err error) { + err = Forums.Each(func(f *Forum) error { + _, err := s.resetTopicCount.Exec(f.ID, f.ID) + if err != nil { + return err + } + count++ + return nil + }) + return count, err +} + func (s *DefaultRecalc) Subscriptions() (count int, err error) { err = eachall(s.getActivitySubscriptions, func(r *sql.Rows) error { var targetID int diff --git a/common/thaw.go b/common/thaw.go index c57f5473..8f39d894 100644 --- a/common/thaw.go +++ b/common/thaw.go @@ -62,5 +62,5 @@ func (t *DefaultThaw) Thawed() bool { } func (t *DefaultThaw) Thaw() { - atomic.StoreInt64(&t.thawed, 4) + atomic.StoreInt64(&t.thawed, 3) } diff --git a/common/topic_list.go b/common/topic_list.go index 64ad68bc..c66b9f67 100644 --- a/common/topic_list.go +++ b/common/topic_list.go @@ -44,6 +44,11 @@ type DefaultTopicList struct { forums map[int]*ForumTopicListHolder forumLock sync.RWMutex + qcounts map[int]*sql.Stmt + qcounts2 map[int]*sql.Stmt + qLock sync.RWMutex + qLock2 sync.RWMutex + //permTree atomic.Value // [string(canSee)]canSee //permTree map[string][]int // [string(canSee)]canSee @@ -59,6 +64,8 @@ func NewDefaultTopicList(acc *qgen.Accumulator) (*DefaultTopicList, error) { oddGroups: make(map[int]*TopicListHolder), evenGroups: make(map[int]*TopicListHolder), forums: make(map[int]*ForumTopicListHolder), + qcounts: make(map[int]*sql.Stmt), + qcounts2: make(map[int]*sql.Stmt), getTopicsByForum: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(), //getTidsByForum: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(), } @@ -119,12 +126,16 @@ func (tList *DefaultTopicList) Tick() error { } canSeeHolders := make(map[string]*TopicListHolder) + forumCounts := make(map[int]int) for name, canSee := range permTree { topicList, forumList, pagi, err := tList.GetListByCanSee(canSee, 1, 0, nil) if err != nil { return err } canSeeHolders[name] = &TopicListHolder{topicList, forumList, pagi} + if len(canSee) > 1 { + forumCounts[len(canSee)] += 1 + } } for gid, canSee := range gidToCanSee { addList(gid, canSeeHolders[canSee]) @@ -138,26 +149,80 @@ func (tList *DefaultTopicList) Tick() error { tList.evenGroups = evenLists tList.evenLock.Unlock() + topc := []int{0, 0, 0, 0, 0, 0} + addC := func(c int) { + lowI, low := 0, topc[0] + for i, top := range topc { + if top < low { + lowI = i + low = top + } + } + if c > low { + topc[lowI] = c + } + } + for forumCount := range forumCounts { + addC(forumCount) + } + + qcounts := make(map[int]*sql.Stmt) + qcounts2 := make(map[int]*sql.Stmt) + for _, top := range topc { + if top == 0 { + continue + } + + var qlist string + for i := 0; i < top; i++ { + if i != 0 { + qlist += "," + } + qlist += "?" + } + cols := "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data" + + stmt, err := qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "views DESC,lastReplyAt DESC,createdBy DESC", "?,?") + if err != nil { + return err + } + qcounts[top] = stmt + + stmt, err = qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "sticky DESC,lastReplyAt DESC,createdBy DESC", "?,?") + if err != nil { + return err + } + qcounts2[top] = stmt + } + + tList.qLock.Lock() + tList.qcounts = qcounts + tList.qLock.Unlock() + + tList.qLock2.Lock() + tList.qcounts2 = qcounts2 + tList.qLock2.Unlock() + forums, err := Forums.GetAll() if err != nil { return err } - top5 := []*Forum{nil, nil, nil, nil, nil} + top8 := []*Forum{nil, nil, nil, nil, nil, nil, nil, nil} z := true addScore2 := func(f *Forum) { - for i, top := range top5 { + for i, top := range top8 { if top.TopicCount < f.TopicCount { - top5[i] = f + top8[i] = f return } } } addScore := func(f *Forum) { if z { - for i, top := range top5 { + for i, top := range top8 { if top == nil { - top5[i] = f + top8[i] = f return } } @@ -178,7 +243,7 @@ func (tList *DefaultTopicList) Tick() error { } addScore(f) } - for _, f := range top5 { + for _, f := range top8 { if f != nil { fshort = append(fshort, f) } @@ -209,6 +274,11 @@ func (tList *DefaultTopicList) GetListByForum(f *Forum, page, orderby int) (topi if page == 0 { page = 1 } + if f.TopicCount == 0 { + _, page, lastPage := PageOffset(f.TopicCount, page, Config.ItemsPerPage) + pageList := Paginate(page, lastPage, 5) + return topicList, Paginator{pageList, page, lastPage}, nil + } if page == 1 && orderby == 0 { var h *ForumTopicListHolder var ok bool @@ -419,21 +489,33 @@ func (tList *DefaultTopicList) GetList(page, orderby int, filterIDs []int) (topi func (tList *DefaultTopicList) getList(page, orderby, topicCount int, argList []interface{}, qlist string) (topicList []*TopicsRow, paginator Paginator, err error) { //log.Printf("argList: %+v\n",argList) //log.Printf("qlist: %+v\n",qlist) - var orderq string + var stmt *sql.Stmt if orderby == TopicListMostViewed { - orderq = "views DESC,lastReplyAt DESC,createdBy DESC" + tList.qLock.RLock() + stmt = tList.qcounts[len(argList)-2] + tList.qLock.RUnlock() + if stmt == nil { + orderq = "views DESC,lastReplyAt DESC,createdBy DESC" + } } else { - orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC" + tList.qLock2.RLock() + stmt = tList.qcounts2[len(argList)-2] + tList.qLock2.RUnlock() + if stmt == nil { + orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC" + } } offset, page, lastPage := PageOffset(topicCount, page, Config.ItemsPerPage) // TODO: Prepare common qlist lengths to speed this up in common cases, prepared statements are prepared lazily anyway, so it probably doesn't matter if we do ten or so - stmt, err := qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?") - if err != nil { - return nil, Paginator{nil, 1, 1}, err + if stmt == nil { + stmt, err = qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?") + if err != nil { + return nil, Paginator{nil, 1, 1}, err + } + defer stmt.Close() } - defer stmt.Close() argList = append(argList, offset) argList = append(argList, Config.ItemsPerPage) diff --git a/query_gen/querygen.go b/query_gen/querygen.go index cf71ca0b..af8a402a 100644 --- a/query_gen/querygen.go +++ b/query_gen/querygen.go @@ -131,27 +131,27 @@ type Adapter interface { DbVersion() string DropTable(name, table string) (string, error) - CreateTable(name, table, charset, collation string, columns []DBTableColumn, keys []DBTableKey) (string, error) + CreateTable(name, table, charset, collation string, cols []DBTableColumn, keys []DBTableKey) (string, error) // TODO: Some way to add indices and keys // TODO: Test this - AddColumn(name, table string, column DBTableColumn, key *DBTableKey) (string, error) + AddColumn(name, table string, col DBTableColumn, key *DBTableKey) (string, error) DropColumn(name, table, colname string) (string, error) RenameColumn(name, table, oldName, newName string) (string, error) ChangeColumn(name, table, colName string, col DBTableColumn) (string, error) SetDefaultColumn(name, table, colName, colType, defaultStr string) (string, error) AddIndex(name, table, iname, colname string) (string, error) - AddKey(name, table, column string, key DBTableKey) (string, error) - RemoveIndex(name, table, column string) (string, error) - AddForeignKey(name, table, column, ftable, fcolumn string, cascade bool) (out string, e error) - SimpleInsert(name, table, columns, fields string) (string, error) - SimpleBulkInsert(name, table, columns string, fieldSet []string) (string, error) + AddKey(name, table, col string, key DBTableKey) (string, error) + RemoveIndex(name, table, col string) (string, error) + AddForeignKey(name, table, col, ftable, fcolumn string, cascade bool) (out string, e error) + SimpleInsert(name, table, cols, fields string) (string, error) + SimpleBulkInsert(name, table, cols string, fieldSet []string) (string, error) SimpleUpdate(b *updatePrebuilder) (string, error) SimpleUpdateSelect(b *updatePrebuilder) (string, error) // ! Experimental SimpleDelete(name, table, where string) (string, error) Purge(name, table string) (string, error) - SimpleSelect(name, table, columns, where, orderby, limit string) (string, error) + SimpleSelect(name, table, cols, where, orderby, limit string) (string, error) ComplexDelete(b *deletePrebuilder) (string, error) - SimpleLeftJoin(name, table1, table2, columns, joiners, where, orderby, limit string) (string, error) + SimpleLeftJoin(name, table1, table2, cols, joiners, where, orderby, limit string) (string, error) SimpleInnerJoin(string, string, string, string, string, string, string, string) (string, error) SimpleInsertSelect(string, DBInsert, DBSelect) (string, error) SimpleInsertLeftJoin(string, DBInsert, DBJoin) (string, error) diff --git a/templates/paginator.html b/templates/paginator.html index 8a604c19..3a38c457 100644 --- a/templates/paginator.html +++ b/templates/paginator.html @@ -2,12 +2,12 @@
{{if gt .Page 1}}
{{lang "paginator.first_page"}}
- {{end}} + {{end}} {{range .PageList}}
{{.}}
{{end}} {{if ne .LastPage .Page}} - +
{{lang "paginator.last_page"}}
{{end}}
diff --git a/tickloop.go b/tickloop.go index 78d66fb9..527b7fb1 100644 --- a/tickloop.go +++ b/tickloop.go @@ -267,6 +267,12 @@ func sched() error { } log.Printf("Deleted %d orphaned replies.", count) + count, err = c.Recalc.Forums() + if err != nil { + return errors.WithStack(err) + } + log.Printf("Recalculated %d forum topic counts.", count) + count, err = c.Recalc.Subscriptions() if err != nil { return errors.WithStack(err)