optimise topic lists by caching common qcounts for getList stmts

recalc forum topic counts
reduce thaw period to 3
cache top 8 forums instead of 5
optimise GetListByForum with zero topics
add Each method to ForumStore

common/forum_store.go

@@ -28,6 +28,7 @@ var ErrNoDeleteReports = errors.New("You cannot delete the Reports forum")
 // ForumStore is an interface for accessing the forums and the metadata stored on them
 type ForumStore interface {
 	LoadForums() error
+	Each(h func(*Forum) error) error
 	DirtyGet(id int) *Forum
 	Get(id int) (*Forum, error)
 	BypassGet(id int) (*Forum, error)
@@ -35,10 +36,10 @@ type ForumStore interface {
 	Reload(id int) error // ? - Should we move this to ForumCache? It might require us to do some unnecessary casting though
 	//Update(Forum) error
 	Delete(id int) error
-	AddTopic(tid int, uid int, fid int) error
+	AddTopic(tid, uid, fid int) error
 	RemoveTopic(fid int) error
 	RemoveTopics(fid, count int) error
-	UpdateLastTopic(tid int, uid int, fid int) error
+	UpdateLastTopic(tid, uid, fid int) error
 	Exists(id int) bool
 	GetAll() ([]*Forum, error)
 	GetAllIDs() ([]int, error)
@@ -46,7 +47,7 @@ type ForumStore interface {
 	GetAllVisibleIDs() ([]int, error)
 	//GetChildren(parentID int, parentType string) ([]*Forum,error)
 	//GetFirstChild(parentID int, parentType string) (*Forum,error)
-	Create(name string, desc string, active bool, preset string) (int, error)
+	Create(name, desc string, active bool, preset string) (int, error)
 	UpdateOrder(updateMap map[int]int) error
 
 	Count() int
@@ -99,10 +100,10 @@ func NewMemoryForumStore() (*MemoryForumStore, error) {
 // TODO: Add support for subforums
 func (s *MemoryForumStore) LoadForums() error {
 	var forumView []*Forum
-	addForum := func(forum *Forum) {
+	addForum := func(f *Forum) {
-		s.forums.Store(forum.ID, forum)
+		s.forums.Store(f.ID, f)
-		if forum.Active && forum.Name != "" && forum.ParentType == "" {
+		if f.Active && f.Name != "" && f.ParentType == "" {
-			forumView = append(forumView, forum)
+			forumView = append(forumView, f)
 		}
 	}
 
@@ -140,11 +141,11 @@ func (s *MemoryForumStore) LoadForums() error {
 // ? - Will this be hit a lot by plugin_guilds?
 func (s *MemoryForumStore) rebuildView() {
 	var forumView []*Forum
-	s.forums.Range(func(_ interface{}, value interface{}) bool {
+	s.forums.Range(func(_, val interface{}) bool {
-		forum := value.(*Forum)
+		f := val.(*Forum)
 		// ? - ParentType blank means that it doesn't have a parent
-		if forum.Active && forum.Name != "" && forum.ParentType == "" {
+		if f.Active && f.Name != "" && f.ParentType == "" {
-			forumView = append(forumView, forum)
+			forumView = append(forumView, f)
 		}
 		return true
 	})
@@ -153,6 +154,17 @@ func (s *MemoryForumStore) rebuildView() {
 	TopicListThaw.Thaw()
 }
 
+func (s *MemoryForumStore) Each(h func(*Forum) error) (err error) {
+	s.forums.Range(func(_, val interface{}) bool {
+		err = h(val.(*Forum))
+		if err != nil {
+			return false
+		}
+		return true
+	})
+	return err
+}
+
 func (s *MemoryForumStore) DirtyGet(id int) *Forum {
 	fint, ok := s.forums.Load(id)
 	if !ok || fint.(*Forum).Name == "" {
@@ -208,11 +220,11 @@ func (s *MemoryForumStore) BypassGet(id int) (*Forum, error) {
 func (s *MemoryForumStore) BulkGetCopy(ids []int) (forums []Forum, err error) {
 	forums = make([]Forum, len(ids))
 	for i, id := range ids {
-		forum, err := s.Get(id)
+		f, err := s.Get(id)
 		if err != nil {
 			return nil, err
 		}
-		forums[i] = forum.Copy()
+		forums[i] = f.Copy()
 	}
 	return forums, nil
 }
@@ -226,16 +238,16 @@ func (s *MemoryForumStore) Reload(id int) error {
 	return nil
 }
 
-func (s *MemoryForumStore) CacheSet(forum *Forum) error {
+func (s *MemoryForumStore) CacheSet(f *Forum) error {
-	s.forums.Store(forum.ID, forum)
+	s.forums.Store(f.ID, f)
 	s.rebuildView()
 	return nil
 }
 
 // ! Has a randomised order
 func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) {
-	s.forums.Range(func(_ interface{}, value interface{}) bool {
+	s.forums.Range(func(_, val interface{}) bool {
-		forumView = append(forumView, value.(*Forum))
+		forumView = append(forumView, val.(*Forum))
 		return true
 	})
 	sort.Sort(SortForum(forumView))
@@ -244,8 +256,8 @@ func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) {
 
 // ? - Can we optimise the sorting?
 func (s *MemoryForumStore) GetAllIDs() (ids []int, err error) {
-	s.forums.Range(func(_ interface{}, value interface{}) bool {
+	s.forums.Range(func(_, val interface{}) bool {
-		ids = append(ids, value.(*Forum).ID)
+		ids = append(ids, val.(*Forum).ID)
 		return true
 	})
 	sort.Ints(ids)
@@ -299,7 +311,7 @@ func (s *MemoryForumStore) Delete(id int) error {
 	return err
 }
 
-func (s *MemoryForumStore) AddTopic(tid int, uid int, fid int) error {
+func (s *MemoryForumStore) AddTopic(tid, uid, fid int) error {
 	_, err := s.updateCache.Exec(tid, uid, fid)
 	if err != nil {
 		return err
@@ -351,7 +363,7 @@ func (s *MemoryForumStore) RemoveTopic(fid int) error {
 	}
 	return s.RefreshTopic(fid)
 }
-func (s *MemoryForumStore) RemoveTopics(fid int, count int) error {
+func (s *MemoryForumStore) RemoveTopics(fid, count int) error {
 	_, err := s.removeTopics.Exec(count, fid)
 	if err != nil {
 		return err
@@ -361,7 +373,7 @@ func (s *MemoryForumStore) RemoveTopics(fid int, count int) error {
 
 // DEPRECATED. forum.Update() will be the way to do this in the future, once it's completed
 // TODO: Have a pointer to the last topic rather than storing it on the forum itself
-func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error {
+func (s *MemoryForumStore) UpdateLastTopic(tid, uid, fid int) error {
 	_, err := s.updateCache.Exec(tid, uid, fid)
 	if err != nil {
 		return err
@@ -370,7 +382,7 @@ func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error {
 	return s.Reload(fid)
 }
 
-func (s *MemoryForumStore) Create(name string, desc string, active bool, preset string) (int, error) {
+func (s *MemoryForumStore) Create(name, desc string, active bool, preset string) (int, error) {
 	if name == "" {
 		return 0, ErrBlankName
 	}
@@ -410,12 +422,12 @@ func (s *MemoryForumStore) UpdateOrder(updateMap map[int]int) error {
 
 // ! Might be slightly inaccurate, if the sync.Map is constantly shifting and churning, but it'll stabilise eventually. Also, slow. Don't use this on every request x.x
 // Length returns the number of forums in the memory cache
-func (s *MemoryForumStore) Length() (length int) {
+func (s *MemoryForumStore) Length() (len int) {
-	s.forums.Range(func(_ interface{}, value interface{}) bool {
+	s.forums.Range(func(_, _ interface{}) bool {
-		length++
+		len++
 		return true
 	})
-	return length
+	return len
 }
 
 // TODO: Get the total count of forums in the forum store rather than doing a heavy query for this?

common/recalc.go

@@ -12,6 +12,7 @@ var Recalc RecalcInt
 
 type RecalcInt interface {
 	Replies() (count int, err error)
+	Forums() (count int, err error)
 	Subscriptions() (count int, err error)
 	ActivityStream() (count int, err error)
 	Users() error
@@ -22,6 +23,8 @@ type DefaultRecalc struct {
 	getActivitySubscriptions *sql.Stmt
 	getActivityStream        *sql.Stmt
 	getAttachments           *sql.Stmt
+	getTopicCount            *sql.Stmt
+	resetTopicCount          *sql.Stmt
 }
 
 func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) {
@@ -29,6 +32,10 @@ func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) {
 		getActivitySubscriptions: acc.Select("activity_subscriptions").Columns("targetID,targetType").Prepare(),
 		getActivityStream:        acc.Select("activity_stream").Columns("asid,event,elementID,elementType,extra").Prepare(),
 		getAttachments:           acc.Select("attachments").Columns("attachID,originID,originTable").Prepare(),
+		getTopicCount:            acc.Count("topics").Where("parentID=?").Prepare(),
+		//resetTopicCount:          acc.SimpleUpdateSelect("forums", "topicCount = tc", "topics", "count(*) as tc", "parentID=?", "", ""),
+		// TODO: Avoid using RawPrepare
+		resetTopicCount: acc.RawPrepare("UPDATE forums, (SELECT COUNT(*) as tc FROM topics WHERE parentID=?) AS src SET forums.topicCount=src.tc WHERE forums.fid=?"),
 	}, acc.FirstError()
 }
 
@@ -50,6 +57,18 @@ func (s *DefaultRecalc) Replies() (count int, err error) {
 	return count, err
 }
 
+func (s *DefaultRecalc) Forums() (count int, err error) {
+	err = Forums.Each(func(f *Forum) error {
+		_, err := s.resetTopicCount.Exec(f.ID, f.ID)
+		if err != nil {
+			return err
+		}
+		count++
+		return nil
+	})
+	return count, err
+}
+
 func (s *DefaultRecalc) Subscriptions() (count int, err error) {
 	err = eachall(s.getActivitySubscriptions, func(r *sql.Rows) error {
 		var targetID int

common/thaw.go

@@ -62,5 +62,5 @@ func (t *DefaultThaw) Thawed() bool {
 }
 
 func (t *DefaultThaw) Thaw() {
-	atomic.StoreInt64(&t.thawed, 4)
+	atomic.StoreInt64(&t.thawed, 3)
 }

common/topic_list.go

@@ -44,6 +44,11 @@ type DefaultTopicList struct {
 	forums    map[int]*ForumTopicListHolder
 	forumLock sync.RWMutex
 
+	qcounts  map[int]*sql.Stmt
+	qcounts2 map[int]*sql.Stmt
+	qLock    sync.RWMutex
+	qLock2   sync.RWMutex
+
 	//permTree atomic.Value // [string(canSee)]canSee
 	//permTree map[string][]int // [string(canSee)]canSee
 
@@ -59,6 +64,8 @@ func NewDefaultTopicList(acc *qgen.Accumulator) (*DefaultTopicList, error) {
 		oddGroups:        make(map[int]*TopicListHolder),
 		evenGroups:       make(map[int]*TopicListHolder),
 		forums:           make(map[int]*ForumTopicListHolder),
+		qcounts:          make(map[int]*sql.Stmt),
+		qcounts2:         make(map[int]*sql.Stmt),
 		getTopicsByForum: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
 		//getTidsByForum: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
 	}
@@ -119,12 +126,16 @@ func (tList *DefaultTopicList) Tick() error {
 	}
 
 	canSeeHolders := make(map[string]*TopicListHolder)
+	forumCounts := make(map[int]int)
 	for name, canSee := range permTree {
 		topicList, forumList, pagi, err := tList.GetListByCanSee(canSee, 1, 0, nil)
 		if err != nil {
 			return err
 		}
 		canSeeHolders[name] = &TopicListHolder{topicList, forumList, pagi}
+		if len(canSee) > 1 {
+			forumCounts[len(canSee)] += 1
+		}
 	}
 	for gid, canSee := range gidToCanSee {
 		addList(gid, canSeeHolders[canSee])
@@ -138,26 +149,80 @@ func (tList *DefaultTopicList) Tick() error {
 	tList.evenGroups = evenLists
 	tList.evenLock.Unlock()
 
+	topc := []int{0, 0, 0, 0, 0, 0}
+	addC := func(c int) {
+		lowI, low := 0, topc[0]
+		for i, top := range topc {
+			if top < low {
+				lowI = i
+				low = top
+			}
+		}
+		if c > low {
+			topc[lowI] = c
+		}
+	}
+	for forumCount := range forumCounts {
+		addC(forumCount)
+	}
+
+	qcounts := make(map[int]*sql.Stmt)
+	qcounts2 := make(map[int]*sql.Stmt)
+	for _, top := range topc {
+		if top == 0 {
+			continue
+		}
+
+		var qlist string
+		for i := 0; i < top; i++ {
+			if i != 0 {
+				qlist += ","
+			}
+			qlist += "?"
+		}
+		cols := "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data"
+
+		stmt, err := qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "views DESC,lastReplyAt DESC,createdBy DESC", "?,?")
+		if err != nil {
+			return err
+		}
+		qcounts[top] = stmt
+
+		stmt, err = qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "sticky DESC,lastReplyAt DESC,createdBy DESC", "?,?")
+		if err != nil {
+			return err
+		}
+		qcounts2[top] = stmt
+	}
+
+	tList.qLock.Lock()
+	tList.qcounts = qcounts
+	tList.qLock.Unlock()
+
+	tList.qLock2.Lock()
+	tList.qcounts2 = qcounts2
+	tList.qLock2.Unlock()
+
 	forums, err := Forums.GetAll()
 	if err != nil {
 		return err
 	}
 
-	top5 := []*Forum{nil, nil, nil, nil, nil}
+	top8 := []*Forum{nil, nil, nil, nil, nil, nil, nil, nil}
 	z := true
 	addScore2 := func(f *Forum) {
-		for i, top := range top5 {
+		for i, top := range top8 {
 			if top.TopicCount < f.TopicCount {
-				top5[i] = f
+				top8[i] = f
 				return
 			}
 		}
 	}
 	addScore := func(f *Forum) {
 		if z {
-			for i, top := range top5 {
+			for i, top := range top8 {
 				if top == nil {
-					top5[i] = f
+					top8[i] = f
 					return
 				}
 			}
@@ -178,7 +243,7 @@ func (tList *DefaultTopicList) Tick() error {
 		}
 		addScore(f)
 	}
-	for _, f := range top5 {
+	for _, f := range top8 {
 		if f != nil {
 			fshort = append(fshort, f)
 		}
@@ -209,6 +274,11 @@ func (tList *DefaultTopicList) GetListByForum(f *Forum, page, orderby int) (topi
 	if page == 0 {
 		page = 1
 	}
+	if f.TopicCount == 0 {
+		_, page, lastPage := PageOffset(f.TopicCount, page, Config.ItemsPerPage)
+		pageList := Paginate(page, lastPage, 5)
+		return topicList, Paginator{pageList, page, lastPage}, nil
+	}
 	if page == 1 && orderby == 0 {
 		var h *ForumTopicListHolder
 		var ok bool
@@ -419,21 +489,33 @@ func (tList *DefaultTopicList) GetList(page, orderby int, filterIDs []int) (topi
 func (tList *DefaultTopicList) getList(page, orderby, topicCount int, argList []interface{}, qlist string) (topicList []*TopicsRow, paginator Paginator, err error) {
 	//log.Printf("argList: %+v\n",argList)
 	//log.Printf("qlist: %+v\n",qlist)
-
 	var orderq string
+	var stmt *sql.Stmt
 	if orderby == TopicListMostViewed {
+		tList.qLock.RLock()
+		stmt = tList.qcounts[len(argList)-2]
+		tList.qLock.RUnlock()
+		if stmt == nil {
 			orderq = "views DESC,lastReplyAt DESC,createdBy DESC"
+		}
 	} else {
+		tList.qLock2.RLock()
+		stmt = tList.qcounts2[len(argList)-2]
+		tList.qLock2.RUnlock()
+		if stmt == nil {
 			orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC"
 		}
+	}
 	offset, page, lastPage := PageOffset(topicCount, page, Config.ItemsPerPage)
 
 	// TODO: Prepare common qlist lengths to speed this up in common cases, prepared statements are prepared lazily anyway, so it probably doesn't matter if we do ten or so
-	stmt, err := qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?")
+	if stmt == nil {
+		stmt, err = qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?")
 		if err != nil {
 			return nil, Paginator{nil, 1, 1}, err
 		}
 		defer stmt.Close()
+	}
 
 	argList = append(argList, offset)
 	argList = append(argList, Config.ItemsPerPage)

query_gen/querygen.go

@@ -131,27 +131,27 @@ type Adapter interface {
 	DbVersion() string
 
 	DropTable(name, table string) (string, error)
-	CreateTable(name, table, charset, collation string, columns []DBTableColumn, keys []DBTableKey) (string, error)
+	CreateTable(name, table, charset, collation string, cols []DBTableColumn, keys []DBTableKey) (string, error)
 	// TODO: Some way to add indices and keys
 	// TODO: Test this
-	AddColumn(name, table string, column DBTableColumn, key *DBTableKey) (string, error)
+	AddColumn(name, table string, col DBTableColumn, key *DBTableKey) (string, error)
 	DropColumn(name, table, colname string) (string, error)
 	RenameColumn(name, table, oldName, newName string) (string, error)
 	ChangeColumn(name, table, colName string, col DBTableColumn) (string, error)
 	SetDefaultColumn(name, table, colName, colType, defaultStr string) (string, error)
 	AddIndex(name, table, iname, colname string) (string, error)
-	AddKey(name, table, column string, key DBTableKey) (string, error)
+	AddKey(name, table, col string, key DBTableKey) (string, error)
-	RemoveIndex(name, table, column string) (string, error)
+	RemoveIndex(name, table, col string) (string, error)
-	AddForeignKey(name, table, column, ftable, fcolumn string, cascade bool) (out string, e error)
+	AddForeignKey(name, table, col, ftable, fcolumn string, cascade bool) (out string, e error)
-	SimpleInsert(name, table, columns, fields string) (string, error)
+	SimpleInsert(name, table, cols, fields string) (string, error)
-	SimpleBulkInsert(name, table, columns string, fieldSet []string) (string, error)
+	SimpleBulkInsert(name, table, cols string, fieldSet []string) (string, error)
 	SimpleUpdate(b *updatePrebuilder) (string, error)
 	SimpleUpdateSelect(b *updatePrebuilder) (string, error) // ! Experimental
 	SimpleDelete(name, table, where string) (string, error)
 	Purge(name, table string) (string, error)
-	SimpleSelect(name, table, columns, where, orderby, limit string) (string, error)
+	SimpleSelect(name, table, cols, where, orderby, limit string) (string, error)
 	ComplexDelete(b *deletePrebuilder) (string, error)
-	SimpleLeftJoin(name, table1, table2, columns, joiners, where, orderby, limit string) (string, error)
+	SimpleLeftJoin(name, table1, table2, cols, joiners, where, orderby, limit string) (string, error)
 	SimpleInnerJoin(string, string, string, string, string, string, string, string) (string, error)
 	SimpleInsertSelect(string, DBInsert, DBSelect) (string, error)
 	SimpleInsertLeftJoin(string, DBInsert, DBJoin) (string, error)

tickloop.go

@@ -267,6 +267,12 @@ func sched() error {
 		}
 		log.Printf("Deleted %d orphaned replies.", count)
 
+		count, err = c.Recalc.Forums()
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		log.Printf("Recalculated %d forum topic counts.", count)
+
 		count, err = c.Recalc.Subscriptions()
 		if err != nil {
 			return errors.WithStack(err)