optimise topic lists by caching common qcounts for getList stmts

recalc forum topic counts
reduce thaw period to 3
cache top 8 forums instead of 5
optimise GetListByForum with zero topics
add Each method to ForumStore
This commit is contained in:
Azareal 2020-03-09 13:51:44 +10:00
parent 42a95d8597
commit b04d77d7b6
7 changed files with 178 additions and 59 deletions

View File

@ -28,6 +28,7 @@ var ErrNoDeleteReports = errors.New("You cannot delete the Reports forum")
// ForumStore is an interface for accessing the forums and the metadata stored on them
type ForumStore interface {
LoadForums() error
Each(h func(*Forum) error) error
DirtyGet(id int) *Forum
Get(id int) (*Forum, error)
BypassGet(id int) (*Forum, error)
@ -35,10 +36,10 @@ type ForumStore interface {
Reload(id int) error // ? - Should we move this to ForumCache? It might require us to do some unnecessary casting though
//Update(Forum) error
Delete(id int) error
AddTopic(tid int, uid int, fid int) error
AddTopic(tid, uid, fid int) error
RemoveTopic(fid int) error
RemoveTopics(fid, count int) error
UpdateLastTopic(tid int, uid int, fid int) error
UpdateLastTopic(tid, uid, fid int) error
Exists(id int) bool
GetAll() ([]*Forum, error)
GetAllIDs() ([]int, error)
@ -46,7 +47,7 @@ type ForumStore interface {
GetAllVisibleIDs() ([]int, error)
//GetChildren(parentID int, parentType string) ([]*Forum,error)
//GetFirstChild(parentID int, parentType string) (*Forum,error)
Create(name string, desc string, active bool, preset string) (int, error)
Create(name, desc string, active bool, preset string) (int, error)
UpdateOrder(updateMap map[int]int) error
Count() int
@ -82,16 +83,16 @@ func NewMemoryForumStore() (*MemoryForumStore, error) {
f := "forums"
// TODO: Do a proper delete
return &MemoryForumStore{
get: acc.Select(f).Columns("name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Where("fid = ?").Prepare(),
get: acc.Select(f).Columns("name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Where("fid=?").Prepare(),
getAll: acc.Select(f).Columns("fid, name, desc, tmpl, active, order, preset, parentID, parentType, topicCount, lastTopicID, lastReplyerID").Orderby("order ASC, fid ASC").Prepare(),
delete: acc.Update(f).Set("name='',active=0").Where("fid = ?").Prepare(),
delete: acc.Update(f).Set("name='',active=0").Where("fid=?").Prepare(),
create: acc.Insert(f).Columns("name, desc, tmpl, active, preset").Fields("?,?,'',?,?").Prepare(),
count: acc.Count(f).Where("name != ''").Prepare(),
updateCache: acc.Update(f).Set("lastTopicID = ?, lastReplyerID = ?").Where("fid = ?").Prepare(),
addTopics: acc.Update(f).Set("topicCount=topicCount+?").Where("fid = ?").Prepare(),
removeTopics: acc.Update(f).Set("topicCount=topicCount-?").Where("fid = ?").Prepare(),
lastTopic: acc.Select("topics").Columns("tid").Where("parentID = ?").Orderby("lastReplyAt DESC, createdAt DESC").Limit("1").Prepare(),
updateOrder: acc.Update(f).Set("order = ?").Where("fid = ?").Prepare(),
updateCache: acc.Update(f).Set("lastTopicID=?, lastReplyerID=?").Where("fid=?").Prepare(),
addTopics: acc.Update(f).Set("topicCount=topicCount+?").Where("fid=?").Prepare(),
removeTopics: acc.Update(f).Set("topicCount=topicCount-?").Where("fid=?").Prepare(),
lastTopic: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("lastReplyAt DESC, createdAt DESC").Limit("1").Prepare(),
updateOrder: acc.Update(f).Set("order=?").Where("fid=?").Prepare(),
}, acc.FirstError()
}
@ -99,10 +100,10 @@ func NewMemoryForumStore() (*MemoryForumStore, error) {
// TODO: Add support for subforums
func (s *MemoryForumStore) LoadForums() error {
var forumView []*Forum
addForum := func(forum *Forum) {
s.forums.Store(forum.ID, forum)
if forum.Active && forum.Name != "" && forum.ParentType == "" {
forumView = append(forumView, forum)
addForum := func(f *Forum) {
s.forums.Store(f.ID, f)
if f.Active && f.Name != "" && f.ParentType == "" {
forumView = append(forumView, f)
}
}
@ -140,11 +141,11 @@ func (s *MemoryForumStore) LoadForums() error {
// ? - Will this be hit a lot by plugin_guilds?
func (s *MemoryForumStore) rebuildView() {
var forumView []*Forum
s.forums.Range(func(_ interface{}, value interface{}) bool {
forum := value.(*Forum)
s.forums.Range(func(_, val interface{}) bool {
f := val.(*Forum)
// ? - ParentType blank means that it doesn't have a parent
if forum.Active && forum.Name != "" && forum.ParentType == "" {
forumView = append(forumView, forum)
if f.Active && f.Name != "" && f.ParentType == "" {
forumView = append(forumView, f)
}
return true
})
@ -153,6 +154,17 @@ func (s *MemoryForumStore) rebuildView() {
TopicListThaw.Thaw()
}
func (s *MemoryForumStore) Each(h func(*Forum) error) (err error) {
s.forums.Range(func(_, val interface{}) bool {
err = h(val.(*Forum))
if err != nil {
return false
}
return true
})
return err
}
func (s *MemoryForumStore) DirtyGet(id int) *Forum {
fint, ok := s.forums.Load(id)
if !ok || fint.(*Forum).Name == "" {
@ -208,11 +220,11 @@ func (s *MemoryForumStore) BypassGet(id int) (*Forum, error) {
func (s *MemoryForumStore) BulkGetCopy(ids []int) (forums []Forum, err error) {
forums = make([]Forum, len(ids))
for i, id := range ids {
forum, err := s.Get(id)
f, err := s.Get(id)
if err != nil {
return nil, err
}
forums[i] = forum.Copy()
forums[i] = f.Copy()
}
return forums, nil
}
@ -226,16 +238,16 @@ func (s *MemoryForumStore) Reload(id int) error {
return nil
}
func (s *MemoryForumStore) CacheSet(forum *Forum) error {
s.forums.Store(forum.ID, forum)
func (s *MemoryForumStore) CacheSet(f *Forum) error {
s.forums.Store(f.ID, f)
s.rebuildView()
return nil
}
// ! Has a randomised order
func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) {
s.forums.Range(func(_ interface{}, value interface{}) bool {
forumView = append(forumView, value.(*Forum))
s.forums.Range(func(_, val interface{}) bool {
forumView = append(forumView, val.(*Forum))
return true
})
sort.Sort(SortForum(forumView))
@ -244,8 +256,8 @@ func (s *MemoryForumStore) GetAll() (forumView []*Forum, err error) {
// ? - Can we optimise the sorting?
func (s *MemoryForumStore) GetAllIDs() (ids []int, err error) {
s.forums.Range(func(_ interface{}, value interface{}) bool {
ids = append(ids, value.(*Forum).ID)
s.forums.Range(func(_, val interface{}) bool {
ids = append(ids, val.(*Forum).ID)
return true
})
sort.Ints(ids)
@ -299,7 +311,7 @@ func (s *MemoryForumStore) Delete(id int) error {
return err
}
func (s *MemoryForumStore) AddTopic(tid int, uid int, fid int) error {
func (s *MemoryForumStore) AddTopic(tid, uid, fid int) error {
_, err := s.updateCache.Exec(tid, uid, fid)
if err != nil {
return err
@ -351,7 +363,7 @@ func (s *MemoryForumStore) RemoveTopic(fid int) error {
}
return s.RefreshTopic(fid)
}
func (s *MemoryForumStore) RemoveTopics(fid int, count int) error {
func (s *MemoryForumStore) RemoveTopics(fid, count int) error {
_, err := s.removeTopics.Exec(count, fid)
if err != nil {
return err
@ -361,7 +373,7 @@ func (s *MemoryForumStore) RemoveTopics(fid int, count int) error {
// DEPRECATED. forum.Update() will be the way to do this in the future, once it's completed
// TODO: Have a pointer to the last topic rather than storing it on the forum itself
func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error {
func (s *MemoryForumStore) UpdateLastTopic(tid, uid, fid int) error {
_, err := s.updateCache.Exec(tid, uid, fid)
if err != nil {
return err
@ -370,7 +382,7 @@ func (s *MemoryForumStore) UpdateLastTopic(tid int, uid int, fid int) error {
return s.Reload(fid)
}
func (s *MemoryForumStore) Create(name string, desc string, active bool, preset string) (int, error) {
func (s *MemoryForumStore) Create(name, desc string, active bool, preset string) (int, error) {
if name == "" {
return 0, ErrBlankName
}
@ -410,12 +422,12 @@ func (s *MemoryForumStore) UpdateOrder(updateMap map[int]int) error {
// ! Might be slightly inaccurate, if the sync.Map is constantly shifting and churning, but it'll stabilise eventually. Also, slow. Don't use this on every request x.x
// Length returns the number of forums in the memory cache
func (s *MemoryForumStore) Length() (length int) {
s.forums.Range(func(_ interface{}, value interface{}) bool {
length++
func (s *MemoryForumStore) Length() (len int) {
s.forums.Range(func(_, _ interface{}) bool {
len++
return true
})
return length
return len
}
// TODO: Get the total count of forums in the forum store rather than doing a heavy query for this?

View File

@ -12,6 +12,7 @@ var Recalc RecalcInt
type RecalcInt interface {
Replies() (count int, err error)
Forums() (count int, err error)
Subscriptions() (count int, err error)
ActivityStream() (count int, err error)
Users() error
@ -22,6 +23,8 @@ type DefaultRecalc struct {
getActivitySubscriptions *sql.Stmt
getActivityStream *sql.Stmt
getAttachments *sql.Stmt
getTopicCount *sql.Stmt
resetTopicCount *sql.Stmt
}
func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) {
@ -29,6 +32,10 @@ func NewDefaultRecalc(acc *qgen.Accumulator) (*DefaultRecalc, error) {
getActivitySubscriptions: acc.Select("activity_subscriptions").Columns("targetID,targetType").Prepare(),
getActivityStream: acc.Select("activity_stream").Columns("asid,event,elementID,elementType,extra").Prepare(),
getAttachments: acc.Select("attachments").Columns("attachID,originID,originTable").Prepare(),
getTopicCount: acc.Count("topics").Where("parentID=?").Prepare(),
//resetTopicCount: acc.SimpleUpdateSelect("forums", "topicCount = tc", "topics", "count(*) as tc", "parentID=?", "", ""),
// TODO: Avoid using RawPrepare
resetTopicCount: acc.RawPrepare("UPDATE forums, (SELECT COUNT(*) as tc FROM topics WHERE parentID=?) AS src SET forums.topicCount=src.tc WHERE forums.fid=?"),
}, acc.FirstError()
}
@ -50,6 +57,18 @@ func (s *DefaultRecalc) Replies() (count int, err error) {
return count, err
}
func (s *DefaultRecalc) Forums() (count int, err error) {
err = Forums.Each(func(f *Forum) error {
_, err := s.resetTopicCount.Exec(f.ID, f.ID)
if err != nil {
return err
}
count++
return nil
})
return count, err
}
func (s *DefaultRecalc) Subscriptions() (count int, err error) {
err = eachall(s.getActivitySubscriptions, func(r *sql.Rows) error {
var targetID int

View File

@ -62,5 +62,5 @@ func (t *DefaultThaw) Thawed() bool {
}
func (t *DefaultThaw) Thaw() {
atomic.StoreInt64(&t.thawed, 4)
atomic.StoreInt64(&t.thawed, 3)
}

View File

@ -44,6 +44,11 @@ type DefaultTopicList struct {
forums map[int]*ForumTopicListHolder
forumLock sync.RWMutex
qcounts map[int]*sql.Stmt
qcounts2 map[int]*sql.Stmt
qLock sync.RWMutex
qLock2 sync.RWMutex
//permTree atomic.Value // [string(canSee)]canSee
//permTree map[string][]int // [string(canSee)]canSee
@ -59,6 +64,8 @@ func NewDefaultTopicList(acc *qgen.Accumulator) (*DefaultTopicList, error) {
oddGroups: make(map[int]*TopicListHolder),
evenGroups: make(map[int]*TopicListHolder),
forums: make(map[int]*ForumTopicListHolder),
qcounts: make(map[int]*sql.Stmt),
qcounts2: make(map[int]*sql.Stmt),
getTopicsByForum: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
//getTidsByForum: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
}
@ -119,12 +126,16 @@ func (tList *DefaultTopicList) Tick() error {
}
canSeeHolders := make(map[string]*TopicListHolder)
forumCounts := make(map[int]int)
for name, canSee := range permTree {
topicList, forumList, pagi, err := tList.GetListByCanSee(canSee, 1, 0, nil)
if err != nil {
return err
}
canSeeHolders[name] = &TopicListHolder{topicList, forumList, pagi}
if len(canSee) > 1 {
forumCounts[len(canSee)] += 1
}
}
for gid, canSee := range gidToCanSee {
addList(gid, canSeeHolders[canSee])
@ -138,26 +149,80 @@ func (tList *DefaultTopicList) Tick() error {
tList.evenGroups = evenLists
tList.evenLock.Unlock()
topc := []int{0, 0, 0, 0, 0, 0}
addC := func(c int) {
lowI, low := 0, topc[0]
for i, top := range topc {
if top < low {
lowI = i
low = top
}
}
if c > low {
topc[lowI] = c
}
}
for forumCount := range forumCounts {
addC(forumCount)
}
qcounts := make(map[int]*sql.Stmt)
qcounts2 := make(map[int]*sql.Stmt)
for _, top := range topc {
if top == 0 {
continue
}
var qlist string
for i := 0; i < top; i++ {
if i != 0 {
qlist += ","
}
qlist += "?"
}
cols := "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data"
stmt, err := qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "views DESC,lastReplyAt DESC,createdBy DESC", "?,?")
if err != nil {
return err
}
qcounts[top] = stmt
stmt, err = qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "sticky DESC,lastReplyAt DESC,createdBy DESC", "?,?")
if err != nil {
return err
}
qcounts2[top] = stmt
}
tList.qLock.Lock()
tList.qcounts = qcounts
tList.qLock.Unlock()
tList.qLock2.Lock()
tList.qcounts2 = qcounts2
tList.qLock2.Unlock()
forums, err := Forums.GetAll()
if err != nil {
return err
}
top5 := []*Forum{nil, nil, nil, nil, nil}
top8 := []*Forum{nil, nil, nil, nil, nil, nil, nil, nil}
z := true
addScore2 := func(f *Forum) {
for i, top := range top5 {
for i, top := range top8 {
if top.TopicCount < f.TopicCount {
top5[i] = f
top8[i] = f
return
}
}
}
addScore := func(f *Forum) {
if z {
for i, top := range top5 {
for i, top := range top8 {
if top == nil {
top5[i] = f
top8[i] = f
return
}
}
@ -178,7 +243,7 @@ func (tList *DefaultTopicList) Tick() error {
}
addScore(f)
}
for _, f := range top5 {
for _, f := range top8 {
if f != nil {
fshort = append(fshort, f)
}
@ -209,6 +274,11 @@ func (tList *DefaultTopicList) GetListByForum(f *Forum, page, orderby int) (topi
if page == 0 {
page = 1
}
if f.TopicCount == 0 {
_, page, lastPage := PageOffset(f.TopicCount, page, Config.ItemsPerPage)
pageList := Paginate(page, lastPage, 5)
return topicList, Paginator{pageList, page, lastPage}, nil
}
if page == 1 && orderby == 0 {
var h *ForumTopicListHolder
var ok bool
@ -419,21 +489,33 @@ func (tList *DefaultTopicList) GetList(page, orderby int, filterIDs []int) (topi
func (tList *DefaultTopicList) getList(page, orderby, topicCount int, argList []interface{}, qlist string) (topicList []*TopicsRow, paginator Paginator, err error) {
//log.Printf("argList: %+v\n",argList)
//log.Printf("qlist: %+v\n",qlist)
var orderq string
var stmt *sql.Stmt
if orderby == TopicListMostViewed {
orderq = "views DESC,lastReplyAt DESC,createdBy DESC"
tList.qLock.RLock()
stmt = tList.qcounts[len(argList)-2]
tList.qLock.RUnlock()
if stmt == nil {
orderq = "views DESC,lastReplyAt DESC,createdBy DESC"
}
} else {
orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC"
tList.qLock2.RLock()
stmt = tList.qcounts2[len(argList)-2]
tList.qLock2.RUnlock()
if stmt == nil {
orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC"
}
}
offset, page, lastPage := PageOffset(topicCount, page, Config.ItemsPerPage)
// TODO: Prepare common qlist lengths to speed this up in common cases, prepared statements are prepared lazily anyway, so it probably doesn't matter if we do ten or so
stmt, err := qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?")
if err != nil {
return nil, Paginator{nil, 1, 1}, err
if stmt == nil {
stmt, err = qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?")
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
defer stmt.Close()
}
defer stmt.Close()
argList = append(argList, offset)
argList = append(argList, Config.ItemsPerPage)

View File

@ -131,27 +131,27 @@ type Adapter interface {
DbVersion() string
DropTable(name, table string) (string, error)
CreateTable(name, table, charset, collation string, columns []DBTableColumn, keys []DBTableKey) (string, error)
CreateTable(name, table, charset, collation string, cols []DBTableColumn, keys []DBTableKey) (string, error)
// TODO: Some way to add indices and keys
// TODO: Test this
AddColumn(name, table string, column DBTableColumn, key *DBTableKey) (string, error)
AddColumn(name, table string, col DBTableColumn, key *DBTableKey) (string, error)
DropColumn(name, table, colname string) (string, error)
RenameColumn(name, table, oldName, newName string) (string, error)
ChangeColumn(name, table, colName string, col DBTableColumn) (string, error)
SetDefaultColumn(name, table, colName, colType, defaultStr string) (string, error)
AddIndex(name, table, iname, colname string) (string, error)
AddKey(name, table, column string, key DBTableKey) (string, error)
RemoveIndex(name, table, column string) (string, error)
AddForeignKey(name, table, column, ftable, fcolumn string, cascade bool) (out string, e error)
SimpleInsert(name, table, columns, fields string) (string, error)
SimpleBulkInsert(name, table, columns string, fieldSet []string) (string, error)
AddKey(name, table, col string, key DBTableKey) (string, error)
RemoveIndex(name, table, col string) (string, error)
AddForeignKey(name, table, col, ftable, fcolumn string, cascade bool) (out string, e error)
SimpleInsert(name, table, cols, fields string) (string, error)
SimpleBulkInsert(name, table, cols string, fieldSet []string) (string, error)
SimpleUpdate(b *updatePrebuilder) (string, error)
SimpleUpdateSelect(b *updatePrebuilder) (string, error) // ! Experimental
SimpleDelete(name, table, where string) (string, error)
Purge(name, table string) (string, error)
SimpleSelect(name, table, columns, where, orderby, limit string) (string, error)
SimpleSelect(name, table, cols, where, orderby, limit string) (string, error)
ComplexDelete(b *deletePrebuilder) (string, error)
SimpleLeftJoin(name, table1, table2, columns, joiners, where, orderby, limit string) (string, error)
SimpleLeftJoin(name, table1, table2, cols, joiners, where, orderby, limit string) (string, error)
SimpleInnerJoin(string, string, string, string, string, string, string, string) (string, error)
SimpleInsertSelect(string, DBInsert, DBSelect) (string, error)
SimpleInsertLeftJoin(string, DBInsert, DBJoin) (string, error)

View File

@ -2,12 +2,12 @@
<div class="pageset">
{{if gt .Page 1}}<div class="pageitem pagefirst"><a href="?page=1" aria-label="{{lang "paginator.first_page_aria"}}">{{lang "paginator.first_page"}}</a></div>
<div class="pageitem pageprev"><a href="?page={{subtract .Page 1}}" rel="prev" aria-label="{{lang "paginator.prev_page_aria"}}">{{lang "paginator.prev_page"}}</a></div>
<link rel="prev" href="?page={{subtract .Page 1}}" />{{end}}
<link rel="prev" href="?page={{subtract .Page 1}}"/>{{end}}
{{range .PageList}}
<div class="pageitem{{if eq . $.Page}} pagecurrent{{end}}"><a href="?page={{.}}">{{.}}</a></div>
{{end}}
{{if ne .LastPage .Page}}
<link rel="next" href="?page={{add .Page 1}}" />
<link rel="next" href="?page={{add .Page 1}}"/>
<div class="pageitem pagenext"><a href="?page={{add .Page 1}}" rel="next" aria-label="{{lang "paginator.next_page_aria"}}">{{lang "paginator.next_page"}}</a></div>
<div class="pageitem pagelast"><a href="?page={{.LastPage}}" aria-label="{{lang "paginator.last_page_aria"}}">{{lang "paginator.last_page"}}</a></div>{{end}}
</div>

View File

@ -267,6 +267,12 @@ func sched() error {
}
log.Printf("Deleted %d orphaned replies.", count)
count, err = c.Recalc.Forums()
if err != nil {
return errors.WithStack(err)
}
log.Printf("Recalculated %d forum topic counts.", count)
count, err = c.Recalc.Subscriptions()
if err != nil {
return errors.WithStack(err)