gosora/common/topic_list.go

679 lines
20 KiB
Go

package common
import (
//"log"
"database/sql"
"strconv"
"sync"
qgen "github.com/Azareal/Gosora/query_gen"
)
var TopicList TopicListInt
const (
TopicListDefault = iota
TopicListMostViewed
)
type TopicListHolder struct {
List []*TopicsRow
ForumList []Forum
Paginator Paginator
}
type ForumTopicListHolder struct {
List []*TopicsRow
Paginator Paginator
}
type TopicListInt interface {
GetListByCanSee(canSee []int, page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error)
GetListByGroup(g *Group, page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error)
GetListByForum(f *Forum, page, orderby int) (topicList []*TopicsRow, pagi Paginator, err error)
GetList(page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error)
}
type DefaultTopicList struct {
// TODO: Rewrite this to put permTree as the primary and put canSeeStr on each group?
oddGroups map[int][2]*TopicListHolder
evenGroups map[int][2]*TopicListHolder
oddLock sync.RWMutex
evenLock sync.RWMutex
forums map[int]*ForumTopicListHolder
forumLock sync.RWMutex
qcounts map[int]*sql.Stmt
qcounts2 map[int]*sql.Stmt
qLock sync.RWMutex
qLock2 sync.RWMutex
//permTree atomic.Value // [string(canSee)]canSee
//permTree map[string][]int // [string(canSee)]canSee
getTopicsByForum *sql.Stmt
//getTidsByForum *sql.Stmt
}
// We've removed the topic list cache cap as admins really shouldn't be abusing groups like this with plugin_guilds around and it was extremely fiddly.
// If this becomes a problem later on, then we can revisit this with a fresh perspective, particularly with regards to what people expect a group to really be
// Also, keep in mind that as-long as the groups don't all have unique sets of forums they can see, then we can optimise a large portion of the work away.
func NewDefaultTopicList(acc *qgen.Accumulator) (*DefaultTopicList, error) {
tList := &DefaultTopicList{
oddGroups: make(map[int][2]*TopicListHolder),
evenGroups: make(map[int][2]*TopicListHolder),
forums: make(map[int]*ForumTopicListHolder),
qcounts: make(map[int]*sql.Stmt),
qcounts2: make(map[int]*sql.Stmt),
getTopicsByForum: acc.Select("topics").Columns("tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,views,postCount,likeCount").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
//getTidsByForum: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(),
}
if err := acc.FirstError(); err != nil {
return nil, err
}
err := tList.Tick()
if err != nil {
return nil, err
}
AddScheduledHalfSecondTask(tList.Tick)
//AddScheduledSecondTask(tList.GroupCountTick) // TODO: Dynamically change the groups in the short list to be optimised every second
return tList, nil
}
func (tList *DefaultTopicList) Tick() error {
//fmt.Println("TopicList.Tick")
if !TopicListThaw.Thawed() {
return nil
}
//fmt.Println("building topic list")
oddLists := make(map[int][2]*TopicListHolder)
evenLists := make(map[int][2]*TopicListHolder)
addList := func(gid int, h [2]*TopicListHolder) {
if gid%2 == 0 {
evenLists[gid] = h
} else {
oddLists[gid] = h
}
}
allGroups, err := Groups.GetAll()
if err != nil {
return err
}
gidToCanSee := make(map[int]string)
permTree := make(map[string][]int) // [string(canSee)]canSee
for _, g := range allGroups {
// ? - Move the user count check to instance initialisation? Might require more book-keeping, particularly when a user moves into a zero user group
if g.UserCount == 0 && g.ID != GuestUser.Group {
continue
}
canSee := make([]byte, len(g.CanSee))
for i, item := range g.CanSee {
canSee[i] = byte(item)
}
canSeeInt := make([]int, len(canSee))
copy(canSeeInt, g.CanSee)
sCanSee := string(canSee)
permTree[sCanSee] = canSeeInt
gidToCanSee[g.ID] = sCanSee
}
canSeeHolders := make(map[string][2]*TopicListHolder)
forumCounts := make(map[int]int)
for name, canSee := range permTree {
topicList, forumList, pagi, err := tList.GetListByCanSee(canSee, 1, 0, nil)
if err != nil {
return err
}
topicList2, forumList2, pagi2, err := tList.GetListByCanSee(canSee, 2, 0, nil)
if err != nil {
return err
}
canSeeHolders[name] = [2]*TopicListHolder{
&TopicListHolder{topicList, forumList, pagi},
&TopicListHolder{topicList2, forumList2, pagi2},
}
if len(canSee) > 1 {
forumCounts[len(canSee)] += 1
}
}
for gid, canSee := range gidToCanSee {
addList(gid, canSeeHolders[canSee])
}
tList.oddLock.Lock()
tList.oddGroups = oddLists
tList.oddLock.Unlock()
tList.evenLock.Lock()
tList.evenGroups = evenLists
tList.evenLock.Unlock()
topc := []int{0, 0, 0, 0, 0, 0}
addC := func(c int) {
lowI, low := 0, topc[0]
for i, top := range topc {
if top < low {
lowI = i
low = top
}
}
if c > low {
topc[lowI] = c
}
}
for forumCount := range forumCounts {
addC(forumCount)
}
qcounts := make(map[int]*sql.Stmt)
qcounts2 := make(map[int]*sql.Stmt)
for _, top := range topc {
if top == 0 {
continue
}
var qlist string
for i := 0; i < top; i++ {
if i != 0 {
qlist += ","
}
qlist += "?"
}
cols := "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data"
stmt, err := qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "views DESC,lastReplyAt DESC,createdBy DESC", "?,?")
if err != nil {
return err
}
qcounts[top] = stmt
stmt, err = qgen.Builder.SimpleSelect("topics", cols, "parentID IN("+qlist+")", "sticky DESC,lastReplyAt DESC,createdBy DESC", "?,?")
if err != nil {
return err
}
qcounts2[top] = stmt
}
tList.qLock.Lock()
tList.qcounts = qcounts
tList.qLock.Unlock()
tList.qLock2.Lock()
tList.qcounts2 = qcounts2
tList.qLock2.Unlock()
forums, err := Forums.GetAll()
if err != nil {
return err
}
top8 := []*Forum{nil, nil, nil, nil, nil, nil, nil, nil}
z := true
addScore2 := func(f *Forum) {
for i, top := range top8 {
if top.TopicCount < f.TopicCount {
top8[i] = f
return
}
}
}
addScore := func(f *Forum) {
if z {
for i, top := range top8 {
if top == nil {
top8[i] = f
return
}
}
z = false
addScore2(f)
}
addScore2(f)
}
var fshort []*Forum
for _, f := range forums {
if f.Name == "" || !f.Active || (f.ParentType != "" && f.ParentType != "forum") {
continue
}
if f.TopicCount == 0 {
fshort = append(fshort, f)
continue
}
addScore(f)
}
for _, f := range top8 {
if f != nil {
fshort = append(fshort, f)
}
}
fList := make(map[int]*ForumTopicListHolder)
for _, f := range fshort {
topicList, pagi, err := tList.GetListByForum(f, 1, 0)
if err != nil {
return err
}
fList[f.ID] = &ForumTopicListHolder{topicList, pagi}
}
tList.forumLock.Lock()
tList.forums = fList
tList.forumLock.Unlock()
hTbl := GetHookTable()
_, _ = hTbl.VhookSkippable("tasks_tick_topic_list", tList)
return nil
}
// TODO: Add Topics() method to *Forum?
// TODO: Implement orderby
func (tList *DefaultTopicList) GetListByForum(f *Forum, page, orderby int) (topicList []*TopicsRow, pagi Paginator, err error) {
if page == 0 {
page = 1
}
if f.TopicCount == 0 {
_, page, lastPage := PageOffset(f.TopicCount, page, Config.ItemsPerPage)
pageList := Paginate(page, lastPage, 5)
return topicList, Paginator{pageList, page, lastPage}, nil
}
if page == 1 && orderby == 0 {
var h *ForumTopicListHolder
var ok bool
tList.forumLock.RLock()
h, ok = tList.forums[f.ID]
tList.forumLock.RUnlock()
if ok {
return h.List, h.Paginator, nil
}
}
// TODO: Does forum.TopicCount take the deleted items into consideration for guests? We don't have soft-delete yet, only hard-delete
offset, page, lastPage := PageOffset(f.TopicCount, page, Config.ItemsPerPage)
rows, err := tList.getTopicsByForum.Query(f.ID, offset, Config.ItemsPerPage)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
defer rows.Close()
// TODO: Use something other than TopicsRow as we don't need to store the forum name and link on each and every topic item?
reqUserList := make(map[int]bool)
for rows.Next() {
t := TopicsRow{Topic:Topic{ID: 0}}
err := rows.Scan(&t.ID, &t.Title, &t.Content, &t.CreatedBy, &t.IsClosed, &t.Sticky, &t.CreatedAt, &t.LastReplyAt, &t.LastReplyBy, &t.LastReplyID, &t.ViewCount, &t.PostCount, &t.LikeCount)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
t.ParentID = f.ID
t.Link = BuildTopicURL(NameToSlug(t.Title), t.ID)
// TODO: Create a specialised function with a bit less overhead for getting the last page for a post count
_, _, lastPage := PageOffset(t.PostCount, 1, Config.ItemsPerPage)
t.LastPage = lastPage
//header.Hooks.VhookNoRet("forum_trow_assign", &t, &forum)
topicList = append(topicList, &t)
reqUserList[t.CreatedBy] = true
reqUserList[t.LastReplyBy] = true
}
if err = rows.Err(); err != nil {
return nil, Paginator{nil, 1, 1}, err
}
// Convert the user ID map to a slice, then bulk load the users
idSlice := make([]int, len(reqUserList))
var i int
for userID := range reqUserList {
idSlice[i] = userID
i++
}
// TODO: What if a user is deleted via the Control Panel?
userList, err := Users.BulkGetMap(idSlice)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
// Second pass to the add the user data
// TODO: Use a pointer to TopicsRow instead of TopicsRow itself?
for _, t := range topicList {
t.Creator = userList[t.CreatedBy]
t.LastUser = userList[t.LastReplyBy]
}
pageList := Paginate(page, lastPage, 5)
return topicList, Paginator{pageList, page, lastPage}, nil
}
func (tList *DefaultTopicList) GetListByGroup(g *Group, page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error) {
if page == 0 {
page = 1
}
// TODO: Cache the first three pages not just the first along with all the topics on this beaten track
// TODO: Move this into CanSee to reduce redundancy
if (page == 1 || page == 2) && orderby == 0 && len(filterIDs) == 0 {
var h [2]*TopicListHolder
var ok bool
if g.ID%2 == 0 {
tList.evenLock.RLock()
h, ok = tList.evenGroups[g.ID]
tList.evenLock.RUnlock()
} else {
tList.oddLock.RLock()
h, ok = tList.oddGroups[g.ID]
tList.oddLock.RUnlock()
}
if ok {
return h[page-1].List, h[page-1].ForumList, h[page-1].Paginator, nil
}
}
// TODO: Make CanSee a method on *Group with a canSee field? Have a CanSee method on *User to cover the case of superadmins?
//log.Printf("deoptimising for %d on page %d\n", g.ID, page)
return tList.GetListByCanSee(g.CanSee, page, orderby, filterIDs)
}
func (tList *DefaultTopicList) GetListByCanSee(canSee []int, page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error) {
// TODO: Optimise this by filtering canSee and then fetching the forums?
// We need a list of the visible forums for Quick Topic
// ? - Would it be useful, if we could post in social groups from /topics/?
for _, fid := range canSee {
f := Forums.DirtyGet(fid)
if f.Name != "" && f.Active && (f.ParentType == "" || f.ParentType == "forum") && f.TopicCount != 0 {
fcopy := f.Copy()
// TODO: Add a hook here for plugin_guilds !!
forumList = append(forumList, fcopy)
}
}
inSlice := func(haystack []int, needle int) bool {
for _, item := range haystack {
if needle == item {
return true
}
}
return false
}
var filteredForums []Forum
if len(filterIDs) > 0 {
for _, f := range forumList {
if inSlice(filterIDs, f.ID) {
filteredForums = append(filteredForums, f)
}
}
} else {
filteredForums = forumList
}
if len(filteredForums) == 1 && orderby == 0 {
topicList, pagi, err = tList.GetListByForum(&filteredForums[0], page, orderby)
return topicList, forumList, pagi, err
}
var topicCount int
for _, f := range filteredForums {
topicCount += f.TopicCount
}
// ? - Should we be showing plugin_guilds posts on /topics/?
argList, qlist := ForumListToArgQ(filteredForums)
if qlist == "" {
// We don't want to kill the page, so pass an empty slice and nil error
return topicList, filteredForums, Paginator{[]int{}, 1, 1}, nil
}
topicList, pagi, err = tList.getList(page, orderby, topicCount, argList, qlist)
return topicList, filteredForums, pagi, err
}
// TODO: Reduce the number of returns
func (tList *DefaultTopicList) GetList(page, orderby int, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, pagi Paginator, err error) {
// TODO: Make CanSee a method on *Group with a canSee field? Have a CanSee method on *User to cover the case of superadmins?
cCanSee, err := Forums.GetAllVisibleIDs()
if err != nil {
return nil, nil, Paginator{nil, 1, 1}, err
}
inSlice := func(haystack []int, needle int) bool {
for _, item := range haystack {
if needle == item {
return true
}
}
return false
}
var canSee []int
if len(filterIDs) > 0 {
for _, fid := range cCanSee {
if inSlice(filterIDs, fid) {
canSee = append(canSee, fid)
}
}
} else {
canSee = cCanSee
}
// We need a list of the visible forums for Quick Topic
// ? - Would it be useful, if we could post in social groups from /topics/?
var topicCount int
for _, fid := range canSee {
f := Forums.DirtyGet(fid)
if f.Name != "" && f.Active && (f.ParentType == "" || f.ParentType == "forum") && f.TopicCount != 0 {
fcopy := f.Copy()
// TODO: Add a hook here for plugin_guilds
forumList = append(forumList, fcopy)
topicCount += fcopy.TopicCount
}
}
if len(forumList) == 1 && orderby == 0 {
topicList, pagi, err = tList.GetListByForum(&forumList[0], page, orderby)
return topicList, forumList, pagi, err
}
// ? - Should we be showing plugin_guilds posts on /topics/?
argList, qlist := ForumListToArgQ(forumList)
if qlist == "" {
// If the super admin can't see anything, then things have gone terribly wrong
return topicList, forumList, Paginator{[]int{}, 1, 1}, err
}
topicList, pagi, err = tList.getList(page, orderby, topicCount, argList, qlist)
return topicList, forumList, pagi, err
}
// TODO: Rename this to TopicListStore and pass back a TopicList instance holding the pagination data and topic list rather than passing them back one argument at a time
// TODO: Make orderby an enum of sorts
func (tList *DefaultTopicList) getList(page, orderby, topicCount int, argList []interface{}, qlist string) (topicList []*TopicsRow, paginator Paginator, err error) {
//log.Printf("argList: %+v\n",argList)
//log.Printf("qlist: %+v\n",qlist)
var orderq string
var stmt *sql.Stmt
if orderby == TopicListMostViewed {
tList.qLock.RLock()
stmt = tList.qcounts[len(argList)-2]
tList.qLock.RUnlock()
if stmt == nil {
orderq = "views DESC,lastReplyAt DESC,createdBy DESC"
}
} else {
tList.qLock2.RLock()
stmt = tList.qcounts2[len(argList)-2]
tList.qLock2.RUnlock()
if stmt == nil {
orderq = "sticky DESC,lastReplyAt DESC,createdBy DESC"
}
}
offset, page, lastPage := PageOffset(topicCount, page, Config.ItemsPerPage)
// TODO: Prepare common qlist lengths to speed this up in common cases, prepared statements are prepared lazily anyway, so it probably doesn't matter if we do ten or so
if stmt == nil {
stmt, err = qgen.Builder.SimpleSelect("topics", "tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,parentID,views,postCount,likeCount,attachCount,poll,data", "parentID IN("+qlist+")", orderq, "?,?")
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
defer stmt.Close()
}
argList = append(argList, offset)
argList = append(argList, Config.ItemsPerPage)
rows, err := stmt.Query(argList...)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
defer rows.Close()
rcache := Rstore.GetCache()
rcap := rcache.GetCapacity()
rlen := rcache.Length()
tc := Topics.GetCache()
reqUserList := make(map[int]bool)
for rows.Next() {
// TODO: Embed Topic structs in TopicsRow to make it easier for us to reuse this work in the topic cache
t := TopicsRow{}
err := rows.Scan(&t.ID, &t.Title, &t.Content, &t.CreatedBy, &t.IsClosed, &t.Sticky, &t.CreatedAt, &t.LastReplyAt, &t.LastReplyBy, &t.LastReplyID, &t.ParentID, &t.ViewCount, &t.PostCount, &t.LikeCount, &t.AttachCount, &t.Poll, &t.Data)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
t.Link = BuildTopicURL(NameToSlug(t.Title), t.ID)
// TODO: Pass forum to something like topicItem.Forum and use that instead of these two properties? Could be more flexible.
forum := Forums.DirtyGet(t.ParentID)
t.ForumName = forum.Name
t.ForumLink = forum.Link
// TODO: Create a specialised function with a bit less overhead for getting the last page for a post count
_, _, lastPage := PageOffset(t.PostCount, 1, Config.ItemsPerPage)
t.LastPage = lastPage
// TODO: Rename this Vhook to better reflect moving the topic list from /routes/ to /common/
GetHookTable().Vhook("topics_topic_row_assign", &t, &forum)
topicList = append(topicList, &t)
reqUserList[t.CreatedBy] = true
reqUserList[t.LastReplyBy] = true
//log.Print("rlen: ", rlen)
//log.Print("rcap: ", rcap)
//log.Print("t.PostCount: ", t.PostCount)
//log.Print("t.PostCount == 2 && rlen < rcap: ", t.PostCount == 2 && rlen < rcap)
// Avoid the extra queries on topic list pages, if we already have what we want...
hRids := false
if tc != nil {
if t, err := tc.Get(t.ID); err == nil {
hRids = len(t.Rids) != 0
}
}
if t.PostCount == 2 && rlen < rcap && !hRids && page < 5 {
rids, err := GetRidsForTopic(t.ID, 0)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
//log.Print("rids: ", rids)
if len(rids) == 0 {
continue
}
_, _ = Rstore.Get(rids[0])
rlen++
t.Rids = []int{rids[0]}
}
if tc != nil {
if _, e := tc.Get(t.ID); e == sql.ErrNoRows {
//_ = tc.Set(t.Topic())
_ = tc.Set(&t.Topic)
}
}
}
if err = rows.Err(); err != nil {
return nil, Paginator{nil, 1, 1}, err
}
// TODO: specialcase for when reqUserList only has one or two items to avoid map alloc
if len(reqUserList) == 1 {
var u *User
for uid, _ := range reqUserList {
u, err = Users.Get(uid)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
}
for _, t := range topicList {
t.Creator = u
t.LastUser = u
}
} else if len(reqUserList) > 0 {
// Convert the user ID map to a slice, then bulk load the users
idSlice := make([]int, len(reqUserList))
var i int
for userID := range reqUserList {
idSlice[i] = userID
i++
}
// TODO: What if a user is deleted via the Control Panel?
userList, err := Users.BulkGetMap(idSlice)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
// Second pass to the add the user data
// TODO: Use a pointer to TopicsRow instead of TopicsRow itself?
for _, t := range topicList {
t.Creator = userList[t.CreatedBy]
t.LastUser = userList[t.LastReplyBy]
}
}
pageList := Paginate(page, lastPage, 5)
return topicList, Paginator{pageList, page, lastPage}, nil
}
// Internal. Don't rely on it.
func ForumListToArgQ(forums []Forum) (argList []interface{}, qlist string) {
for _, forum := range forums {
argList = append(argList, strconv.Itoa(forum.ID))
qlist += "?,"
}
if qlist != "" {
qlist = qlist[0 : len(qlist)-1]
}
return argList, qlist
}
// Internal. Don't rely on it.
// TODO: Check the TopicCount field on the forums instead? Make sure it's in sync first.
func ArgQToTopicCount(argList []interface{}, qlist string) (topicCount int, err error) {
topicCountStmt, err := qgen.Builder.SimpleCount("topics", "parentID IN("+qlist+")", "")
if err != nil {
return 0, err
}
defer topicCountStmt.Close()
err = topicCountStmt.QueryRow(argList...).Scan(&topicCount)
if err != nil && err != ErrNoRows {
return 0, err
}
return topicCount, err
}
func TopicCountInForums(forums []Forum) (topicCount int, err error) {
for _, f := range forums {
topicCount += f.TopicCount
}
return topicCount, nil
}