a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added a small reply cache. 

Preloaded a small number of users and topics.
Use cache.Set instead of cache.Add in a few spots for topics to avoid issues with the counter falling out of sync with the cache length.
Embed Reply in ReplyUser instead of duplicating the same fields.
Add the missing AttachCount and ActionType fields to Reply.
Add the missing Poll field to TopicsRow.
Added the TopicUser.Replies method to better abstract the reply list generation logic.
Shortened some common.s to c.s
Moved memStuff out of the analytics memory panes and into analytics.js
Removed the temporary preStats fix for label overflow now that we have a better solution.
Added the panel_analytics_script_memory template to help de-dupe logic in the analytics memory panes.
Added the Topic method to TopicsRow.
Added the GetRidsForTopic method to help cache replies in a small set of scenarios.

Added the ReplyCache config.json setting.
Added the ReplyCacheCapacity config.json setting.

Added a parser test case.
Added more Reply and ReplyStore related test cases.
This commit is contained in:
Azareal 2019-05-17 18:40:41 +10:00
parent 9cd3cbadab
commit b9973719a5
27 changed files with 843 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/install/install.go
View File

@ -118,8 +118,10 @@ func main() {
"MaxRequestSizeStr":"5MB",
"UserCache":"static",
"TopicCache":"static",
"ReplyCache":"static",
"UserCacheCapacity":180,
"TopicCacheCapacity":400,
"ReplyCacheCapacity":20,
"DefaultPath":"/topics/",
"DefaultGroup":3,
"ActivationGroup":5,

46
common/null_reply_cache.go Normal file
View File

@ -0,0 +1,46 @@
package common
// NullReplyCache is a reply cache to be used when you don't want a cache and just want queries to passthrough to the database
type NullReplyCache struct {
}
// NewNullReplyCache gives you a new instance of NullReplyCache
func NewNullReplyCache() *NullReplyCache {
return &NullReplyCache{}
}
// nolint
func (c *NullReplyCache) Get(id int) (*Reply, error) {
return nil, ErrNoRows
}
func (c *NullReplyCache) GetUnsafe(id int) (*Reply, error) {
return nil, ErrNoRows
}
func (c *NullReplyCache) BulkGet(ids []int) (list []*Reply) {
return make([]*Reply, len(ids))
}
func (c *NullReplyCache) Set(_ *Reply) error {
return nil
}
func (c *NullReplyCache) Add(_ *Reply) error {
return nil
}
func (c *NullReplyCache) AddUnsafe(_ *Reply) error {
return nil
}
func (c *NullReplyCache) Remove(id int) error {
return nil
}
func (c *NullReplyCache) RemoveUnsafe(id int) error {
return nil
}
func (c *NullReplyCache) Flush() {
}
func (c *NullReplyCache) Length() int {
return 0
}
func (c *NullReplyCache) SetCapacity(_ int) {
}
func (c *NullReplyCache) GetCapacity() int {
return 0
}

5
common/pages.go
View File

@ -180,7 +180,7 @@ type TopicCAttachItem struct {
type TopicPage struct {
*Header
ItemList []ReplyUser
ItemList []*ReplyUser
Topic TopicUser
Forum *Forum
Poll Poll
@ -215,7 +215,7 @@ type ForumsPage struct {
type ProfilePage struct {
*Header
ItemList []ReplyUser
ItemList []*ReplyUser
ProfileOwner User
CurrentScore int
NextScore int
@ -594,6 +594,7 @@ type PanelDebugPage struct {
TCache int
UCache int
RCache int
TopicListThaw bool
}

37
common/reply.go
View File

@ -16,31 +16,32 @@ import (
)
type ReplyUser struct {
ID int
ParentID int
Content string
Reply
//ID int
//ParentID int
//Content string
ContentHtml string
CreatedBy int
//CreatedBy int
UserLink string
CreatedByName string
Group int
CreatedAt time.Time
LastEdit int
LastEditBy int
//Group int
//CreatedAt time.Time
//LastEdit int
//LastEditBy int
Avatar string
MicroAvatar string
ClassName string
ContentLines int
//ContentLines int
Tag string
URL string
URLPrefix string
URLName string
Level int
IPAddress string
Liked bool
LikeCount int
AttachCount int
ActionType string
//IPAddress string
//Liked bool
//LikeCount int
//AttachCount int
//ActionType string
ActionIcon string
Attachments []*MiniAttachment
@ -59,6 +60,8 @@ type Reply struct {
IPAddress string
Liked bool
LikeCount int
AttachCount int
ActionType string
}
var ErrAlreadyLiked = errors.New("You already liked this!")
@ -110,10 +113,10 @@ func (reply *Reply) Like(uid int) (err error) {
return err
}
_, err = userStmts.incrementLiked.Exec(1, uid)
_ = Rstore.GetCache().Remove(reply.ID)
return err
}
// TODO: Write tests for this
func (reply *Reply) Delete() error {
_, err := replyStmts.delete.Exec(reply.ID)
if err != nil {
@ -125,6 +128,7 @@ func (reply *Reply) Delete() error {
if tcache != nil {
tcache.Remove(reply.ParentID)
}
_ = Rstore.GetCache().Remove(reply.ID)
return err
}
@ -136,11 +140,14 @@ func (reply *Reply) SetPost(content string) error {
content = PreparseMessage(html.UnescapeString(content))
parsedContent := ParseMessage(content, topic.ParentID, "forums")
_, err = replyStmts.edit.Exec(content, parsedContent, reply.ID) // TODO: Sniff if this changed anything to see if we hit an existing poll
_ = Rstore.GetCache().Remove(reply.ID)
return err
}
// TODO: Write tests for this
func (reply *Reply) SetPoll(pollID int) error {
_, err := replyStmts.setPoll.Exec(pollID, reply.ID) // TODO: Sniff if this changed anything to see if we hit a poll
_ = Rstore.GetCache().Remove(reply.ID)
return err
}

163
common/reply_cache.go Normal file
View File

@ -0,0 +1,163 @@
package common
import (
"log"
"sync"
"sync/atomic"
)
// ReplyCache is an interface which spits out replies from a fast cache rather than the database, whether from memory or from an application like Redis. Replies may not be present in the cache but may be in the database
type ReplyCache interface {
Get(id int) (*Reply, error)
GetUnsafe(id int) (*Reply, error)
BulkGet(ids []int) (list []*Reply)
Set(item *Reply) error
Add(item *Reply) error
AddUnsafe(item *Reply) error
Remove(id int) error
RemoveUnsafe(id int) error
Flush()
Length() int
SetCapacity(capacity int)
GetCapacity() int
}
// MemoryReplyCache stores and pulls replies out of the current process' memory
type MemoryReplyCache struct {
items map[int]*Reply
length int64 // sync/atomic only lets us operate on int32s and int64s
capacity int
sync.RWMutex
}
// NewMemoryReplyCache gives you a new instance of MemoryReplyCache
func NewMemoryReplyCache(capacity int) *MemoryReplyCache {
return &MemoryReplyCache{
items: make(map[int]*Reply),
capacity: capacity,
}
}
// Get fetches a reply by ID. Returns ErrNoRows if not present.
func (mts *MemoryReplyCache) Get(id int) (*Reply, error) {
mts.RLock()
item, ok := mts.items[id]
mts.RUnlock()
if ok {
return item, nil
}
return item, ErrNoRows
}
// GetUnsafe fetches a reply by ID. Returns ErrNoRows if not present. THIS METHOD IS NOT THREAD-SAFE.
func (mts *MemoryReplyCache) GetUnsafe(id int) (*Reply, error) {
item, ok := mts.items[id]
if ok {
return item, nil
}
return item, ErrNoRows
}
// BulkGet fetches multiple replies by their IDs. Indices without replies will be set to nil, so make sure you check for those, we might want to change this behaviour to make it less confusing.
func (c *MemoryReplyCache) BulkGet(ids []int) (list []*Reply) {
list = make([]*Reply, len(ids))
c.RLock()
for i, id := range ids {
list[i] = c.items[id]
}
c.RUnlock()
return list
}
// Set overwrites the value of a reply in the cache, whether it's present or not. May return a capacity overflow error.
func (mts *MemoryReplyCache) Set(item *Reply) error {
mts.Lock()
_, ok := mts.items[item.ID]
if ok {
mts.items[item.ID] = item
} else if int(mts.length) >= mts.capacity {
mts.Unlock()
return ErrStoreCapacityOverflow
} else {
mts.items[item.ID] = item
atomic.AddInt64(&mts.length, 1)
}
mts.Unlock()
return nil
}
// Add adds a reply to the cache, similar to Set, but it's only intended for new items. This method might be deprecated in the near future, use Set. May return a capacity overflow error.
// ? Is this redundant if we have Set? Are the efficiency wins worth this? Is this even used?
func (mts *MemoryReplyCache) Add(item *Reply) error {
log.Print("MemoryReplyCache.Add")
mts.Lock()
if int(mts.length) >= mts.capacity {
mts.Unlock()
return ErrStoreCapacityOverflow
}
mts.items[item.ID] = item
mts.Unlock()
atomic.AddInt64(&mts.length, 1)
return nil
}
// AddUnsafe is the unsafe version of Add. May return a capacity overflow error. THIS METHOD IS NOT THREAD-SAFE.
func (mts *MemoryReplyCache) AddUnsafe(item *Reply) error {
if int(mts.length) >= mts.capacity {
return ErrStoreCapacityOverflow
}
mts.items[item.ID] = item
mts.length = int64(len(mts.items))
return nil
}
// Remove removes a reply from the cache by ID, if they exist. Returns ErrNoRows if no items exist.
func (mts *MemoryReplyCache) Remove(id int) error {
mts.Lock()
_, ok := mts.items[id]
if !ok {
mts.Unlock()
return ErrNoRows
}
delete(mts.items, id)
mts.Unlock()
atomic.AddInt64(&mts.length, -1)
return nil
}
// RemoveUnsafe is the unsafe version of Remove. THIS METHOD IS NOT THREAD-SAFE.
func (s *MemoryReplyCache) RemoveUnsafe(id int) error {
_, ok := s.items[id]
if !ok {
return ErrNoRows
}
delete(s.items, id)
atomic.AddInt64(&s.length, -1)
return nil
}
// Flush removes all the replies from the cache, useful for tests.
func (s *MemoryReplyCache) Flush() {
s.Lock()
s.items = make(map[int]*Reply)
s.length = 0
s.Unlock()
}
// ! Is this concurrent?
// Length returns the number of replies in the memory cache
func (s *MemoryReplyCache) Length() int {
return int(s.length)
}
// SetCapacity sets the maximum number of replies which this cache can hold
func (s *MemoryReplyCache) SetCapacity(capacity int) {
// Ints are moved in a single instruction, so this should be thread-safe
s.capacity = capacity
}
// GetCapacity returns the maximum number of replies this cache can hold
func (s *MemoryReplyCache) GetCapacity() int {
return s.capacity
}

43
common/reply_store.go
View File

@ -1,5 +1,6 @@
package common
//import "log"
import "database/sql"
import "github.com/Azareal/Gosora/query_gen"
@ -8,30 +9,48 @@ var Rstore ReplyStore
type ReplyStore interface {
Get(id int) (*Reply, error)
Create(topic *Topic, content string, ipaddress string, uid int) (id int, err error)
SetCache(cache ReplyCache)
GetCache() ReplyCache
}
type SQLReplyStore struct {
cache ReplyCache
get *sql.Stmt
create *sql.Stmt
}
func NewSQLReplyStore(acc *qgen.Accumulator) (*SQLReplyStore, error) {
func NewSQLReplyStore(acc *qgen.Accumulator, cache ReplyCache) (*SQLReplyStore, error) {
if cache == nil {
cache = NewNullReplyCache()
}
return &SQLReplyStore{
get: acc.Select("replies").Columns("tid, content, createdBy, createdAt, lastEdit, lastEditBy, ipaddress, likeCount").Where("rid = ?").Prepare(),
cache: cache,
get: acc.Select("replies").Columns("tid, content, createdBy, createdAt, lastEdit, lastEditBy, ipaddress, likeCount, attachCount, actionType").Where("rid = ?").Prepare(),
create: acc.Insert("replies").Columns("tid, content, parsed_content, createdAt, lastUpdated, ipaddress, words, createdBy").Fields("?,?,?,UTC_TIMESTAMP(),UTC_TIMESTAMP(),?,?,?").Prepare(),
}, acc.FirstError()
}
func (store *SQLReplyStore) Get(id int) (*Reply, error) {
reply := Reply{ID: id}
err := store.get.QueryRow(id).Scan(&reply.ParentID, &reply.Content, &reply.CreatedBy, &reply.CreatedAt, &reply.LastEdit, &reply.LastEditBy, &reply.IPAddress, &reply.LikeCount)
return &reply, err
func (s *SQLReplyStore) Get(id int) (*Reply, error) {
//log.Print("SQLReplyStore.Get")
reply, err := s.cache.Get(id)
if err == nil {
return reply, nil
}
reply = &Reply{ID: id}
err = s.get.QueryRow(id).Scan(&reply.ParentID, &reply.Content, &reply.CreatedBy, &reply.CreatedAt, &reply.LastEdit, &reply.LastEditBy, &reply.IPAddress, &reply.LikeCount, &reply.AttachCount, &reply.ActionType)
if err == nil {
_ = s.cache.Set(reply)
}
return reply, err
}
// TODO: Write a test for this
func (store *SQLReplyStore) Create(topic *Topic, content string, ipaddress string, uid int) (id int, err error) {
func (s *SQLReplyStore) Create(topic *Topic, content string, ipaddress string, uid int) (id int, err error) {
wcount := WordCount(content)
res, err := store.create.Exec(topic.ID, content, ParseMessage(content, topic.ParentID, "forums"), ipaddress, wcount, uid)
res, err := s.create.Exec(topic.ID, content, ParseMessage(content, topic.ParentID, "forums"), ipaddress, wcount, uid)
if err != nil {
return 0, err
}
@ -42,3 +61,11 @@ func (store *SQLReplyStore) Create(topic *Topic, content string, ipaddress strin
}
return int(lastID), topic.AddReply(int(lastID), uid)
}
func (s *SQLReplyStore) SetCache(cache ReplyCache) {
s.cache = cache
}
func (s *SQLReplyStore) GetCache() ReplyCache {
return s.cache
}

2
common/site.go
View File

@ -69,6 +69,8 @@ type config struct {
UserCacheCapacity int
TopicCache string
TopicCacheCapacity int
ReplyCache string
ReplyCacheCapacity int
SMTPServer string
SMTPUsername string

36
common/template_init.go
View File

@ -232,7 +232,7 @@ func compileCommons(c *tmpl.CTemplateSet, header *Header, header2 *Header, out T
}
var topicsList []*TopicsRow
topicsList = append(topicsList, &TopicsRow{1, "topic-title", "Topic Title", "The topic content.", 1, false, false, now, now, user3.ID, 1, 1, "", "127.0.0.1", 1, 0, 1, 1, 0, "classname", "", &user2, "", 0, &user3, "General", "/forum/general.2"})
topicsList = append(topicsList, &TopicsRow{1, "topic-title", "Topic Title", "The topic content.", 1, false, false, now, now, user3.ID, 1, 1, "", "127.0.0.1", 1, 0, 1, 1, 0, "classname", 0, "", &user2, "", 0, &user3, "General", "/forum/general.2", nil})
topicListPage := TopicListPage{htitle("Topic List"), topicsList, forumList, Config.DefaultForum, TopicListSort{"lastupdated", false}, Paginator{[]int{1}, 1, 1}}
out.Add("topics", "common.TopicListPage", topicListPage)
@ -247,9 +247,13 @@ func compileCommons(c *tmpl.CTemplateSet, header *Header, header2 *Header, out T
}, VoteCount: 7}
avatar, microAvatar := BuildAvatar(62, "")
miniAttach := []*MiniAttachment{&MiniAttachment{Path: "/"}}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach}
var replyList []ReplyUser
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach})
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach, nil}
var replyList []*ReplyUser
reply := Reply{1, 1, "Yo!", 1, Config.DefaultGroup, now, 0, 0, 1, "::1", true, 1, 1, ""}
ru := &ReplyUser{ClassName: "", Reply: reply, CreatedByName: "Alice", Avatar: avatar, URLPrefix: "", URLName: "", Level: 0, Attachments: miniAttach}
ru.Init(topic.ID)
replyList = append(replyList, ru)
tpage := TopicPage{htitle("Topic Name"), replyList, topic, &Forum{ID: 1, Name: "Hahaha"}, poll, Paginator{[]int{1}, 1, 1}}
tpage.Forum.Link = BuildForumURL(NameToSlug(tpage.Forum.Name), tpage.Forum.ID)
out.Add("topic", "common.TopicPage", tpage)
@ -264,17 +268,20 @@ func compileTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName string
header, header2, _ := tmplInitHeaders(user, user2, user3)
now := time.Now()
/*poll := Poll{ID: 1, Type: 0, Options: map[int]string{0: "Nothing", 1: "Something"}, Results: map[int]int{0: 5, 1: 2}, QuickOptions: []PollOption{
poll := Poll{ID: 1, Type: 0, Options: map[int]string{0: "Nothing", 1: "Something"}, Results: map[int]int{0: 5, 1: 2}, QuickOptions: []PollOption{
PollOption{0, "Nothing"},
PollOption{1, "Something"},
}, VoteCount: 7}*/
}, VoteCount: 7}
avatar, microAvatar := BuildAvatar(62, "")
miniAttach := []*MiniAttachment{&MiniAttachment{Path: "/"}}
var replyList []ReplyUser
//topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach}
var replyList []*ReplyUser
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach, nil}
// TODO: Do we want the UID on this to be 0?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach})
reply := Reply{1, 1, "Yo!", 1, Config.DefaultGroup, now, 0, 0, 1, "::1", true, 1, 1, ""}
ru := &ReplyUser{ClassName: "", Reply: reply, CreatedByName: "Alice", Avatar: avatar, URLPrefix: "", URLName: "", Level: 0, Attachments: miniAttach}
ru.Init(topic.ID)
replyList = append(replyList, ru)
// Convienience function to save a line here and there
var htitle = func(name string) *Header {
@ -455,7 +462,7 @@ func compileJSTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName stri
tmpls := TItemHold(make(map[string]TItem))
var topicsRow = &TopicsRow{1, "topic-title", "Topic Title", "The topic content.", 1, false, false, now, now, user3.ID, 1, 1, "", "127.0.0.1", 1, 0, 1, 0, 1, "classname", "", &user2, "", 0, &user3, "General", "/forum/general.2"}
var topicsRow = &TopicsRow{1, "topic-title", "Topic Title", "The topic content.", 1, false, false, now, now, user3.ID, 1, 1, "", "127.0.0.1", 1, 0, 1, 0, 1, "classname", 0, "", &user2, "", 0, &user3, "General", "/forum/general.2", nil}
tmpls.AddStd("topics_topic", "common.TopicsRow", topicsRow)
poll := Poll{ID: 1, Type: 0, Options: map[int]string{0: "Nothing", 1: "Something"}, Results: map[int]int{0: 5, 1: 2}, QuickOptions: []PollOption{
@ -464,11 +471,14 @@ func compileJSTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName stri
}, VoteCount: 7}
avatar, microAvatar := BuildAvatar(62, "")
miniAttach := []*MiniAttachment{&MiniAttachment{Path: "/"}}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach}
var replyList []ReplyUser
topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, now, 1, 1, 0, "", "127.0.0.1", 1, 0, 1, 0, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false, miniAttach, nil}
var replyList []*ReplyUser
// TODO: Do we really want the UID here to be zero?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach})
reply := Reply{1, 1, "Yo!", 1, Config.DefaultGroup, now, 0, 0, 1, "::1", true, 1, 1, ""}
ru := &ReplyUser{ClassName: "", Reply: reply, CreatedByName: "Alice", Avatar: avatar, URLPrefix: "", URLName: "", Level: 0, Attachments: miniAttach}
ru.Init(topic.ID)
replyList = append(replyList, ru)
varList = make(map[string]tmpl.VarItem)
header.Title = "Topic Name"

250
common/topic.go
View File

@ -10,9 +10,12 @@ import (
"database/sql"
"html"
"html/template"
//"log"
"strconv"
"strings"
"time"
p "github.com/Azareal/Gosora/common/phrases"
"github.com/Azareal/Gosora/query_gen"
)
@ -43,6 +46,8 @@ type Topic struct {
ClassName string // CSS Class Name
Poll int
Data string // Used for report metadata
Rids []int
}
type TopicUser struct {
@ -83,8 +88,10 @@ type TopicUser struct {
Liked bool
Attachments []*MiniAttachment
Rids []int
}
// TODO: Embed TopicUser to simplify this structure and it's related logic?
type TopicsRow struct {
ID int
Link string
@ -106,6 +113,7 @@ type TopicsRow struct {
AttachCount int
LastPage int
ClassName string
Poll int
Data string // Used for report metadata
Creator *User
@ -115,6 +123,7 @@ type TopicsRow struct {
ForumName string //TopicsRow
ForumLink string
Rids []int
}
type WsTopicsRow struct {
@ -156,7 +165,12 @@ func (t *Topic) TopicsRow() *TopicsRow {
forumName := ""
forumLink := ""
return &TopicsRow{t.ID, t.Link, t.Title, t.Content, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, t.LastReplyBy, t.LastReplyID, t.ParentID, t.Status, t.IPAddress, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, lastPage, t.ClassName, t.Data, creator, "", contentLines, lastUser, forumName, forumLink}
return &TopicsRow{t.ID, t.Link, t.Title, t.Content, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, t.LastReplyBy, t.LastReplyID, t.ParentID, t.Status, t.IPAddress, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, lastPage, t.ClassName, t.Poll, t.Data, creator, "", contentLines, lastUser, forumName, forumLink, t.Rids}
}
// ! Some data may be lost in the conversion
func (t *TopicsRow) Topic() *Topic {
return &Topic{t.ID, t.Link, t.Title, t.Content, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, t.LastReplyBy, t.LastReplyID, t.ParentID, t.Status, t.IPAddress, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, t.ClassName, t.Poll, t.Data, t.Rids}
}
// ! Not quite safe as Topic doesn't contain all the data needed to constructs a WsTopicsRow
@ -169,6 +183,8 @@ func (t *Topic) TopicsRow() *TopicsRow {
}*/
type TopicStmts struct {
getRids *sql.Stmt
getReplies *sql.Stmt
addReplies *sql.Stmt
updateLastReply *sql.Stmt
lock *sql.Stmt
@ -195,6 +211,8 @@ var topicStmts TopicStmts
func init() {
DbInits.Add(func(acc *qgen.Accumulator) error {
topicStmts = TopicStmts{
getRids: acc.Select("replies").Columns("rid").Where("tid = ?").Orderby("rid ASC").Limit("?,?").Prepare(),
getReplies: acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.attachCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"),
addReplies: acc.Update("topics").Set("postCount = postCount + ?, lastReplyBy = ?, lastReplyAt = UTC_TIMESTAMP()").Where("tid = ?").Prepare(),
updateLastReply: acc.Update("topics").Set("lastReplyID = ?").Where("lastReplyID > ? AND tid = ?").Prepare(),
lock: acc.Update("topics").Set("is_closed = 1").Where("tid = ?").Prepare(),
@ -385,6 +403,233 @@ func (topic *Topic) CreateActionReply(action string, ipaddress string, uid int)
return err
}
func GetRidsForTopic(tid int, offset int) (rids []int, err error) {
rows, err := topicStmts.getRids.Query(tid, offset, Config.ItemsPerPage)
if err != nil {
return nil, err
}
defer rows.Close()
var rid int
for rows.Next() {
err := rows.Scan(&rid)
if err != nil {
return nil, err
}
rids = append(rids, rid)
}
return rids, rows.Err()
}
func (ru *ReplyUser) Init(parentID int) error {
ru.UserLink = BuildProfileURL(NameToSlug(ru.CreatedByName), ru.CreatedBy)
ru.ParentID = parentID
ru.ContentLines = strings.Count(ru.Content, "\n")
postGroup, err := Groups.Get(ru.Group)
if err != nil {
return err
}
if postGroup.IsMod {
ru.ClassName = Config.StaffCSS
}
// TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the c.UserStore initialise this?
ru.Avatar, ru.MicroAvatar = BuildAvatar(ru.CreatedBy, ru.Avatar)
if ru.Tag == "" {
ru.Tag = postGroup.Tag
}
// We really shouldn't have inline HTML, we should do something about this...
if ru.ActionType != "" {
var action string
aarr := strings.Split(ru.ActionType, "-")
switch aarr[0] {
case "lock":
action = aarr[0]
ru.ActionIcon = "🔒&#xFE0E"
case "unlock":
action = aarr[0]
ru.ActionIcon = "🔓&#xFE0E"
case "stick":
action = aarr[0]
ru.ActionIcon = "📌&#xFE0E"
case "unstick":
action = aarr[0]
ru.ActionIcon = "📌&#xFE0E"
case "move":
if len(aarr) == 2 {
fid, _ := strconv.Atoi(aarr[1])
forum, err := Forums.Get(fid)
if err == nil {
ru.ActionType = p.GetTmplPhrasef("topic.action_topic_move_dest", forum.Link, forum.Name, ru.UserLink, ru.CreatedByName)
} else {
action = aarr[0]
}
} else {
action = aarr[0]
}
default:
// TODO: Only fire this off if a corresponding phrase for the ActionType doesn't exist? Or maybe have some sort of action registry?
ru.ActionType = p.GetTmplPhrasef("topic.action_topic_default", ru.ActionType)
}
if action != "" {
ru.ActionType = p.GetTmplPhrasef("topic.action_topic_"+action, ru.UserLink, ru.CreatedByName)
}
}
return nil
}
// TODO: Factor TopicUser into a *Topic and *User, as this starting to become overly complicated x.x
func (topic *TopicUser) Replies(offset int, pFrag int, user *User) (rlist []*ReplyUser, ogdesc string, err error) {
var likedMap map[int]int
if user.Liked > 0 {
likedMap = make(map[int]int)
}
var likedQueryList = []int{user.ID}
var attachMap map[int]int
if user.Perms.EditReply {
attachMap = make(map[int]int)
}
var attachQueryList = []int{}
var rid int
if len(topic.Rids) > 0 {
//log.Print("have rid")
rid = topic.Rids[0]
}
re, err := Rstore.GetCache().Get(rid)
ucache := Users.GetCache()
var ruser *User
if err == nil && ucache != nil {
//log.Print("ucache step")
ruser, err = ucache.Get(re.CreatedBy)
}
// TODO: Factor the user fields out and embed a user struct instead
var reply *ReplyUser
hTbl := GetHookTable()
if err == nil {
//log.Print("reply cached serve")
reply = &ReplyUser{ClassName: "", Reply: *re, CreatedByName: ruser.Name, Avatar: ruser.Avatar, URLPrefix: ruser.URLPrefix, URLName: ruser.URLName, Level: ruser.Level}
err := reply.Init(topic.ID)
if err != nil {
return nil, "", err
}
reply.ContentHtml = ParseMessage(reply.Content, topic.ParentID, "forums")
if reply.ID == pFrag {
ogdesc = reply.Content
if len(ogdesc) > 200 {
ogdesc = ogdesc[:197] + "..."
}
}
if reply.LikeCount > 0 && user.Liked > 0 {
likedMap[reply.ID] = len(rlist)
likedQueryList = append(likedQueryList, reply.ID)
}
if user.Perms.EditReply && reply.AttachCount > 0 {
attachMap[reply.ID] = len(rlist)
attachQueryList = append(attachQueryList, reply.ID)
}
hTbl.VhookNoRet("topic_reply_row_assign", &rlist, &reply)
rlist = append(rlist, reply)
} else {
rows, err := topicStmts.getReplies.Query(topic.ID, offset, Config.ItemsPerPage)
if err != nil {
return nil, "", err
}
defer rows.Close()
for rows.Next() {
reply = &ReplyUser{}
err := rows.Scan(&reply.ID, &reply.Content, &reply.CreatedBy, &reply.CreatedAt, &reply.LastEdit, &reply.LastEditBy, &reply.Avatar, &reply.CreatedByName, &reply.Group, &reply.URLPrefix, &reply.URLName, &reply.Level, &reply.IPAddress, &reply.LikeCount, &reply.AttachCount, &reply.ActionType)
if err != nil {
return nil, "", err
}
err = reply.Init(topic.ID)
if err != nil {
return nil, "", err
}
reply.ContentHtml = ParseMessage(reply.Content, topic.ParentID, "forums")
if reply.ID == pFrag {
ogdesc = reply.Content
if len(ogdesc) > 200 {
ogdesc = ogdesc[:197] + "..."
}
}
if reply.LikeCount > 0 && user.Liked > 0 {
likedMap[reply.ID] = len(rlist)
likedQueryList = append(likedQueryList, reply.ID)
}
if user.Perms.EditReply && reply.AttachCount > 0 {
attachMap[reply.ID] = len(rlist)
attachQueryList = append(attachQueryList, reply.ID)
}
hTbl.VhookNoRet("topic_reply_row_assign", &rlist, &reply)
// TODO: Use a pointer instead to make it easier to abstract this loop? What impact would this have on escape analysis?
rlist = append(rlist, reply)
//log.Printf("r: %d-%d", reply.ID, len(rlist)-1)
}
err = rows.Err()
if err != nil {
return nil, "", err
}
}
// TODO: Add a config setting to disable the liked query for a burst of extra speed
if user.Liked > 0 && len(likedQueryList) > 1 /*&& user.LastLiked <= time.Now()*/ {
// TODO: Abstract this
rows, err := qgen.NewAcc().Select("likes").Columns("targetItem").Where("sentBy = ? AND targetType = 'replies'").In("targetItem", likedQueryList[1:]).Query(user.ID)
if err != nil && err != sql.ErrNoRows {
return nil, "", err
}
defer rows.Close()
for rows.Next() {
var likeRid int
err := rows.Scan(&likeRid)
if err != nil {
return nil, "", err
}
rlist[likedMap[likeRid]].Liked = true
}
err = rows.Err()
if err != nil {
return nil, "", err
}
}
if user.Perms.EditReply && len(attachQueryList) > 0 {
//log.Printf("attachQueryList: %+v\n", attachQueryList)
amap, err := Attachments.BulkMiniGetList("replies", attachQueryList)
if err != nil && err != sql.ErrNoRows {
return nil, "", err
}
//log.Printf("amap: %+v\n", amap)
//log.Printf("attachMap: %+v\n", attachMap)
for id, attach := range amap {
//log.Print("id:", id)
rlist[attachMap[id]].Attachments = attach
/*for _, a := range attach {
log.Printf("a: %+v\n", a)
}*/
}
}
return rlist, ogdesc, nil
}
// TODO: Test this
func (topic *Topic) Author() (*User, error) {
return Users.Get(topic.CreatedBy)
@ -452,7 +697,7 @@ func GetTopicUser(user *User, tid int) (tu TopicUser, err error) {
if tcache != nil {
theTopic := Topic{ID: tu.ID, Link: tu.Link, Title: tu.Title, Content: tu.Content, CreatedBy: tu.CreatedBy, IsClosed: tu.IsClosed, Sticky: tu.Sticky, CreatedAt: tu.CreatedAt, LastReplyAt: tu.LastReplyAt, LastReplyID: tu.LastReplyID, ParentID: tu.ParentID, IPAddress: tu.IPAddress, ViewCount: tu.ViewCount, PostCount: tu.PostCount, LikeCount: tu.LikeCount, AttachCount: tu.AttachCount, Poll: tu.Poll}
//log.Printf("theTopic: %+v\n", theTopic)
_ = tcache.Add(&theTopic)
_ = tcache.Set(&theTopic)
}
return tu, err
}
@ -485,6 +730,7 @@ func copyTopicToTopicUser(topic *Topic, user *User) (tu TopicUser) {
tu.AttachCount = topic.AttachCount
tu.Poll = topic.Poll
tu.Data = topic.Data
tu.Rids = topic.Rids
return tu
}

38
common/topic_list.go
View File

@ -82,10 +82,12 @@ func (tList *DefaultTopicList) Tick() error {
if group.UserCount == 0 && group.ID != GuestUser.Group {
continue
}
var canSee = make([]byte, len(group.CanSee))
for i, item := range group.CanSee {
canSee[i] = byte(item)
}
var canSeeInt = make([]int, len(canSee))
copy(canSeeInt, group.CanSee)
sCanSee := string(canSee)
@ -244,7 +246,6 @@ func (tList *DefaultTopicList) GetList(page int, orderby string, filterIDs []int
func (tList *DefaultTopicList) getList(page int, orderby string, argList []interface{}, qlist string) (topicList []*TopicsRow, paginator Paginator, err error) {
//log.Printf("argList: %+v\n",argList)
//log.Printf("qlist: %+v\n",qlist)
topicCount, err := ArgQToTopicCount(argList, qlist)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
@ -259,7 +260,7 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter
}
// TODO: Prepare common qlist lengths to speed this up in common cases, prepared statements are prepared lazily anyway, so it probably doesn't matter if we do ten or so
stmt, err := qgen.Builder.SimpleSelect("topics", "tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount", "parentID IN("+qlist+")", orderq, "?,?")
stmt, err := qgen.Builder.SimpleSelect("topics", "tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount, attachCount, poll, data", "parentID IN("+qlist+")", orderq, "?,?")
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
@ -274,11 +275,15 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter
}
defer rows.Close()
var reqUserList = make(map[int]bool)
rcache := Rstore.GetCache()
rcap := rcache.GetCapacity()
rlen := rcache.Length()
tcache := Topics.GetCache()
reqUserList := make(map[int]bool)
for rows.Next() {
// TODO: Embed Topic structs in TopicsRow to make it easier for us to reuse this work in the topic cache
topic := TopicsRow{ID: 0}
err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.IsClosed, &topic.Sticky, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyBy, &topic.LastReplyID, &topic.ParentID, &topic.ViewCount, &topic.PostCount, &topic.LikeCount)
topic := TopicsRow{}
err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.IsClosed, &topic.Sticky, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyBy, &topic.LastReplyID, &topic.ParentID, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
@ -298,6 +303,29 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter
topicList = append(topicList, &topic)
reqUserList[topic.CreatedBy] = true
reqUserList[topic.LastReplyBy] = true
//log.Print("rlen: ", rlen)
//log.Print("rcap: ", rcap)
//log.Print("topic.PostCount: ", topic.PostCount)
//log.Print("topic.PostCount == 2 && rlen < rcap: ", topic.PostCount == 2 && rlen < rcap)
if topic.PostCount == 2 && rlen < rcap {
rids, err := GetRidsForTopic(topic.ID, 0)
if err != nil {
return nil, Paginator{nil, 1, 1}, err
}
//log.Print("rids: ", rids)
if len(rids) == 0 {
continue
}
_, _ = Rstore.Get(rids[0])
rlen++
topic.Rids = []int{rids[0]}
}
if tcache != nil {
_ = tcache.Set(topic.Topic())
}
}
err = rows.Err()
if err != nil {

4
common/topic_store.go
View File

@ -76,7 +76,7 @@ func (mts *DefaultTopicStore) DirtyGet(id int) *Topic {
err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
if err == nil {
topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
_ = mts.cache.Add(topic)
_ = mts.cache.Set(topic)
return topic
}
return BlankTopic()
@ -93,7 +93,7 @@ func (mts *DefaultTopicStore) Get(id int) (topic *Topic, err error) {
err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
if err == nil {
topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
_ = mts.cache.Add(topic)
_ = mts.cache.Set(topic)
}
return topic, err
}

2
config/config_example.json
View File

@ -20,8 +20,10 @@
"MaxRequestSizeStr":"5MB",
"UserCache":"static",
"TopicCache":"static",
"ReplyCache":"static",
"UserCacheCapacity":180,
"TopicCacheCapacity":400,
"ReplyCacheCapacity":20,
"DefaultPath":"/topics/",
"DefaultGroup":3,
"ActivationGroup":5,

41
database.go
View File

@ -4,7 +4,7 @@ import (
"database/sql"
"log"
"github.com/Azareal/Gosora/common"
c "github.com/Azareal/Gosora/common"
"github.com/pkg/errors"
)
@ -29,17 +29,17 @@ func InitDatabase() (err error) {
globs = &Globs{stmts}
log.Print("Running the db handlers.")
err = common.DbInits.Run()
err = c.DbInits.Run()
if err != nil {
return errors.WithStack(err)
}
log.Print("Loading the usergroups.")
common.Groups, err = common.NewMemoryGroupStore()
c.Groups, err = c.NewMemoryGroupStore()
if err != nil {
return errors.WithStack(err)
}
err2 := common.Groups.LoadGroups()
err2 := c.Groups.LoadGroups()
if err2 != nil {
return errors.WithStack(err2)
}
@ -47,59 +47,58 @@ func InitDatabase() (err error) {
// We have to put this here, otherwise LoadForums() won't be able to get the last poster data when building it's forums
log.Print("Initialising the user and topic stores")
var ucache common.UserCache
if common.Config.UserCache == "static" {
ucache = common.NewMemoryUserCache(common.Config.UserCacheCapacity)
var ucache c.UserCache
if c.Config.UserCache == "static" {
ucache = c.NewMemoryUserCache(c.Config.UserCacheCapacity)
}
var tcache c.TopicCache
if c.Config.TopicCache == "static" {
tcache = c.NewMemoryTopicCache(c.Config.TopicCacheCapacity)
}
var tcache common.TopicCache
if common.Config.TopicCache == "static" {
tcache = common.NewMemoryTopicCache(common.Config.TopicCacheCapacity)
}
common.Users, err = common.NewDefaultUserStore(ucache)
c.Users, err = c.NewDefaultUserStore(ucache)
if err != nil {
return errors.WithStack(err)
}
common.Topics, err = common.NewDefaultTopicStore(tcache)
c.Topics, err = c.NewDefaultTopicStore(tcache)
if err != nil {
return errors.WithStack(err2)
}
log.Print("Loading the forums.")
common.Forums, err = common.NewMemoryForumStore()
c.Forums, err = c.NewMemoryForumStore()
if err != nil {
return errors.WithStack(err)
}
err = common.Forums.LoadForums()
err = c.Forums.LoadForums()
if err != nil {
return errors.WithStack(err)
}
log.Print("Loading the forum permissions.")
common.FPStore, err = common.NewMemoryForumPermsStore()
c.FPStore, err = c.NewMemoryForumPermsStore()
if err != nil {
return errors.WithStack(err)
}
err = common.FPStore.Init()
err = c.FPStore.Init()
if err != nil {
return errors.WithStack(err)
}
log.Print("Loading the settings.")
err = common.LoadSettings()
err = c.LoadSettings()
if err != nil {
return errors.WithStack(err)
}
log.Print("Loading the plugins.")
err = common.InitExtend()
err = c.InitExtend()
if err != nil {
return errors.WithStack(err)
}
log.Print("Loading the themes.")
err = common.Themes.LoadActiveStatus()
err = c.Themes.LoadActiveStatus()
if err != nil {
return errors.WithStack(err)
}

4
docs/configuration.md
View File

@ -60,10 +60,14 @@ UserCache - The type of user cache you want to use. You can leave this blank to
TopicCache - The type of topic cache you want to use. You can leave this blank to disable this feature or use `static` for a small in-memory cache.
ReplyCache - The type of reply cache you want to use. You can leave this blank to disable this feature or use `static` for a small in-memory cache.
UserCacheCapacity - The maximum number of users you want in the in-memory user cache, if enabled in the UserCache setting.
TopicCacheCapacity - The maximum number of topics you want in the in-memory topic cache, if enabled in the TopicCache setting.
ReplyCacheCapacity - The maximum number of replies you want in the in-memory reply cache, if enabled in the ReplyCache setting.
DefaultPath - The route you want the homepage `/` to default to. Examples: `/topics/` or `/forums/`
DefaultGroup - The group you want users to be moved to once they're activated. Example: 3

52
main.go
View File

@ -27,8 +27,8 @@ import (
c "github.com/Azareal/Gosora/common"
"github.com/Azareal/Gosora/common/counters"
"github.com/Azareal/Gosora/common/phrases"
"github.com/Azareal/Gosora/routes"
"github.com/Azareal/Gosora/query_gen"
"github.com/Azareal/Gosora/routes"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
)
@ -47,11 +47,57 @@ func init() {
c.RenderTemplateAlias = routes.RenderTemplate
}
func afterDBInit() (err error) {
err = storeInit()
if err != nil {
return err
}
var uids []int
tcache := c.Topics.GetCache()
if tcache != nil {
// Preload ten topics to get the wheels going
var count = 10
if tcache.GetCapacity() <= 10 {
count = 2
if tcache.GetCapacity() <= 2 {
count = 0
}
}
// TODO: Use the same cached data for both the topic list and the topic fetches...
tList, _, _, err := c.TopicList.GetList(1, "", nil)
if err != nil {
return err
}
if count > len(tList) {
count = len(tList)
}
for i := 0; i < count; i++ {
_, _ = c.Topics.Get(tList[i].ID)
}
}
ucache := c.Users.GetCache()
if ucache != nil {
// Preload associated users too...
for _, uid := range uids {
_, _ = c.Users.Get(uid)
}
}
return nil
}
// Experimenting with a new error package here to try to reduce the amount of debugging we have to do
// TODO: Dynamically register these items to avoid maintaining as much code here?
func afterDBInit() (err error) {
func storeInit() (err error) {
acc := qgen.NewAcc()
c.Rstore, err = c.NewSQLReplyStore(acc)
var rcache c.ReplyCache
if c.Config.ReplyCache == "static" {
rcache = c.NewMemoryReplyCache(c.Config.ReplyCacheCapacity)
}
c.Rstore, err = c.NewSQLReplyStore(acc, rcache)
if err != nil {
return errors.WithStack(err)
}

42
misc_test.go
View File

@ -709,8 +709,7 @@ func TestReplyStore(t *testing.T) {
_, err = c.Rstore.Get(0)
recordMustNotExist(t, err, "RID #0 shouldn't exist")
var replyTest = func(rid int, parentID int, createdBy int, content string, ip string) {
reply, err := c.Rstore.Get(rid)
var replyTest2 = func(reply *c.Reply, err error, rid int, parentID int, createdBy int, content string, ip string) {
expectNilErr(t, err)
expect(t, reply.ID == rid, fmt.Sprintf("RID #%d has the wrong ID. It should be %d not %d", rid, rid, reply.ID))
expect(t, reply.ParentID == parentID, fmt.Sprintf("The parent topic of RID #%d should be %d not %d", rid, parentID, reply.ParentID))
@ -718,16 +717,29 @@ func TestReplyStore(t *testing.T) {
expect(t, reply.Content == content, fmt.Sprintf("The contents of RID #%d should be '%s' not %s", rid, content, reply.Content))
expect(t, reply.IPAddress == ip, fmt.Sprintf("The IPAddress of RID#%d should be '%s' not %s", rid, ip, reply.IPAddress))
}
var replyTest = func(rid int, parentID int, createdBy int, content string, ip string) {
reply, err := c.Rstore.Get(rid)
replyTest2(reply, err, rid, parentID, createdBy, content, ip)
reply, err = c.Rstore.GetCache().Get(rid)
replyTest2(reply, err, rid, parentID, createdBy, content, ip)
}
replyTest(1, 1, 1, "A reply!", "::1")
// ! This is hard to do deterministically as the system may pre-load certain items but let's give it a try:
//_, err = c.Rstore.GetCache().Get(1)
//recordMustNotExist(t, err, "RID #1 shouldn't be in the cache")
_, err = c.Rstore.Get(2)
recordMustNotExist(t, err, "RID #2 shouldn't exist")
// TODO: Test Create and Get
//Create(tid int, content string, ipaddress string, fid int, uid int) (id int, err error)
topic, err := c.Topics.Get(1)
expectNilErr(t, err)
expect(t, topic.PostCount == 1, fmt.Sprintf("TID #1's post count should be one, not %d", topic.PostCount))
_, err = c.Rstore.GetCache().Get(2)
recordMustNotExist(t, err, "RID #2 shouldn't be in the cache")
rid, err := c.Rstore.Create(topic, "Fofofo", "::1", 1)
expectNilErr(t, err)
expect(t, rid == 2, fmt.Sprintf("The next reply ID should be 2 not %d", rid))
@ -754,6 +766,28 @@ func TestReplyStore(t *testing.T) {
rid, err = c.Rstore.Create(topic, "hiii", "::1", 1)
expectNilErr(t, err)
replyTest(rid, topic.ID, 1, "hiii", "::1")
reply, err := c.Rstore.Get(rid)
expectNilErr(t, err)
expectNilErr(t, reply.SetPost("huuu"))
expect(t, reply.Content == "hiii", fmt.Sprintf("topic.Content should be hiii, not %s", reply.Content))
reply, err = c.Rstore.Get(rid)
expectNilErr(t, err)
expect(t, reply.Content == "huuu", fmt.Sprintf("topic.Content should be huuu, not %s", reply.Content))
expectNilErr(t, reply.Delete())
// No pointer shenanigans x.x
expect(t, reply.ID == rid, fmt.Sprintf("pointer shenanigans"))
_, err = c.Rstore.GetCache().Get(rid)
recordMustNotExist(t, err, fmt.Sprintf("RID #%d shouldn't be in the cache", rid))
_, err = c.Rstore.Get(rid)
recordMustNotExist(t, err, fmt.Sprintf("RID #%d shouldn't exist", rid))
// TODO: Write a test for this
//(topic *TopicUser) Replies(offset int, pFrag int, user *User) (rlist []*ReplyUser, ogdesc string, err error)
// TODO: Add tests for *Reply
// TODO: Add tests for ReplyCache
}
func TestProfileReplyStore(t *testing.T) {

2
parser_test.go
View File

@ -105,6 +105,8 @@ func TestPreparser(t *testing.T) {
msgList.Add("@Admin\ndd", "@1\ndd")
msgList.Add("d@Admin", "d@Admin")
msgList.Add("\\@Admin", "@Admin")
msgList.Add("@元気", "@元気")
// TODO: More tests for unicode names?
//msgList.Add("\\\\@Admin", "@1")
//msgList.Add("byte 0", string([]byte{0}), "")
msgList.Add("byte 'a'", string([]byte{'a'}), "a")

40
public/analytics.js
View File

@ -1,6 +1,42 @@
/*addHook(() => {
function memStuff(window, document, Chartist) {
'use strict';
})*/
Chartist.plugins = Chartist.plugins || {};
Chartist.plugins.byteUnits = function(options) {
options = Chartist.extend({}, {}, options);
return function byteUnits(chart) {
if(!chart instanceof Chartist.Line) return;
chart.on('created', function() {
console.log("running created")
const vbits = document.getElementsByClassName("ct-vertical");
if(vbits==null) return;
let tbits = [];
for(let i = 0; i < vbits.length; i++) {
tbits[i] = vbits[i].innerHTML;
}
console.log("tbits:",tbits);
const calc = (places) => {
if(places==3) return;
const matcher = vbits[0].innerHTML;
let allMatch = true;
for(let i = 0; i < tbits.length; i++) {
let val = convertByteUnit(tbits[i], places);
if(val!=matcher) allMatch = false;
vbits[i].innerHTML = val;
}
if(allMatch) calc(places + 1);
}
calc(0);
});
};
};
}
const Kilobyte = 1024;
const Megabyte = Kilobyte * 1024;

8
routes/panel/debug.go
View File

@ -41,7 +41,7 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
var tlen, ulen int
var tlen, ulen, rlen int
tcache := c.Topics.GetCache()
if tcache != nil {
tlen = tcache.Length()
@ -50,8 +50,12 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
if ucache != nil {
ulen = ucache.Length()
}
rcache := c.Rstore.GetCache()
if rcache != nil {
rlen = rcache.Length()
}
topicListThawed := c.TopicListThaw.Thawed()
pi := c.PanelDebugPage{basePage, goVersion, dbVersion, uptime, openConnCount, qgen.Builder.GetAdapter().GetName(), goroutines, cpus, memStats, tlen, ulen, topicListThawed}
pi := c.PanelDebugPage{basePage, goVersion, dbVersion, uptime, openConnCount, qgen.Builder.GetAdapter().GetName(), goroutines, cpus, memStats, tlen, ulen, rlen, topicListThawed}
return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_dashboard_right", "debug_page", "panel_debug", pi})
}

35
routes/profile.go
View File

@ -3,7 +3,6 @@ package routes
import (
"database/sql"
"net/http"
"strings"
"time"
c "github.com/Azareal/Gosora/common"
@ -37,9 +36,9 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user c.User, header *c.
var err error
var replyCreatedAt time.Time
var replyContent, replyCreatedByName, replyAvatar, replyMicroAvatar, replyTag, replyClassName string
var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int
var replyList []c.ReplyUser
var replyContent, replyCreatedByName, replyAvatar string
var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyGroup int
var replyList []*c.ReplyUser
// TODO: Do a 301 if it's the wrong username? Do a canonical too?
_, pid, err := ParseSEOURL(r.URL.Path[len("/user/"):])
@ -78,32 +77,26 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user c.User, header *c.
return c.InternalError(err, w, r)
}
group, err := c.Groups.Get(replyGroup)
replyLiked := false
replyLikeCount := 0
ru := &c.ReplyUser{Reply: c.Reply{rid, 0, replyContent, replyCreatedBy, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, 0, "", replyLiked, replyLikeCount, 0, ""}, ContentHtml: c.ParseMessage(replyContent, 0, ""), CreatedByName: replyCreatedByName, Avatar: replyAvatar, Level: 0}
ru.Init(puser.ID)
group, err := c.Groups.Get(ru.Group)
if err != nil {
return c.InternalError(err, w, r)
}
replyLines = strings.Count(replyContent, "\n")
if group.IsMod {
replyClassName = c.Config.StaffCSS
} else {
replyClassName = ""
}
replyAvatar, replyMicroAvatar = c.BuildAvatar(replyCreatedBy, replyAvatar)
if group.Tag != "" {
replyTag = group.Tag
} else if puser.ID == replyCreatedBy {
replyTag = phrases.GetTmplPhrase("profile_owner_tag")
ru.Tag = group.Tag
} else if puser.ID == ru.CreatedBy {
ru.Tag = phrases.GetTmplPhrase("profile_owner_tag")
} else {
replyTag = ""
ru.Tag = ""
}
replyLiked := false
replyLikeCount := 0
// TODO: Add a hook here
replyList = append(replyList, c.ReplyUser{rid, puser.ID, replyContent, c.ParseMessage(replyContent, 0, ""), replyCreatedBy, c.BuildProfileURL(c.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, 0, "", "", nil})
replyList = append(replyList, ru)
}
err = rows.Err()
if err != nil {

2
routes/reply.go
View File

@ -517,7 +517,6 @@ func ProfileReplyEditSubmit(w http.ResponseWriter, r *http.Request, user c.User,
if err != nil {
return c.InternalErrorJSQ(err, w, r, isJs)
}
// ? Does the admin understand that this group perm affects this?
if user.ID != creator.ID && !user.Perms.EditReply {
return c.NoPermissionsJSQ(w, r, user, isJs)
@ -555,7 +554,6 @@ func ProfileReplyDeleteSubmit(w http.ResponseWriter, r *http.Request, user c.Use
if err != nil {
return c.InternalErrorJSQ(err, w, r, isJs)
}
if user.ID != creator.ID && !user.Perms.DeleteReply {
return c.NoPermissionsJSQ(w, r, user, isJs)
}

154
routes/topic.go
View File

@ -21,7 +21,6 @@ import (
)
type TopicStmts struct {
getReplies *sql.Stmt
getLikedTopic *sql.Stmt
updateAttachs *sql.Stmt
}
@ -32,7 +31,6 @@ var topicStmts TopicStmts
func init() {
c.DbInits.Add(func(acc *qgen.Accumulator) error {
topicStmts = TopicStmts{
getReplies: acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.attachCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"),
getLikedTopic: acc.Select("likes").Columns("targetItem").Where("sentBy = ? && targetItem = ? && targetType = 'topics'").Prepare(),
// TODO: Less race-y attachment count updates
updateAttachs: acc.Update("topics").Set("attachCount = ?").Where("tid = ?").Prepare(),
@ -124,7 +122,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user c.User, header *c.He
// Calculate the offset
offset, page, lastPage := c.PageOffset(topic.PostCount, page, c.Config.ItemsPerPage)
pageList := c.Paginate(topic.PostCount, c.Config.ItemsPerPage, 5)
tpage := c.TopicPage{header, []c.ReplyUser{}, topic, forum, poll, c.Paginator{pageList, page, lastPage}}
tpage := c.TopicPage{header, nil, topic, forum, poll, c.Paginator{pageList, page, lastPage}}
// Get the replies if we have any...
if topic.PostCount > 0 {
@ -132,159 +130,15 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user c.User, header *c.He
if strings.HasPrefix(r.URL.Fragment, "post-") {
pFrag, _ = strconv.Atoi(strings.TrimPrefix(r.URL.Fragment, "post-"))
}
var likedMap map[int]int
if user.Liked > 0 {
likedMap = make(map[int]int)
}
var likedQueryList = []int{user.ID}
var attachMap map[int]int
if user.Perms.EditReply {
attachMap = make(map[int]int)
}
var attachQueryList = []int{}
rows, err := topicStmts.getReplies.Query(topic.ID, offset, c.Config.ItemsPerPage)
rlist, ogdesc, err := topic.Replies(offset, pFrag, &user)
if err == sql.ErrNoRows {
return c.LocalError("Bad Page. Some of the posts may have been deleted or you got here by directly typing in the page number.", w, r, user)
} else if err != nil {
return c.InternalError(err, w, r)
}
defer rows.Close()
// TODO: Factor the user fields out and embed a user struct instead
replyItem := c.ReplyUser{ClassName: ""}
for rows.Next() {
err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.AttachCount, &replyItem.ActionType)
if err != nil {
return c.InternalError(err, w, r)
}
replyItem.UserLink = c.BuildProfileURL(c.NameToSlug(replyItem.CreatedByName), replyItem.CreatedBy)
replyItem.ParentID = topic.ID
replyItem.ContentHtml = c.ParseMessage(replyItem.Content, topic.ParentID, "forums")
replyItem.ContentLines = strings.Count(replyItem.Content, "\n")
if replyItem.ID == pFrag {
header.OGDesc = replyItem.Content
if len(header.OGDesc) > 200 {
header.OGDesc = header.OGDesc[:197] + "..."
}
}
postGroup, err = c.Groups.Get(replyItem.Group)
if err != nil {
return c.InternalError(err, w, r)
}
if postGroup.IsMod {
replyItem.ClassName = c.Config.StaffCSS
} else {
replyItem.ClassName = ""
}
// TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the c.UserStore initialise this?
replyItem.Avatar, replyItem.MicroAvatar = c.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
replyItem.Tag = postGroup.Tag
// We really shouldn't have inline HTML, we should do something about this...
if replyItem.ActionType != "" {
var action string
aarr := strings.Split(replyItem.ActionType, "-")
switch aarr[0] {
case "lock":
action = "lock"
replyItem.ActionIcon = "&#x1F512;&#xFE0E"
case "unlock":
action = "unlock"
replyItem.ActionIcon = "&#x1F513;&#xFE0E"
case "stick":
action = "stick"
replyItem.ActionIcon = "&#x1F4CC;&#xFE0E"
case "unstick":
action = "unstick"
replyItem.ActionIcon = "&#x1F4CC;&#xFE0E"
case "move":
if len(aarr) == 2 {
fid, _ := strconv.Atoi(aarr[1])
forum, err := c.Forums.Get(fid)
if err == nil {
replyItem.ActionType = phrases.GetTmplPhrasef("topic.action_topic_move_dest", forum.Link, forum.Name, replyItem.UserLink, replyItem.CreatedByName)
} else {
action = "move"
}
} else {
action = "move"
}
replyItem.ActionIcon = ""
default:
// TODO: Only fire this off if a corresponding phrase for the ActionType doesn't exist? Or maybe have some sort of action registry?
replyItem.ActionType = phrases.GetTmplPhrasef("topic.action_topic_default", replyItem.ActionType)
replyItem.ActionIcon = ""
}
if action != "" {
replyItem.ActionType = phrases.GetTmplPhrasef("topic.action_topic_"+action, replyItem.UserLink, replyItem.CreatedByName)
}
}
if replyItem.LikeCount > 0 && user.Liked > 0 {
likedMap[replyItem.ID] = len(tpage.ItemList)
likedQueryList = append(likedQueryList, replyItem.ID)
}
if user.Perms.EditReply && replyItem.AttachCount > 0 {
attachMap[replyItem.ID] = len(tpage.ItemList)
attachQueryList = append(attachQueryList, replyItem.ID)
}
header.Hooks.VhookNoRet("topic_reply_row_assign", &tpage, &replyItem)
// TODO: Use a pointer instead to make it easier to abstract this loop? What impact would this have on escape analysis?
tpage.ItemList = append(tpage.ItemList, replyItem)
//log.Printf("r: %d-%d", replyItem.ID, len(tpage.ItemList)-1)
}
err = rows.Err()
if err != nil {
return c.InternalError(err, w, r)
}
// TODO: Add a config setting to disable the liked query for a burst of extra speed
if user.Liked > 0 && len(likedQueryList) > 1 /*&& user.LastLiked <= time.Now()*/ {
// TODO: Abstract this
rows, err := qgen.NewAcc().Select("likes").Columns("targetItem").Where("sentBy = ? AND targetType = 'replies'").In("targetItem", likedQueryList[1:]).Query(user.ID)
if err != nil && err != sql.ErrNoRows {
return c.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var likeRid int
err := rows.Scan(&likeRid)
if err != nil {
return c.InternalError(err, w, r)
}
tpage.ItemList[likedMap[likeRid]].Liked = true
}
err = rows.Err()
if err != nil {
return c.InternalError(err, w, r)
}
}
if user.Perms.EditReply && len(attachQueryList) > 0 {
//log.Printf("attachQueryList: %+v\n", attachQueryList)
amap, err := c.Attachments.BulkMiniGetList("replies", attachQueryList)
if err != nil && err != sql.ErrNoRows {
return c.InternalError(err, w, r)
}
//log.Printf("amap: %+v\n", amap)
//log.Printf("attachMap: %+v\n", attachMap)
for id, attach := range amap {
//log.Print("id:", id)
tpage.ItemList[attachMap[id]].Attachments = attach
/*for _, a := range attach {
log.Printf("a: %+v\n", a)
}*/
}
}
header.OGDesc = ogdesc
tpage.ItemList = rlist
}
header.Zone = "view_topic"

74
templates/panel_analytics_active_memory.html
View File

@ -21,76 +21,4 @@
</div>
{{else}}<div class="rowitem passive rowmsg">{{lang "panel_statistics_memory_no_memory"}}</div>{{end}}
</div>
<script>
let rawLabels = [{{range .Graph.Labels}}
{{.}},{{end}}
];
let seriesData = [{{range .Graph.Series}}[{{range .}}
{{.}},{{end}}
],{{end}}
];
let legendNames = [{{range .Graph.Legends}}
{{.}},{{end}}
];
(function(window, document, Chartist) {
'use strict';
Chartist.plugins = Chartist.plugins || {};
Chartist.plugins.byteUnits = function(options) {
options = Chartist.extend({}, {}, options);
return function byteUnits(chart) {
if(!chart instanceof Chartist.Line) return;
chart.on('created', function() {
console.log("running created")
const vbits = document.getElementsByClassName("ct-vertical");
if(vbits==null) return;
let tbits = [];
for(let i = 0; i < vbits.length; i++) {
tbits[i] = vbits[i].innerHTML;
}
console.log("tbits:",tbits);
const calc = (places) => {
if(places==3) return;
const matcher = vbits[0].innerHTML;
let allMatch = true;
for(let i = 0; i < tbits.length; i++) {
let val = convertByteUnit(tbits[i], places);
if(val!=matcher) allMatch = false;
vbits[i].innerHTML = val;
}
if(allMatch) calc(places + 1);
}
calc(0);
});
};
};
}(window, document, Chartist));
const noPre = ["twelve-hours","one-day","two-days","one-week","one-month"];
const preStats = () => {
if((!"{{.TimeRange}}" in noPre) && seriesData.length > 0 && seriesData[0].length > 12) {
let elem = document.getElementsByClassName("colstack_graph_holder")[0];
let w = elem.clientWidth;
console.log("w:",w);
elem.classList.add("scrolly");
console.log("elem.clientWidth:",elem.clientWidth);
elem.setAttribute("style","width:"+w+"px;");
console.log("elem.clientWidth:",elem.clientWidth);
}
};
addInitHook("after_phrases", () => {
addInitHook("end_init", () => {
addInitHook("analytics_loaded", () => {
preStats();
buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}",legendNames,true);
});
});
});
</script>
{{template "panel_analytics_script_memory.html" . }}

74
templates/panel_analytics_memory.html
View File

@ -21,76 +21,4 @@
</div>
{{else}}<div class="rowitem passive rowmsg">{{lang "panel_statistics_memory_no_memory"}}</div>{{end}}
</div>
<script>
let rawLabels = [{{range .Graph.Labels}}
{{.}},{{end}}
];
let seriesData = [{{range .Graph.Series}}[{{range .}}
{{.}},{{end}}
],{{end}}
];
let legendNames = [{{range .Graph.Legends}}
{{.}},{{end}}
];
(function(window, document, Chartist) {
'use strict';
Chartist.plugins = Chartist.plugins || {};
Chartist.plugins.byteUnits = function(options) {
options = Chartist.extend({}, {}, options);
return function byteUnits(chart) {
if(!chart instanceof Chartist.Line) return;
chart.on('created', function() {
console.log("running created")
const vbits = document.getElementsByClassName("ct-vertical");
if(vbits==null) return;
let tbits = [];
for(let i = 0; i < vbits.length; i++) {
tbits[i] = vbits[i].innerHTML;
}
console.log("tbits:",tbits);
const calc = (places) => {
if(places==3) return;
const matcher = vbits[0].innerHTML;
let allMatch = true;
for(let i = 0; i < tbits.length; i++) {
let val = convertByteUnit(tbits[i], places);
if(val!=matcher) allMatch = false;
vbits[i].innerHTML = val;
}
if(allMatch) calc(places + 1);
}
calc(0);
});
};
};
}(window, document, Chartist));
const noPre = ["twelve-hours","one-day","two-days","one-week","one-month"];
const preStats = () => {
if((!"{{.TimeRange}}" in noPre) && seriesData.length > 0 && seriesData[0].length > 12) {
let elem = document.getElementsByClassName("colstack_graph_holder")[0];
let w = elem.clientWidth;
console.log("w:",w);
elem.classList.add("scrolly");
console.log("elem.clientWidth:",elem.clientWidth);
elem.setAttribute("style","width:"+w+"px;");
console.log("elem.clientWidth:",elem.clientWidth);
}
};
addInitHook("after_phrases", () => {
addInitHook("end_init", () => {
addInitHook("analytics_loaded", () => {
preStats();
buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}",legendNames,true);
});
});
});
</script>
{{template "panel_analytics_script_memory.html" . }}

13
templates/panel_analytics_script.html
View File

@ -10,22 +10,9 @@ let seriesData = [{{range .Graph.Series}}[{{range .}}
let legendNames = [{{range .Graph.Legends}}
{{.}},{{end}}
];
const noPre = ["twelve-hours","one-day","two-days","one-week","one-month"];
const preStats = () => {
if((!"{{.TimeRange}}" in noPre) && seriesData.length > 0 && seriesData[0].length > 12) {
let elem = document.getElementsByClassName("colstack_graph_holder")[0];
let w = elem.clientWidth;
console.log("w:",w);
elem.classList.add("scrolly");
console.log("elem.clientWidth:",elem.clientWidth);
elem.setAttribute("style","width:"+w+"px;");
console.log("elem.clientWidth:",elem.clientWidth);
}
};
addInitHook("after_phrases", () => {
addInitHook("end_init", () => {
addInitHook("analytics_loaded", () => {
preStats();
buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}",legendNames);
});
});

21
templates/panel_analytics_script_memory.html Normal file
View File

@ -0,0 +1,21 @@
<script>
let rawLabels = [{{range .Graph.Labels}}
{{.}},{{end}}
];
let seriesData = [{{range .Graph.Series}}[{{range .}}
{{.}},{{end}}
],{{end}}
];
let legendNames = [{{range .Graph.Legends}}
{{.}},{{end}}
];
addInitHook("after_phrases", () => {
addInitHook("end_init", () => {
addInitHook("analytics_loaded", () => {
memStuff(window, document, Chartist);
buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}",legendNames,true);
});
});
});
</script>

2
templates/panel_debug.html
View File

@ -79,7 +79,7 @@
<div class="grid_item grid_stat"><span>{{.TCache}}</span></div>
<div class="grid_item grid_stat"><span>{{.UCache}}</span></div>
<div class="grid_item grid_stat"><span>0</span></div>
<div class="grid_item grid_stat"><span>{{.RCache}}</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>Topic List</span></div>