More conversations progress, still not usable yet.

Flush the debug page incrementally.
Eliminate a tiny bit of code in PreparseMessage.
Shorten some of the pointer capture variables in the poll store.
Tweak the advanced installation instructions for EasyJSON to make it more repeatable.

Added the ConvoKey config setting.
Added /.git/ disk usage to the debug page, if it exists.
Added a UserStore.BulkGetMap multi-user test.
This commit is contained in:
Azareal 2019-06-16 16:30:58 +10:00
parent 0334e6c68c
commit a5441f18de
9 changed files with 251 additions and 56 deletions

View File

@ -1,37 +1,173 @@
package common
import "database/sql"
import "github.com/Azareal/Gosora/query_gen"
import (
"io"
"time"
"database/sql"
"encoding/hex"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
qgen "github.com/Azareal/Gosora/query_gen"
)
/*
conversations
conversations_posts
*/
var ConvoPostProcess ConvoPostProcessor = NewDefaultConvoPostProcessor()
type ConvoPostProcessor interface {
OnLoad(co *ConversationPost) (*ConversationPost, error)
OnSave(co *ConversationPost) (*ConversationPost, error)
}
type DefaultConvoPostProcessor struct {
}
func NewDefaultConvoPostProcessor() *DefaultConvoPostProcessor {
return &DefaultConvoPostProcessor{}
}
func (pr *DefaultConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) {
return co, nil
}
func (pr *DefaultConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) {
return co, nil
}
type AesConvoPostProcessor struct {
}
func NewAesConvoPostProcessor() *AesConvoPostProcessor {
return &AesConvoPostProcessor{}
}
func (pr *AesConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) {
if co.Post != "aes" {
return co, nil
}
key, _ := hex.DecodeString(Config.ConvoKey)
ciphertext, err := hex.DecodeString(co.Body)
if err != nil {
return nil, err
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonceSize := aesgcm.NonceSize()
if len(ciphertext) < nonceSize {
return nil, err
}
nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, err
}
lco := *co
lco.Body = string(plaintext)
return &lco, nil
}
func (pr *AesConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) {
key, _ := hex.DecodeString(Config.ConvoKey)
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
nonce := make([]byte, 12)
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, err
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
ciphertext := aesgcm.Seal(nil, nonce, []byte(co.Body), nil)
lco := *co
lco.Body = hex.EncodeToString(ciphertext)
lco.Post = "aes"
return &lco, nil
}
var convoStmts ConvoStmts
type ConvoStmts struct {
getPosts *sql.Stmt
edit *sql.Stmt
create *sql.Stmt
editPost *sql.Stmt
createPost *sql.Stmt
}
func init() {
/*DbInits.Add(func(acc *qgen.Accumulator) error {
/*func init() {
DbInits.Add(func(acc *qgen.Accumulator) error {
convoStmts = ConvoStmts{
edit: acc.Update("conversations").Set("participants = ?").Where("cid = ?").Prepare(),
create: acc.Insert("conversations").Columns("participants").Fields("?").Prepare(),
getPosts: acc.Select("conversations_posts").Columns("pid, body, post").Where("cid = ?").Prepare(),
edit: acc.Update("conversations").Set("participants = ?, lastReplyAt = ?").Where("cid = ?").Prepare(),
create: acc.Insert("conversations").Columns("participants, createdAt, lastReplyAt").Fields("?,UTC_TIMESTAMP(),UTC_TIMESTAMP()").Prepare(),
editPost: acc.Update("conversations_posts").Set("body = ?").Where("cid = ?").Prepare(),
createPost: acc.Insert("conversations_posts").Columns("body").Fields("?").Prepare(),
}
return acc.FirstError()
})*/
}
})
}*/
type Conversation struct {
ID int
Participants string
CreatedAt time.Time
LastReplyAt time.Time
}
func (co *Conversation) Posts(offset int) (posts []*ConversationPost, err error) {
rows, err := convoStmts.getPosts.Query(co.ID, offset, Config.ItemsPerPage)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
convo := &ConversationPost{CID: co.ID}
err := rows.Scan(&convo.ID, &convo.Body, &convo.Post)
if err != nil {
return nil, err
}
convo, err = ConvoPostProcess.OnLoad(convo)
if err != nil {
return nil, err
}
posts = append(posts, convo)
}
err = rows.Err()
if err != nil {
return nil, err
}
return posts, err
}
func (co *Conversation) Update() error {
_, err := convoStmts.edit.Exec(co.Participants, co.ID)
_, err := convoStmts.edit.Exec(co.Participants, co.CreatedAt, co.LastReplyAt, co.ID)
return err
}
@ -46,6 +182,35 @@ func (co *Conversation) Create() (int, error) {
}
type ConversationPost struct {
ID int
CID int
Body string
Post string // aes, ''
}
func (co *ConversationPost) Update() error {
lco, err := ConvoPostProcess.OnSave(co)
if err != nil {
return err
}
//GetHookTable().VhookNoRet("convo_post_update", lco)
_, err = convoStmts.editPost.Exec(lco.Body, lco.ID)
return err
}
func (co *ConversationPost) Create() (int, error) {
lco, err := ConvoPostProcess.OnSave(co)
if err != nil {
return 0, err
}
//GetHookTable().VhookNoRet("convo_post_create", lco)
res, err := convoStmts.createPost.Exec(lco.Body)
if err != nil {
return 0, err
}
lastID, err := res.LastInsertId()
return int(lastID), err
}
type ConversationStore interface {
@ -62,7 +227,7 @@ type DefaultConversationStore struct {
func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationStore, error) {
return &DefaultConversationStore{
get: acc.Select("conversations").Columns("participants").Where("cid = ?").Prepare(),
get: acc.Select("conversations").Columns("participants, createdAt, lastReplyAt").Where("cid = ?").Prepare(),
delete: acc.Delete("conversations").Where("cid = ?").Prepare(),
count: acc.Count("conversations").Prepare(),
}, acc.FirstError()
@ -70,7 +235,7 @@ func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationSto
func (s *DefaultConversationStore) Get(id int) (*Conversation, error) {
convo := &Conversation{ID: id}
err := s.get.QueryRow(id).Scan(&convo.Participants)
err := s.get.QueryRow(id).Scan(&convo.Participants, &convo.CreatedAt, &convo.LastReplyAt)
return nil, err
}

View File

@ -642,6 +642,7 @@ type DebugPageDisk struct {
Avatars int
Logs int
Backups int
Git int
}
type PanelDebugPage struct {

View File

@ -338,12 +338,13 @@ func PreparseMessage(msg string) string {
tags := allowedTags[char]
if len(tags) == 0 {
//fmt.Println("couldn't find char in allowedTags")
msg += "&"
if closeTag {
//msg += "&lt;/"
msg += "&"
//msg += "&"
i -= 5
} else {
msg += "&"
//msg += "&"
i -= 4
}
continue

View File

@ -90,38 +90,38 @@ func NewDefaultPollStore(cache PollCache) (*DefaultPollStore, error) {
}, acc.FirstError()
}
func (store *DefaultPollStore) Exists(id int) bool {
err := store.exists.QueryRow(id).Scan(&id)
func (s *DefaultPollStore) Exists(id int) bool {
err := s.exists.QueryRow(id).Scan(&id)
if err != nil && err != ErrNoRows {
LogError(err)
}
return err != ErrNoRows
}
func (store *DefaultPollStore) Get(id int) (*Poll, error) {
poll, err := store.cache.Get(id)
func (s *DefaultPollStore) Get(id int) (*Poll, error) {
poll, err := s.cache.Get(id)
if err == nil {
return poll, nil
}
poll = &Poll{ID: id}
var optionTxt []byte
err = store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
err = s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
if err != nil {
return nil, err
}
err = json.Unmarshal(optionTxt, &poll.Options)
if err == nil {
poll.QuickOptions = store.unpackOptionsMap(poll.Options)
store.cache.Set(poll)
poll.QuickOptions = s.unpackOptionsMap(poll.Options)
s.cache.Set(poll)
}
return poll, err
}
// TODO: Optimise the query to avoid preparing it on the spot? Maybe, use knowledge of the most common IN() parameter counts?
// TODO: ID of 0 should always error?
func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) {
func (s *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) {
var idCount = len(ids)
list = make(map[int]*Poll)
if idCount == 0 {
@ -129,7 +129,7 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
}
var stillHere []int
sliceList := store.cache.BulkGet(ids)
sliceList := s.cache.BulkGet(ids)
for i, sliceItem := range sliceList {
if sliceItem != nil {
list[sliceItem.ID] = sliceItem
@ -170,8 +170,8 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
if err != nil {
return list, err
}
poll.QuickOptions = store.unpackOptionsMap(poll.Options)
store.cache.Set(poll)
poll.QuickOptions = s.unpackOptionsMap(poll.Options)
s.cache.Set(poll)
list[poll.ID] = poll
}
@ -205,27 +205,27 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
return list, err
}
func (store *DefaultPollStore) Reload(id int) error {
func (s *DefaultPollStore) Reload(id int) error {
poll := &Poll{ID: id}
var optionTxt []byte
err := store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
err := s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
if err != nil {
store.cache.Remove(id)
s.cache.Remove(id)
return err
}
err = json.Unmarshal(optionTxt, &poll.Options)
if err != nil {
store.cache.Remove(id)
s.cache.Remove(id)
return err
}
poll.QuickOptions = store.unpackOptionsMap(poll.Options)
_ = store.cache.Set(poll)
poll.QuickOptions = s.unpackOptionsMap(poll.Options)
_ = s.cache.Set(poll)
return nil
}
func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption {
func (s *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption {
options := make([]PollOption, len(rawOptions))
for id, option := range rawOptions {
options[id] = PollOption{id, option}
@ -234,27 +234,27 @@ func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []Pol
}
// TODO: Use a transaction for this?
func (store *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error {
_, err := store.addVote.Exec(pollID, uid, optionIndex, ipaddress)
func (s *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error {
_, err := s.addVote.Exec(pollID, uid, optionIndex, ipaddress)
if err != nil {
return err
}
_, err = store.incrementVoteCount.Exec(pollID)
_, err = s.incrementVoteCount.Exec(pollID)
if err != nil {
return err
}
_, err = store.incrementVoteCountForOption.Exec(optionIndex, pollID)
_, err = s.incrementVoteCountForOption.Exec(optionIndex, pollID)
return err
}
// TODO: Use a transaction for this
func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) {
func (s *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) {
pollOptionsTxt, err := json.Marshal(pollOptions)
if err != nil {
return 0, err
}
res, err := store.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt)
res, err := s.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt)
if err != nil {
return 0, err
}
@ -265,23 +265,24 @@ func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions
}
for i := 0; i < len(pollOptions); i++ {
_, err := store.createPollOption.Exec(lastID, i)
_, err := s.createPollOption.Exec(lastID, i)
if err != nil {
return 0, err
}
}
return int(lastID), parent.SetPoll(int(lastID)) // TODO: Delete the poll (and options) if SetPoll fails
}
func (store *DefaultPollStore) SetCache(cache PollCache) {
store.cache = cache
func (s *DefaultPollStore) SetCache(cache PollCache) {
s.cache = cache
}
// TODO: We're temporarily doing this so that you can do ucache != nil in getTopicUser. Refactor it.
func (store *DefaultPollStore) GetCache() PollCache {
_, ok := store.cache.(*NullPollCache)
func (s *DefaultPollStore) GetCache() PollCache {
_, ok := s.cache.(*NullPollCache)
if ok {
return nil
}
return store.cache
return s.cache
}

View File

@ -62,6 +62,7 @@ type config struct {
SslPrivkey string
SslFullchain string
HashAlgo string // Defaults to bcrypt, and in the future, possibly something stronger
ConvoKey string
MaxRequestSizeStr string
MaxRequestSize int

View File

@ -146,7 +146,7 @@ go build "./cmd/install"
install.exe
go get github.com/mailru/easyjson/...
go get -u github.com/mailru/easyjson/...
easyjson -pkg common

View File

@ -184,6 +184,31 @@ func userStoreTest(t *testing.T, newUserID int) {
expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
}
userList, _ = c.Users.BulkGetMap([]int{1,uid})
expect(t, len(userList) == 2, fmt.Sprintf("Returned map should have two results, not %d", len(userList)))
if ucache != nil {
expectIntToBeX(t, ucache.Length(), 2, "User cache length should be 2, not %d")
user, err = ucache.Get(1)
recordMustExist(t, err, "Couldn't find UID #%d in the cache", 1)
expect(t, user.ID == 1, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
user, err = ucache.Get(newUserID)
recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID)
expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
ucache.Flush()
}
user, err = c.Users.Get(newUserID)
recordMustExist(t, err, "Couldn't find UID #%d", newUserID)
expectUser(user, newUserID, "Sam", 5, false, false, false, false)
if ucache != nil {
expectIntToBeX(t, ucache.Length(), 1, "User cache length should be 1, not %d")
user, err = ucache.Get(newUserID)
recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID)
expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
}
err = user.Activate()
expectNilErr(t, err)
expectIntToBeX(t, user.Group, 5, "Sam should still be in group 5 in this copy")

View File

@ -113,8 +113,9 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
if fErr != nil {
return c.InternalError(fErr,w,r)
}
gitSize, _ := c.DirSize("./.git")
debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize}
debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize,gitSize}
pi := c.PanelDebugPage{basePage, goVersion, dbVersion, uptime, openConnCount, qgen.Builder.GetAdapter().GetName(), goroutines, cpus, memStats, debugCache, debugDatabase, debugDisk}
return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_dashboard_right", "debug_page", "panel_debug", pi})

View File

@ -32,7 +32,7 @@
<div class="colstack_item colstack_head colstack_sub_head">
<div class="rowitem"><h2>Memory Statistics</h2></div>
</div>
</div>{{flush}}
<div id="panel_debug" class="colstack_grid">
<div class="grid_item grid_stat grid_stat_head"><span>Sys</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>HeapSys</span></div>
@ -92,7 +92,7 @@
</div>
<div class="colstack_item colstack_head colstack_sub_head">
<div class="rowitem"><h2>Database</h2></div>
</div>
</div>{{flush}}
<div id="panel_debug" class="colstack_grid">
<div class="grid_item grid_stat grid_stat_head"><span>Topics</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>Users</span></div>
@ -167,7 +167,7 @@
</div>
<div class="colstack_item colstack_head colstack_sub_head">
<div class="rowitem"><h2>Disk</h2></div>
</div>
</div>{{flush}}
<div id="panel_debug" class="colstack_grid">
<div class="grid_item grid_stat grid_stat_head"><span>Static Files</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>Attachments</span></div>
@ -180,9 +180,9 @@
<div class="grid_item grid_stat grid_stat_head"><span>Log Files</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>Backups</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>???</span></div>
<div class="grid_item grid_stat grid_stat_head"><span>Git</span></div>
<div class="grid_item grid_stat"><span>{{bunit .Disk.Logs}}</span></div>
<div class="grid_item grid_stat"><span>{{bunit .Disk.Backups}}</span></div>
<div class="grid_item grid_stat"><span>?</span></div>
<div class="grid_item grid_stat"><span>{{bunit .Disk.Git}}</span></div>
</div>