From a5441f18de43c3f2be523f9b9a7f5cb206cbfd9b Mon Sep 17 00:00:00 2001 From: Azareal Date: Sun, 16 Jun 2019 16:30:58 +1000 Subject: [PATCH] More conversations progress, still not usable yet. Flush the debug page incrementally. Eliminate a tiny bit of code in PreparseMessage. Shorten some of the pointer capture variables in the poll store. Tweak the advanced installation instructions for EasyJSON to make it more repeatable. Added the ConvoKey config setting. Added /.git/ disk usage to the debug page, if it exists. Added a UserStore.BulkGetMap multi-user test. --- common/conversations.go | 199 +++++++++++++++++++++++++++++++++---- common/pages.go | 1 + common/parser.go | 5 +- common/poll_store.go | 61 ++++++------ common/site.go | 1 + docs/installation.md | 2 +- misc_test.go | 25 +++++ routes/panel/debug.go | 3 +- templates/panel_debug.html | 10 +- 9 files changed, 251 insertions(+), 56 deletions(-) diff --git a/common/conversations.go b/common/conversations.go index 20daff81..54a173ca 100644 --- a/common/conversations.go +++ b/common/conversations.go @@ -1,37 +1,173 @@ package common -import "database/sql" -import "github.com/Azareal/Gosora/query_gen" +import ( + "io" + "time" + "database/sql" + "encoding/hex" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + + qgen "github.com/Azareal/Gosora/query_gen" +) /* conversations conversations_posts */ +var ConvoPostProcess ConvoPostProcessor = NewDefaultConvoPostProcessor() + +type ConvoPostProcessor interface { + OnLoad(co *ConversationPost) (*ConversationPost, error) + OnSave(co *ConversationPost) (*ConversationPost, error) +} + +type DefaultConvoPostProcessor struct { +} + +func NewDefaultConvoPostProcessor() *DefaultConvoPostProcessor { + return &DefaultConvoPostProcessor{} +} + +func (pr *DefaultConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) { + return co, nil +} + +func (pr *DefaultConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) { + return co, nil +} + +type AesConvoPostProcessor struct { +} + +func NewAesConvoPostProcessor() *AesConvoPostProcessor { + return &AesConvoPostProcessor{} +} + +func (pr *AesConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) { + if co.Post != "aes" { + return co, nil + } + key, _ := hex.DecodeString(Config.ConvoKey) + + ciphertext, err := hex.DecodeString(co.Body) + if err != nil { + return nil, err + } + + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonceSize := aesgcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, err + } + + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, err + } + + lco := *co + lco.Body = string(plaintext) + return &lco, nil +} + +func (pr *AesConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) { + key, _ := hex.DecodeString(Config.ConvoKey) + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + nonce := make([]byte, 12) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + ciphertext := aesgcm.Seal(nil, nonce, []byte(co.Body), nil) + + lco := *co + lco.Body = hex.EncodeToString(ciphertext) + lco.Post = "aes" + return &lco, nil +} + var convoStmts ConvoStmts type ConvoStmts struct { - edit *sql.Stmt + getPosts *sql.Stmt + edit *sql.Stmt create *sql.Stmt + + editPost *sql.Stmt + createPost *sql.Stmt } -func init() { - /*DbInits.Add(func(acc *qgen.Accumulator) error { +/*func init() { + DbInits.Add(func(acc *qgen.Accumulator) error { convoStmts = ConvoStmts{ - edit: acc.Update("conversations").Set("participants = ?").Where("cid = ?").Prepare(), - create: acc.Insert("conversations").Columns("participants").Fields("?").Prepare(), + getPosts: acc.Select("conversations_posts").Columns("pid, body, post").Where("cid = ?").Prepare(), + edit: acc.Update("conversations").Set("participants = ?, lastReplyAt = ?").Where("cid = ?").Prepare(), + create: acc.Insert("conversations").Columns("participants, createdAt, lastReplyAt").Fields("?,UTC_TIMESTAMP(),UTC_TIMESTAMP()").Prepare(), + + editPost: acc.Update("conversations_posts").Set("body = ?").Where("cid = ?").Prepare(), + createPost: acc.Insert("conversations_posts").Columns("body").Fields("?").Prepare(), } return acc.FirstError() - })*/ -} + }) +}*/ type Conversation struct { - ID int + ID int Participants string + CreatedAt time.Time + LastReplyAt time.Time +} + +func (co *Conversation) Posts(offset int) (posts []*ConversationPost, err error) { + rows, err := convoStmts.getPosts.Query(co.ID, offset, Config.ItemsPerPage) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + convo := &ConversationPost{CID: co.ID} + err := rows.Scan(&convo.ID, &convo.Body, &convo.Post) + if err != nil { + return nil, err + } + convo, err = ConvoPostProcess.OnLoad(convo) + if err != nil { + return nil, err + } + posts = append(posts, convo) + } + err = rows.Err() + if err != nil { + return nil, err + } + + return posts, err } func (co *Conversation) Update() error { - _, err := convoStmts.edit.Exec(co.Participants, co.ID) + _, err := convoStmts.edit.Exec(co.Participants, co.CreatedAt, co.LastReplyAt, co.ID) return err } @@ -46,6 +182,35 @@ func (co *Conversation) Create() (int, error) { } type ConversationPost struct { + ID int + CID int + Body string + Post string // aes, '' +} + +func (co *ConversationPost) Update() error { + lco, err := ConvoPostProcess.OnSave(co) + if err != nil { + return err + } + //GetHookTable().VhookNoRet("convo_post_update", lco) + _, err = convoStmts.editPost.Exec(lco.Body, lco.ID) + return err +} + +func (co *ConversationPost) Create() (int, error) { + lco, err := ConvoPostProcess.OnSave(co) + if err != nil { + return 0, err + } + //GetHookTable().VhookNoRet("convo_post_create", lco) + res, err := convoStmts.createPost.Exec(lco.Body) + if err != nil { + return 0, err + } + + lastID, err := res.LastInsertId() + return int(lastID), err } type ConversationStore interface { @@ -55,22 +220,22 @@ type ConversationStore interface { } type DefaultConversationStore struct { - get *sql.Stmt + get *sql.Stmt delete *sql.Stmt - count *sql.Stmt + count *sql.Stmt } func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationStore, error) { return &DefaultConversationStore{ - get: acc.Select("conversations").Columns("participants").Where("cid = ?").Prepare(), + get: acc.Select("conversations").Columns("participants, createdAt, lastReplyAt").Where("cid = ?").Prepare(), delete: acc.Delete("conversations").Where("cid = ?").Prepare(), - count: acc.Count("conversations").Prepare(), + count: acc.Count("conversations").Prepare(), }, acc.FirstError() } func (s *DefaultConversationStore) Get(id int) (*Conversation, error) { - convo := &Conversation{ID:id} - err := s.get.QueryRow(id).Scan(&convo.Participants) + convo := &Conversation{ID: id} + err := s.get.QueryRow(id).Scan(&convo.Participants, &convo.CreatedAt, &convo.LastReplyAt) return nil, err } diff --git a/common/pages.go b/common/pages.go index eb25c6a1..42af2971 100644 --- a/common/pages.go +++ b/common/pages.go @@ -642,6 +642,7 @@ type DebugPageDisk struct { Avatars int Logs int Backups int + Git int } type PanelDebugPage struct { diff --git a/common/parser.go b/common/parser.go index f73709b1..ee7bc11f 100644 --- a/common/parser.go +++ b/common/parser.go @@ -338,12 +338,13 @@ func PreparseMessage(msg string) string { tags := allowedTags[char] if len(tags) == 0 { //fmt.Println("couldn't find char in allowedTags") + msg += "&" if closeTag { //msg += "</" - msg += "&" + //msg += "&" i -= 5 } else { - msg += "&" + //msg += "&" i -= 4 } continue diff --git a/common/poll_store.go b/common/poll_store.go index cc23b69f..942ea1d8 100644 --- a/common/poll_store.go +++ b/common/poll_store.go @@ -90,38 +90,38 @@ func NewDefaultPollStore(cache PollCache) (*DefaultPollStore, error) { }, acc.FirstError() } -func (store *DefaultPollStore) Exists(id int) bool { - err := store.exists.QueryRow(id).Scan(&id) +func (s *DefaultPollStore) Exists(id int) bool { + err := s.exists.QueryRow(id).Scan(&id) if err != nil && err != ErrNoRows { LogError(err) } return err != ErrNoRows } -func (store *DefaultPollStore) Get(id int) (*Poll, error) { - poll, err := store.cache.Get(id) +func (s *DefaultPollStore) Get(id int) (*Poll, error) { + poll, err := s.cache.Get(id) if err == nil { return poll, nil } poll = &Poll{ID: id} var optionTxt []byte - err = store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount) + err = s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount) if err != nil { return nil, err } err = json.Unmarshal(optionTxt, &poll.Options) if err == nil { - poll.QuickOptions = store.unpackOptionsMap(poll.Options) - store.cache.Set(poll) + poll.QuickOptions = s.unpackOptionsMap(poll.Options) + s.cache.Set(poll) } return poll, err } // TODO: Optimise the query to avoid preparing it on the spot? Maybe, use knowledge of the most common IN() parameter counts? // TODO: ID of 0 should always error? -func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) { +func (s *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) { var idCount = len(ids) list = make(map[int]*Poll) if idCount == 0 { @@ -129,7 +129,7 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er } var stillHere []int - sliceList := store.cache.BulkGet(ids) + sliceList := s.cache.BulkGet(ids) for i, sliceItem := range sliceList { if sliceItem != nil { list[sliceItem.ID] = sliceItem @@ -170,8 +170,8 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er if err != nil { return list, err } - poll.QuickOptions = store.unpackOptionsMap(poll.Options) - store.cache.Set(poll) + poll.QuickOptions = s.unpackOptionsMap(poll.Options) + s.cache.Set(poll) list[poll.ID] = poll } @@ -205,27 +205,27 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er return list, err } -func (store *DefaultPollStore) Reload(id int) error { +func (s *DefaultPollStore) Reload(id int) error { poll := &Poll{ID: id} var optionTxt []byte - err := store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount) + err := s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount) if err != nil { - store.cache.Remove(id) + s.cache.Remove(id) return err } err = json.Unmarshal(optionTxt, &poll.Options) if err != nil { - store.cache.Remove(id) + s.cache.Remove(id) return err } - poll.QuickOptions = store.unpackOptionsMap(poll.Options) - _ = store.cache.Set(poll) + poll.QuickOptions = s.unpackOptionsMap(poll.Options) + _ = s.cache.Set(poll) return nil } -func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption { +func (s *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption { options := make([]PollOption, len(rawOptions)) for id, option := range rawOptions { options[id] = PollOption{id, option} @@ -234,27 +234,27 @@ func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []Pol } // TODO: Use a transaction for this? -func (store *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error { - _, err := store.addVote.Exec(pollID, uid, optionIndex, ipaddress) +func (s *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error { + _, err := s.addVote.Exec(pollID, uid, optionIndex, ipaddress) if err != nil { return err } - _, err = store.incrementVoteCount.Exec(pollID) + _, err = s.incrementVoteCount.Exec(pollID) if err != nil { return err } - _, err = store.incrementVoteCountForOption.Exec(optionIndex, pollID) + _, err = s.incrementVoteCountForOption.Exec(optionIndex, pollID) return err } // TODO: Use a transaction for this -func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) { +func (s *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) { pollOptionsTxt, err := json.Marshal(pollOptions) if err != nil { return 0, err } - res, err := store.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt) + res, err := s.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt) if err != nil { return 0, err } @@ -265,23 +265,24 @@ func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions } for i := 0; i < len(pollOptions); i++ { - _, err := store.createPollOption.Exec(lastID, i) + _, err := s.createPollOption.Exec(lastID, i) if err != nil { return 0, err } } + return int(lastID), parent.SetPoll(int(lastID)) // TODO: Delete the poll (and options) if SetPoll fails } -func (store *DefaultPollStore) SetCache(cache PollCache) { - store.cache = cache +func (s *DefaultPollStore) SetCache(cache PollCache) { + s.cache = cache } // TODO: We're temporarily doing this so that you can do ucache != nil in getTopicUser. Refactor it. -func (store *DefaultPollStore) GetCache() PollCache { - _, ok := store.cache.(*NullPollCache) +func (s *DefaultPollStore) GetCache() PollCache { + _, ok := s.cache.(*NullPollCache) if ok { return nil } - return store.cache + return s.cache } diff --git a/common/site.go b/common/site.go index a90d604e..682a3483 100644 --- a/common/site.go +++ b/common/site.go @@ -62,6 +62,7 @@ type config struct { SslPrivkey string SslFullchain string HashAlgo string // Defaults to bcrypt, and in the future, possibly something stronger + ConvoKey string MaxRequestSizeStr string MaxRequestSize int diff --git a/docs/installation.md b/docs/installation.md index 3fffe217..47302960 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -146,7 +146,7 @@ go build "./cmd/install" install.exe -go get github.com/mailru/easyjson/... +go get -u github.com/mailru/easyjson/... easyjson -pkg common diff --git a/misc_test.go b/misc_test.go index 85ef51f5..fdd6d2f8 100644 --- a/misc_test.go +++ b/misc_test.go @@ -184,6 +184,31 @@ func userStoreTest(t *testing.T, newUserID int) { expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID)) } + userList, _ = c.Users.BulkGetMap([]int{1,uid}) + expect(t, len(userList) == 2, fmt.Sprintf("Returned map should have two results, not %d", len(userList))) + + if ucache != nil { + expectIntToBeX(t, ucache.Length(), 2, "User cache length should be 2, not %d") + user, err = ucache.Get(1) + recordMustExist(t, err, "Couldn't find UID #%d in the cache", 1) + expect(t, user.ID == 1, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID)) + user, err = ucache.Get(newUserID) + recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID) + expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID)) + ucache.Flush() + } + + user, err = c.Users.Get(newUserID) + recordMustExist(t, err, "Couldn't find UID #%d", newUserID) + expectUser(user, newUserID, "Sam", 5, false, false, false, false) + + if ucache != nil { + expectIntToBeX(t, ucache.Length(), 1, "User cache length should be 1, not %d") + user, err = ucache.Get(newUserID) + recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID) + expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID)) + } + err = user.Activate() expectNilErr(t, err) expectIntToBeX(t, user.Group, 5, "Sam should still be in group 5 in this copy") diff --git a/routes/panel/debug.go b/routes/panel/debug.go index 42b8a392..c9946513 100644 --- a/routes/panel/debug.go +++ b/routes/panel/debug.go @@ -113,8 +113,9 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { if fErr != nil { return c.InternalError(fErr,w,r) } + gitSize, _ := c.DirSize("./.git") - debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize} + debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize,gitSize} pi := c.PanelDebugPage{basePage, goVersion, dbVersion, uptime, openConnCount, qgen.Builder.GetAdapter().GetName(), goroutines, cpus, memStats, debugCache, debugDatabase, debugDisk} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_dashboard_right", "debug_page", "panel_debug", pi}) diff --git a/templates/panel_debug.html b/templates/panel_debug.html index c73fba1f..1882f806 100644 --- a/templates/panel_debug.html +++ b/templates/panel_debug.html @@ -32,7 +32,7 @@

Memory Statistics

-
+{{flush}}
Sys
HeapSys
@@ -92,7 +92,7 @@

Database

-
+{{flush}}
Topics
Users
@@ -167,7 +167,7 @@

Disk

-
+{{flush}}
Static Files
Attachments
@@ -180,9 +180,9 @@
Log Files
Backups
-
???
+
Git
{{bunit .Disk.Logs}}
{{bunit .Disk.Backups}}
-
?
+
{{bunit .Disk.Git}}
\ No newline at end of file