From 5db5bc0c7e069778b4bcee43785a45740610d89c Mon Sep 17 00:00:00 2001 From: Azareal Date: Mon, 31 Dec 2018 19:03:49 +1000 Subject: [PATCH] Reply attachments can be managed too now. Added eight database indices. Fixed a bug where the second tick wouldn't fire. Tweaked the .topic_forum in Nox by a pixel. Replaced some _installer strings with blank strings for consistency with the builders. Greatly reduced the number of allocations in the user agent parser. Added ampersand entities in more attachment URLs to avoid accidental mangling. .edit_source is now hidden for guests. Guest noavatars are now pre-calculated to reduce the number of allocations. Lazily initialised a couple of maps in ViewTopic to reduce the number of unnecessary allocations slightly. Added the unsafe BytesToString function. Please don't use this, if you don't have to. Added the AddIndex method to the adapter and associated components. Added the /reply/attach/add/submit/ route. Added the /reply/attach/remove/submit/ route. Added the topic_alt_userinfo template. Replaced Attachments.MiniTopicGet with MiniGetList. Added Attachments.BulkMiniGetList. Added a quick test for ReplyStore.Create. Added BenchmarkPopulateTopicWithRouter. Added BenchmarkTopicAdminFullPageRouteParallelWithRouter. Added BenchmarkTopicGuestFullPageRouteParallelWithRouter. You will need to run the updater or patcher for this commit. --- cmd/query_gen/main.go | 9 ++ common/attachments.go | 51 ++++++- common/counters/systems.go | 9 +- common/reply.go | 3 + common/site.go | 1 + common/template_init.go | 4 +- common/templates/templates.go | 4 + common/user.go | 10 ++ gen_router.go | 262 ++++++++++++++++++--------------- general_test.go | 78 ++++++++-- main.go | 10 +- misc_test.go | 4 + patcher/patches.go | 37 +++++ public/global.js | 18 ++- query_gen/builder.go | 4 + query_gen/install.go | 22 ++- query_gen/mssql.go | 17 ++- query_gen/mysql.go | 18 +++ query_gen/pgsql.go | 15 ++ query_gen/querygen.go | 1 + router_gen/main.go | 171 ++++++++++++--------- router_gen/routes.go | 2 + routes/profile.go | 2 +- routes/reply.go | 102 +++++++++++++ routes/topic.go | 31 +++- schema/mysql/inserts.sql | 8 + templates/topic_alt.html | 13 +- templates/topic_alt_posts.html | 22 ++- templates/topic_posts.html | 1 + themes/nox/public/main.css | 2 +- tickloop.go | 8 +- tmpl_stub.go | 9 ++ 32 files changed, 697 insertions(+), 251 deletions(-) diff --git a/cmd/query_gen/main.go b/cmd/query_gen/main.go index 9725dd0f..6c45311a 100644 --- a/cmd/query_gen/main.go +++ b/cmd/query_gen/main.go @@ -110,6 +110,15 @@ func writeStatements(adapter qgen.Adapter) error { } func seedTables(adapter qgen.Adapter) error { + qgen.Install.AddIndex("topics", "parentID", "parentID") + qgen.Install.AddIndex("replies", "tid", "tid") + qgen.Install.AddIndex("polls", "parentID", "parentID") + qgen.Install.AddIndex("likes", "targetItem", "targetItem") + qgen.Install.AddIndex("emails", "uid", "uid") + qgen.Install.AddIndex("attachments", "originID", "originID") + qgen.Install.AddIndex("attachments", "path", "path") + qgen.Install.AddIndex("activity_stream_matches", "watcher", "watcher") + qgen.Install.SimpleInsert("sync", "last_update", "UTC_TIMESTAMP()") qgen.Install.SimpleInsert("settings", "name, content, type, constraints", "'activation_type','1','list','1-3'") qgen.Install.SimpleInsert("settings", "name, content, type", "'bigpost_min_words','250','int'") diff --git a/common/attachments.go b/common/attachments.go index 1c9a63da..a8ad1f51 100644 --- a/common/attachments.go +++ b/common/attachments.go @@ -23,7 +23,8 @@ type MiniAttachment struct { type AttachmentStore interface { Get(id int) (*MiniAttachment, error) - MiniTopicGet(id int) (alist []*MiniAttachment, err error) + MiniGetList(originTable string, originID int) (alist []*MiniAttachment, err error) + BulkMiniGetList(originTable string, ids []int) (amap map[int][]*MiniAttachment, err error) Add(sectionID int, sectionTable string, originID int, originTable string, uploadedBy int, path string) (int, error) GlobalCount() int CountIn(originTable string, oid int) int @@ -33,7 +34,7 @@ type AttachmentStore interface { type DefaultAttachmentStore struct { get *sql.Stmt - getByTopic *sql.Stmt + getByObj *sql.Stmt add *sql.Stmt count *sql.Stmt countIn *sql.Stmt @@ -45,7 +46,7 @@ func NewDefaultAttachmentStore() (*DefaultAttachmentStore, error) { acc := qgen.NewAcc() return &DefaultAttachmentStore{ get: acc.Select("attachments").Columns("originID, sectionID, uploadedBy, path").Where("attachID = ?").Prepare(), - getByTopic: acc.Select("attachments").Columns("attachID, sectionID, uploadedBy, path").Where("originTable = 'topics' AND originID = ?").Prepare(), + getByObj: acc.Select("attachments").Columns("attachID, sectionID, uploadedBy, path").Where("originTable = ? AND originID = ?").Prepare(), add: acc.Insert("attachments").Columns("sectionID, sectionTable, originID, originTable, uploadedBy, path").Fields("?,?,?,?,?,?").Prepare(), count: acc.Count("attachments").Prepare(), countIn: acc.Count("attachments").Where("originTable = ? and originID = ?").Prepare(), @@ -54,12 +55,11 @@ func NewDefaultAttachmentStore() (*DefaultAttachmentStore, error) { }, acc.FirstError() } -// TODO: Make this more generic so we can use it for reply attachments too -func (store *DefaultAttachmentStore) MiniTopicGet(id int) (alist []*MiniAttachment, err error) { - rows, err := store.getByTopic.Query(id) +func (store *DefaultAttachmentStore) MiniGetList(originTable string, originID int) (alist []*MiniAttachment, err error) { + rows, err := store.getByObj.Query(originTable, originID) defer rows.Close() for rows.Next() { - attach := &MiniAttachment{OriginID: id} + attach := &MiniAttachment{OriginID: originID} err := rows.Scan(&attach.ID, &attach.SectionID, &attach.UploadedBy, &attach.Path) if err != nil { return nil, err @@ -75,6 +75,43 @@ func (store *DefaultAttachmentStore) MiniTopicGet(id int) (alist []*MiniAttachme return alist, rows.Err() } +func (store *DefaultAttachmentStore) BulkMiniGetList(originTable string, ids []int) (amap map[int][]*MiniAttachment, err error) { + if len(ids) == 0 { + return nil, sql.ErrNoRows + } + if len(ids) == 1 { + res, err := store.MiniGetList(originTable, ids[0]) + return map[int][]*MiniAttachment{0: res}, err + } + + amap = make(map[int][]*MiniAttachment) + var buffer []*MiniAttachment + var currentID int + rows, err := qgen.NewAcc().Select("attachments").Columns("attachID, sectionID, originID, uploadedBy, path").Where("originTable = ?").In("originID", ids).Orderby("originID ASC").Query(originTable) + defer rows.Close() + for rows.Next() { + attach := &MiniAttachment{} + err := rows.Scan(&attach.ID, &attach.SectionID, &attach.OriginID, &attach.UploadedBy, &attach.Path) + if err != nil { + return nil, err + } + extarr := strings.Split(attach.Path, ".") + if len(extarr) < 2 { + return nil, errors.New("corrupt attachment path") + } + attach.Ext = extarr[len(extarr)-1] + attach.Image = ImageFileExts.Contains(attach.Ext) + if attach.ID != currentID { + if len(buffer) > 0 { + amap[currentID] = buffer + buffer = nil + } + } + buffer = append(buffer, attach) + } + return amap, rows.Err() +} + func (store *DefaultAttachmentStore) Get(id int) (*MiniAttachment, error) { attach := &MiniAttachment{ID: id} err := store.get.QueryRow(id).Scan(&attach.OriginID, &attach.SectionID, &attach.UploadedBy, &attach.Path) diff --git a/common/counters/systems.go b/common/counters/systems.go index 8138be05..3f24367a 100644 --- a/common/counters/systems.go +++ b/common/counters/systems.go @@ -1,8 +1,11 @@ package counters -import "database/sql" -import "github.com/Azareal/Gosora/common" -import "github.com/Azareal/Gosora/query_gen" +import ( + "database/sql" + + "github.com/Azareal/Gosora/common" + "github.com/Azareal/Gosora/query_gen" +) var OSViewCounter *DefaultOSViewCounter diff --git a/common/reply.go b/common/reply.go index f1c294a0..fdf3558c 100644 --- a/common/reply.go +++ b/common/reply.go @@ -39,8 +39,11 @@ type ReplyUser struct { IPAddress string Liked bool LikeCount int + AttachCount int ActionType string ActionIcon string + + Attachments []*MiniAttachment } type Reply struct { diff --git a/common/site.go b/common/site.go index 5fdc7489..c818a832 100644 --- a/common/site.go +++ b/common/site.go @@ -129,6 +129,7 @@ func LoadConfig() error { func ProcessConfig() (err error) { Config.Noavatar = strings.Replace(Config.Noavatar, "{site_url}", Site.URL, -1) + guestAvatar = GuestAvatar{buildNoavatar(0, 200), buildNoavatar(0, 48)} Site.Host = Site.URL if Site.Port != "80" && Site.Port != "443" { Site.URL = strings.TrimSuffix(Site.URL, "/") diff --git a/common/template_init.go b/common/template_init.go index 33893171..2786c060 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -231,7 +231,7 @@ func CompileTemplates() error { var replyList []ReplyUser // TODO: Do we want the UID on this to be 0? avatar, microAvatar = BuildAvatar(0, "") - replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) + replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach}) var varList = make(map[string]tmpl.VarItem) var compile = func(name string, expects string, expectsInt interface{}) (tmpl string, err error) { @@ -456,7 +456,7 @@ func CompileJSTemplates() error { var replyList []ReplyUser // TODO: Do we really want the UID here to be zero? avatar, microAvatar = BuildAvatar(0, "") - replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) + replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach}) varList = make(map[string]tmpl.VarItem) header.Title = "Topic Name" diff --git a/common/templates/templates.go b/common/templates/templates.go index f856daaa..d8801008 100644 --- a/common/templates/templates.go +++ b/common/templates/templates.go @@ -1120,6 +1120,10 @@ func (c *CTemplateSet) compileIfVarSub(con CContext, varname string) (out string cur = cur.FieldByName(bit) out += "." + bit + if !cur.IsValid() { + fmt.Println("cur: ", cur) + panic(out + "^\n" + "Invalid value. Maybe, it doesn't exist?") + } stepInterface() if !cur.IsValid() { fmt.Println("cur: ", cur) diff --git a/common/user.go b/common/user.go index 5948a288..6358f428 100644 --- a/common/user.go +++ b/common/user.go @@ -446,6 +446,13 @@ func (user *User) InitPerms() { } } +var guestAvatar GuestAvatar + +type GuestAvatar struct { + Normal string + Micro string +} + func buildNoavatar(uid int, width int) string { return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1) } @@ -464,6 +471,9 @@ func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar strin } return avatar, avatar } + if uid == 0 { + return guestAvatar.Normal, guestAvatar.Micro + } return buildNoavatar(uid, 200), buildNoavatar(uid, 48) } diff --git a/gen_router.go b/gen_router.go index 9555b05b..354e77ab 100644 --- a/gen_router.go +++ b/gen_router.go @@ -5,6 +5,7 @@ package main import ( "log" "strings" + "bytes" "strconv" "compress/gzip" "sync" @@ -135,6 +136,8 @@ var RouteMap = map[string]interface{}{ "routes.ReplyEditSubmit": routes.ReplyEditSubmit, "routes.ReplyDeleteSubmit": routes.ReplyDeleteSubmit, "routes.ReplyLikeSubmit": routes.ReplyLikeSubmit, + "routes.AddAttachToReplySubmit": routes.AddAttachToReplySubmit, + "routes.RemoveAttachFromReplySubmit": routes.RemoveAttachFromReplySubmit, "routes.ProfileReplyCreateSubmit": routes.ProfileReplyCreateSubmit, "routes.ProfileReplyEditSubmit": routes.ProfileReplyEditSubmit, "routes.ProfileReplyDeleteSubmit": routes.ProfileReplyDeleteSubmit, @@ -270,24 +273,26 @@ var routeMapEnum = map[string]int{ "routes.ReplyEditSubmit": 110, "routes.ReplyDeleteSubmit": 111, "routes.ReplyLikeSubmit": 112, - "routes.ProfileReplyCreateSubmit": 113, - "routes.ProfileReplyEditSubmit": 114, - "routes.ProfileReplyDeleteSubmit": 115, - "routes.PollVote": 116, - "routes.PollResults": 117, - "routes.AccountLogin": 118, - "routes.AccountRegister": 119, - "routes.AccountLogout": 120, - "routes.AccountLoginSubmit": 121, - "routes.AccountLoginMFAVerify": 122, - "routes.AccountLoginMFAVerifySubmit": 123, - "routes.AccountRegisterSubmit": 124, - "routes.DynamicRoute": 125, - "routes.UploadedFile": 126, - "routes.StaticFile": 127, - "routes.RobotsTxt": 128, - "routes.SitemapXml": 129, - "routes.BadRoute": 130, + "routes.AddAttachToReplySubmit": 113, + "routes.RemoveAttachFromReplySubmit": 114, + "routes.ProfileReplyCreateSubmit": 115, + "routes.ProfileReplyEditSubmit": 116, + "routes.ProfileReplyDeleteSubmit": 117, + "routes.PollVote": 118, + "routes.PollResults": 119, + "routes.AccountLogin": 120, + "routes.AccountRegister": 121, + "routes.AccountLogout": 122, + "routes.AccountLoginSubmit": 123, + "routes.AccountLoginMFAVerify": 124, + "routes.AccountLoginMFAVerifySubmit": 125, + "routes.AccountRegisterSubmit": 126, + "routes.DynamicRoute": 127, + "routes.UploadedFile": 128, + "routes.StaticFile": 129, + "routes.RobotsTxt": 130, + "routes.SitemapXml": 131, + "routes.BadRoute": 132, } var reverseRouteMapEnum = map[int]string{ 0: "routes.Overview", @@ -403,24 +408,26 @@ var reverseRouteMapEnum = map[int]string{ 110: "routes.ReplyEditSubmit", 111: "routes.ReplyDeleteSubmit", 112: "routes.ReplyLikeSubmit", - 113: "routes.ProfileReplyCreateSubmit", - 114: "routes.ProfileReplyEditSubmit", - 115: "routes.ProfileReplyDeleteSubmit", - 116: "routes.PollVote", - 117: "routes.PollResults", - 118: "routes.AccountLogin", - 119: "routes.AccountRegister", - 120: "routes.AccountLogout", - 121: "routes.AccountLoginSubmit", - 122: "routes.AccountLoginMFAVerify", - 123: "routes.AccountLoginMFAVerifySubmit", - 124: "routes.AccountRegisterSubmit", - 125: "routes.DynamicRoute", - 126: "routes.UploadedFile", - 127: "routes.StaticFile", - 128: "routes.RobotsTxt", - 129: "routes.SitemapXml", - 130: "routes.BadRoute", + 113: "routes.AddAttachToReplySubmit", + 114: "routes.RemoveAttachFromReplySubmit", + 115: "routes.ProfileReplyCreateSubmit", + 116: "routes.ProfileReplyEditSubmit", + 117: "routes.ProfileReplyDeleteSubmit", + 118: "routes.PollVote", + 119: "routes.PollResults", + 120: "routes.AccountLogin", + 121: "routes.AccountRegister", + 122: "routes.AccountLogout", + 123: "routes.AccountLoginSubmit", + 124: "routes.AccountLoginMFAVerify", + 125: "routes.AccountLoginMFAVerifySubmit", + 126: "routes.AccountRegisterSubmit", + 127: "routes.DynamicRoute", + 128: "routes.UploadedFile", + 129: "routes.StaticFile", + 130: "routes.RobotsTxt", + 131: "routes.SitemapXml", + 132: "routes.BadRoute", } var osMapEnum = map[string]int{ "unknown": 0, @@ -500,33 +507,31 @@ var reverseAgentMapEnum = map[int]string{ 27: "suspicious", 28: "zgrab", } -var markToAgent = map[string]string{ - "OPR":"opera", - "Chrome":"chrome", - "Firefox":"firefox", - "MSIE":"internetexplorer", - "Trident":"trident", // Hack to support IE11 - "Edge":"edge", - "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this - "SamsungBrowser":"samsung", - "UCBrowser":"ucbrowser", - - "Google":"googlebot", - "Googlebot":"googlebot", - "yandex": "yandex", // from the URL - "DuckDuckBot":"duckduckgo", - "Baiduspider":"baidu", - "bingbot":"bing", - "BingPreview":"bing", - "SeznamBot":"seznambot", - "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? - "Uptimebot":"uptimebot", - "Slackbot":"slackbot", - "Discordbot":"discord", - "Twitterbot":"twitter", - "Discourse":"discourse", - - "zgrab":"zgrab", +var markToAgent = map[string]string{ + "OPR": "opera", + "Chrome": "chrome", + "Firefox": "firefox", + "MSIE": "internetexplorer", + "Trident": "trident", + "Edge": "edge", + "Lynx": "lynx", + "SamsungBrowser": "samsung", + "UCBrowser": "ucbrowser", + "Google": "googlebot", + "Googlebot": "googlebot", + "yandex": "yandex", + "DuckDuckBot": "duckduckgo", + "Baiduspider": "baidu", + "bingbot": "bing", + "BingPreview": "bing", + "SeznamBot": "seznambot", + "CloudFlare": "cloudflare", + "Uptimebot": "uptimebot", + "Slackbot": "slackbot", + "Discordbot": "discord", + "Twitterbot": "twitter", + "Discourse": "discourse", + "zgrab": "zgrab", } /*var agentRank = map[string]int{ "opera":9, @@ -711,7 +716,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { counters.GlobalViewCounter.Bump() if prefix == "/static" { - counters.RouteViewCounter.Bump(127) + counters.RouteViewCounter.Bump(129) req.URL.Path += extraData routes.StaticFile(w, req) return @@ -737,42 +742,49 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } r.DumpRequest(req,"Blank UA: " + prepend) } - } else { - var runeEquals = func(a []rune, b []rune) bool { - if len(a) != len(b) { - return false - } - for i, item := range a { - if item != b[i] { - return false - } - } - return true - } - + } else { // WIP UA Parser - var indices []int var items []string - var buffer []rune - for index, item := range ua { + var buffer []byte + var os string + for _, item := range StringToBytes(ua) { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && (runeEquals(buffer,[]rune("http")) || runeEquals(buffer,[]rune("rv")))) || item == ',' || item == '/' { + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) + if len(buffer) > 2 { + // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append + switch(BytesToString(buffer)) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + case "like": + // Skip this word + default: + items = append(items, string(buffer)) + } + } buffer = buffer[:0] } } else { // TODO: Test this items = items[:0] - indices = indices[:0] r.SuspiciousRequest(req,"Illegal char in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break } } + if os == "" { + os = "unknown" + } // Iterate over this in reverse as the real UA tends to be on the right side var agent string @@ -789,24 +801,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.requestLogger.Print("parsed agent: ", agent) } - var os string - for _, mark := range items { - switch(mark) { - case "Windows": - os = "windows" - case "Linux": - os = "linux" - case "Mac": - os = "mac" - case "iPhone": - os = "iphone" - case "Android": - os = "android" - } - } - if os == "" { - os = "unknown" - } if common.Dev.SuperDebug { r.requestLogger.Print("os: ", os) r.requestLogger.Printf("items: %+v\n",items) @@ -1884,6 +1878,36 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c counters.RouteViewCounter.Bump(112) err = routes.ReplyLikeSubmit(w,req,user,extraData) + case "/reply/attach/add/submit/": + err = common.MemberOnly(w,req,user) + if err != nil { + return err + } + + err = common.HandleUploadRoute(w,req,user,int(common.Config.MaxRequestSize)) + if err != nil { + return err + } + err = common.NoUploadSessionMismatch(w,req,user) + if err != nil { + return err + } + + counters.RouteViewCounter.Bump(113) + err = routes.AddAttachToReplySubmit(w,req,user,extraData) + case "/reply/attach/remove/submit/": + err = common.NoSessionMismatch(w,req,user) + if err != nil { + return err + } + + err = common.MemberOnly(w,req,user) + if err != nil { + return err + } + + counters.RouteViewCounter.Bump(114) + err = routes.RemoveAttachFromReplySubmit(w,req,user,extraData) } case "/profile": switch(req.URL.Path) { @@ -1898,7 +1922,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(113) + counters.RouteViewCounter.Bump(115) err = routes.ProfileReplyCreateSubmit(w,req,user) case "/profile/reply/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1911,7 +1935,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(114) + counters.RouteViewCounter.Bump(116) err = routes.ProfileReplyEditSubmit(w,req,user,extraData) case "/profile/reply/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1924,7 +1948,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(115) + counters.RouteViewCounter.Bump(117) err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData) } case "/poll": @@ -1940,23 +1964,23 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(116) + counters.RouteViewCounter.Bump(118) err = routes.PollVote(w,req,user,extraData) case "/poll/results/": - counters.RouteViewCounter.Bump(117) + counters.RouteViewCounter.Bump(119) err = routes.PollResults(w,req,user,extraData) } case "/accounts": switch(req.URL.Path) { case "/accounts/login/": - counters.RouteViewCounter.Bump(118) + counters.RouteViewCounter.Bump(120) head, err := common.UserCheck(w,req,&user) if err != nil { return err } err = routes.AccountLogin(w,req,user,head) case "/accounts/create/": - counters.RouteViewCounter.Bump(119) + counters.RouteViewCounter.Bump(121) head, err := common.UserCheck(w,req,&user) if err != nil { return err @@ -1973,7 +1997,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(120) + counters.RouteViewCounter.Bump(122) err = routes.AccountLogout(w,req,user) case "/accounts/login/submit/": err = common.ParseForm(w,req,user) @@ -1981,10 +2005,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(121) + counters.RouteViewCounter.Bump(123) err = routes.AccountLoginSubmit(w,req,user) case "/accounts/mfa_verify/": - counters.RouteViewCounter.Bump(122) + counters.RouteViewCounter.Bump(124) head, err := common.UserCheck(w,req,&user) if err != nil { return err @@ -1996,7 +2020,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(123) + counters.RouteViewCounter.Bump(125) err = routes.AccountLoginMFAVerifySubmit(w,req,user) case "/accounts/create/submit/": err = common.ParseForm(w,req,user) @@ -2004,7 +2028,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return err } - counters.RouteViewCounter.Bump(124) + counters.RouteViewCounter.Bump(126) err = routes.AccountRegisterSubmit(w,req,user) } /*case "/sitemaps": // TODO: Count these views @@ -2020,7 +2044,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c w.Header().Del("Content-Type") w.Header().Del("Content-Encoding") } - counters.RouteViewCounter.Bump(126) + counters.RouteViewCounter.Bump(128) req.URL.Path += extraData // TODO: Find a way to propagate errors up from this? r.UploadHandler(w,req) // TODO: Count these views @@ -2030,10 +2054,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c // TODO: Add support for favicons and robots.txt files switch(extraData) { case "robots.txt": - counters.RouteViewCounter.Bump(128) + counters.RouteViewCounter.Bump(130) return routes.RobotsTxt(w,req) /*case "sitemap.xml": - counters.RouteViewCounter.Bump(129) + counters.RouteViewCounter.Bump(131) return routes.SitemapXml(w,req)*/ } return common.NotFound(w,req,nil) @@ -2044,7 +2068,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c r.RUnlock() if ok { - counters.RouteViewCounter.Bump(125) // TODO: Be more specific about *which* dynamic route it is + counters.RouteViewCounter.Bump(127) // TODO: Be more specific about *which* dynamic route it is req.URL.Path += extraData return handle(w,req,user) } @@ -2055,7 +2079,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c } else { r.DumpRequest(req,"Bad Route") } - counters.RouteViewCounter.Bump(130) + counters.RouteViewCounter.Bump(132) return common.NotFound(w,req,nil) } return err diff --git a/general_test.go b/general_test.go index f45907f1..f32a252d 100644 --- a/general_test.go +++ b/general_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" "time" + "runtime/debug" "github.com/pkg/errors" @@ -109,6 +110,7 @@ func init() { } } +const benchTidI = 1 const benchTid = "1" // TODO: Swap out LocalError for a panic for this? @@ -175,7 +177,7 @@ func BenchmarkTopicAdminRouteParallelWithRouter(b *testing.B) { } uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: common.Year} sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: common.Year} - path := "/topic/hm."+benchTid + path := "/topic/hm." + benchTid b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -229,8 +231,8 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) { } uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: common.Year} sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: common.Year} - path := "/topic/hm."+benchTid - + path := "/topic/hm." + benchTid + b.RunParallel(func(pb *testing.PB) { for pb.Next() { w := httptest.NewRecorder() @@ -246,18 +248,18 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) { b.Fatal("HTTP Error!") } -{ - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", path, bytes.NewReader(nil)) - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36") - req.Header.Set("Host", "localhost") - req.Host = "localhost" - router.ServeHTTP(w, req) - if w.Code != 200 { - b.Log(w.Body) - b.Fatal("HTTP Error!") + { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", path, bytes.NewReader(nil)) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36") + req.Header.Set("Host", "localhost") + req.Host = "localhost" + router.ServeHTTP(w, req) + if w.Code != 200 { + b.Log(w.Body) + b.Fatal("HTTP Error!") + } } -} } }) @@ -428,6 +430,54 @@ func BenchmarkProfileGuestRouteParallelWithRouter(b *testing.B) { obRoute(b, "/profile/admin.1") } +func BenchmarkPopulateTopicWithRouter(b *testing.B) { + b.ReportAllocs() + topic, err := common.Topics.Get(benchTidI) + if err != nil { + debug.PrintStack() + b.Fatal(err) + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for i := 0; i < 25; i++ { + _, err := common.Rstore.Create(topic, "hiii", "::1", 1) + if err != nil { + debug.PrintStack() + b.Fatal(err) + } + } + } + }) +} + +//var fullPage = false + +func BenchmarkTopicAdminFullPageRouteParallelWithRouter(b *testing.B) { + /*if !fullPage { + topic, err := common.Topics.Get(benchTidI) + panicIfErr(err) + for i := 0; i < 25; i++ { + _, err = common.Rstore.Create(topic, "hiii", "::1", 1) + panicIfErr(err) + } + fullPage = true + }*/ + BenchmarkTopicAdminRouteParallel(b) +} + +func BenchmarkTopicGuestFullPageRouteParallelWithRouter(b *testing.B) { + /*if !fullPage { + topic, err := common.Topics.Get(benchTidI) + panicIfErr(err) + for i := 0; i < 25; i++ { + _, err = common.Rstore.Create(topic, "hiii", "::1", 1) + panicIfErr(err) + } + fullPage = true + }*/ + obRoute(b, "/topic/hm."+benchTid) +} + // TODO: Make these routes compatible with the changes to the router /* func BenchmarkForumsAdminRouteParallel(b *testing.B) { diff --git a/main.go b/main.go index 2e43cf94..23ed1768 100644 --- a/main.go +++ b/main.go @@ -371,14 +371,7 @@ func main() { // TODO: Could we expand this to attachments and other things too? thumbChan := make(chan bool) go common.ThumbTask(thumbChan) - - // TODO: Write tests for these - // Run this goroutine once every half second - halfSecondTicker := time.NewTicker(time.Second / 2) - secondTicker := time.NewTicker(time.Second) - fifteenMinuteTicker := time.NewTicker(15 * time.Minute) - hourTicker := time.NewTicker(time.Hour) - go tickLoop(thumbChan, halfSecondTicker, secondTicker, fifteenMinuteTicker, hourTicker) + go tickLoop(thumbChan) // Resource Management Goroutine go func() { @@ -390,6 +383,7 @@ func main() { var lastEvictedCount int var couldNotDealloc bool + var secondTicker = time.NewTicker(time.Second) for { select { case <-secondTicker.C: diff --git a/misc_test.go b/misc_test.go index 8b180a15..f9be1e50 100644 --- a/misc_test.go +++ b/misc_test.go @@ -749,6 +749,10 @@ func TestReplyStore(t *testing.T) { topic, err = common.Topics.Get(1) expectNilErr(t, err) expect(t, topic.PostCount == 3, fmt.Sprintf("TID #1's post count should be three, not %d", topic.PostCount)) + + rid, err = common.Rstore.Create(topic, "hiii", "::1", 1) + expectNilErr(t, err) + replyTest(rid, topic.ID, 1, "hiii", "::1") } func TestProfileReplyStore(t *testing.T) { diff --git a/patcher/patches.go b/patcher/patches.go index b3a81f44..9f82e3ba 100644 --- a/patcher/patches.go +++ b/patcher/patches.go @@ -23,6 +23,7 @@ func init() { addPatch(9, patch9) addPatch(10, patch10) addPatch(11, patch11) + addPatch(12, patch12) } func patch0(scanner *bufio.Scanner) (err error) { @@ -467,3 +468,39 @@ func patch11(scanner *bufio.Scanner) error { return err })*/ } + +func patch12(scanner *bufio.Scanner) error { + err := execStmt(qgen.Builder.AddIndex("topics", "parentID", "parentID")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("replies", "tid", "tid")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("polls", "parentID", "parentID")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("likes", "targetItem", "targetItem")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("emails", "uid", "uid")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("attachments", "originID", "originID")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("attachments", "path", "path")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddIndex("activity_stream_matches", "watcher", "watcher")) + if err != nil { + return err + } + return nil +} diff --git a/public/global.js b/public/global.js index 4fe797f0..c202243b 100644 --- a/public/global.js +++ b/public/global.js @@ -427,6 +427,8 @@ function mainInit(){ $(".edit_item").click(function(event){ event.preventDefault(); let blockParent = this.closest('.editable_parent'); + $(blockParent).find('.hide_on_edit').addClass("edit_opened"); + $(blockParent).find('.show_on_edit').addClass("edit_opened"); let srcNode = blockParent.querySelector(".edit_source"); let block = blockParent.querySelector('.editable_block'); block.classList.add("in_edit"); @@ -438,6 +440,8 @@ function mainInit(){ $(".submit_edit").click(function(event){ event.preventDefault(); + $(blockParent).find('.hide_on_edit').removeClass("edit_opened"); + $(blockParent).find('.show_on_edit').removeClass("edit_opened"); block.classList.remove("in_edit"); let newContent = block.querySelector('textarea').value; block.innerHTML = quickParse(newContent); @@ -668,7 +672,7 @@ function mainInit(){ $(".attach_item_copy").unbind("click"); bindAttachItems() }); - req.open("POST","//"+window.location.host+"/topic/attach/add/submit/"+fileDock.getAttribute("tid")); + req.open("POST","//"+window.location.host+"/"+fileDock.getAttribute("type")+"/attach/add/submit/"+fileDock.getAttribute("id")); req.send(formData); }); } catch(e) { @@ -714,14 +718,20 @@ function mainInit(){ } } - var uploadFiles = document.getElementById("upload_files"); + let uploadFiles = document.getElementById("upload_files"); if(uploadFiles != null) { uploadFiles.addEventListener("change", uploadAttachHandler, false); } - var uploadFilesOp = document.getElementById("upload_files_op"); + let uploadFilesOp = document.getElementById("upload_files_op"); if(uploadFilesOp != null) { uploadFilesOp.addEventListener("change", uploadAttachHandler2, false); } + let uploadFilesPost = document.getElementsByClassName("upload_files_post"); + if(uploadFilesPost != null) { + for(let i = 0; i < uploadFilesPost.length; i++) { + uploadFilesPost[i].addEventListener("change", uploadAttachHandler2, false); + } + } function copyToClipboard(str) { const el = document.createElement('textarea'); @@ -772,7 +782,7 @@ function mainInit(){ let req = new XMLHttpRequest(); let fileDock = this.closest(".attach_edit_bay"); - req.open("POST","//"+window.location.host+"/topic/attach/remove/submit/"+fileDock.getAttribute("tid"),true); + req.open("POST","//"+window.location.host+"/"+fileDock.getAttribute("type")+"/attach/remove/submit/"+fileDock.getAttribute("id"),true); req.send(formData); }); diff --git a/query_gen/builder.go b/query_gen/builder.go index 02ab85d1..4e298b53 100644 --- a/query_gen/builder.go +++ b/query_gen/builder.go @@ -112,6 +112,10 @@ func (build *builder) AddColumn(table string, column DBTableColumn) (stmt *sql.S return build.prepare(build.adapter.AddColumn("", table, column)) } +func (build *builder) AddIndex(table string, iname string, colname string) (stmt *sql.Stmt, err error) { + return build.prepare(build.adapter.AddIndex("", table, iname, colname)) +} + func (build *builder) SimpleInsert(table string, columns string, fields string) (stmt *sql.Stmt, err error) { return build.prepare(build.adapter.SimpleInsert("", table, columns, fields)) } diff --git a/query_gen/install.go b/query_gen/install.go index f7219cd1..797fa863 100644 --- a/query_gen/install.go +++ b/query_gen/install.go @@ -54,7 +54,7 @@ func (install *installer) CreateTable(table string, charset string, collation st if err != nil { return err } - res, err := install.adapter.CreateTable("_installer", table, charset, collation, columns, keys) + res, err := install.adapter.CreateTable("", table, charset, collation, columns, keys) if err != nil { return err } @@ -67,13 +67,31 @@ func (install *installer) CreateTable(table string, charset string, collation st return nil } +// TODO: Let plugins manipulate the parameters like in CreateTable +func (install *installer) AddIndex(table string, iname string, colname string) error { + err := install.RunHook("AddIndexStart", table, iname, colname) + if err != nil { + return err + } + res, err := install.adapter.AddIndex("", table, iname, colname) + if err != nil { + return err + } + err = install.RunHook("AddIndexAfter", table, iname, colname) + if err != nil { + return err + } + install.instructions = append(install.instructions, DB_Install_Instruction{table, res, "index"}) + return nil +} + // TODO: Let plugins manipulate the parameters like in CreateTable func (install *installer) SimpleInsert(table string, columns string, fields string) error { err := install.RunHook("SimpleInsertStart", table, columns, fields) if err != nil { return err } - res, err := install.adapter.SimpleInsert("_installer", table, columns, fields) + res, err := install.adapter.SimpleInsert("", table, columns, fields) if err != nil { return err } diff --git a/query_gen/mssql.go b/query_gen/mssql.go index f294f642..e336aa8e 100644 --- a/query_gen/mssql.go +++ b/query_gen/mssql.go @@ -146,6 +146,21 @@ func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTable return querystr, nil } +// TODO: Implement this +// TODO: Test to make sure everything works here +func (adapter *MssqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if iname == "" { + return "", errors.New("You need a name for the index") + } + if colname == "" { + return "", errors.New("You need a name for the column") + } + return "", errors.New("not implemented") +} + func (adapter *MssqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") @@ -1134,7 +1149,7 @@ func _gen_mssql() (err error) { // Internal methods, not exposed in the interface func (adapter *MssqlAdapter) pushStatement(name string, stype string, querystr string) { - if name[0] == '_' { + if name == "" { return } adapter.Buffer[name] = DBStmt{querystr, stype} diff --git a/query_gen/mysql.go b/query_gen/mysql.go index d7d78990..03c2b575 100644 --- a/query_gen/mysql.go +++ b/query_gen/mysql.go @@ -185,6 +185,24 @@ func (adapter *MysqlAdapter) AddColumn(name string, table string, column DBTable return querystr, nil } +// TODO: Test to make sure everything works here +func (adapter *MysqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if iname == "" { + return "", errors.New("You need a name for the index") + } + if colname == "" { + return "", errors.New("You need a name for the column") + } + + querystr := "ALTER TABLE `" + table + "` ADD INDEX " + "`" + iname + "` (`" + colname + "`);" + // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator + adapter.pushStatement(name, "add-index", querystr) + return querystr, nil +} + func (adapter *MysqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") diff --git a/query_gen/pgsql.go b/query_gen/pgsql.go index be64bbb2..a6641c1d 100644 --- a/query_gen/pgsql.go +++ b/query_gen/pgsql.go @@ -120,6 +120,21 @@ func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTable return "", nil } +// TODO: Implement this +// TODO: Test to make sure everything works here +func (adapter *PgsqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if iname == "" { + return "", errors.New("You need a name for the index") + } + if colname == "" { + return "", errors.New("You need a name for the column") + } + return "", errors.New("not implemented") +} + // TODO: Test this // ! We need to get the last ID out of this somehow, maybe add returning to every query? Might require some sort of wrapper over the sql statements func (adapter *PgsqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { diff --git a/query_gen/querygen.go b/query_gen/querygen.go index 367278ba..37fdfd67 100644 --- a/query_gen/querygen.go +++ b/query_gen/querygen.go @@ -109,6 +109,7 @@ type Adapter interface { // TODO: Some way to add indices and keys // TODO: Test this AddColumn(name string, table string, column DBTableColumn) (string, error) + AddIndex(name string, table string, iname string, colname string) (string, error) SimpleInsert(name string, table string, columns string, fields string) (string, error) SimpleUpdate(up *updatePrebuilder) (string, error) SimpleUpdateSelect(up *updatePrebuilder) (string, error) // ! Experimental diff --git a/router_gen/main.go b/router_gen/main.go index e728e49b..e744f3e9 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -10,14 +10,16 @@ import ( ) type TmplVars struct { - RouteList []*RouteImpl - RouteGroups []*RouteGroup - AllRouteNames []string - AllRouteMap map[string]int - AllAgentNames []string - AllAgentMap map[string]int - AllOSNames []string - AllOSMap map[string]int + RouteList []*RouteImpl + RouteGroups []*RouteGroup + AllRouteNames []string + AllRouteMap map[string]int + AllAgentNames []string + AllAgentMap map[string]int + AllAgentMarkNames []string + AllAgentMarks map[string]string + AllOSNames []string + AllOSMap map[string]int } func main() { @@ -227,6 +229,64 @@ func main() { tmplVars.AllAgentMap[agent] = id } + tmplVars.AllAgentMarkNames = []string{ + "OPR", + "Chrome", + "Firefox", + "MSIE", + "Trident", + "Edge", + "Lynx", + "SamsungBrowser", + "UCBrowser", + + "Google", + "Googlebot", + "yandex", + "DuckDuckBot", + "Baiduspider", + "bingbot", + "BingPreview", + "SeznamBot", + "CloudFlare", + "Uptimebot", + "Slackbot", + "Discordbot", + "Twitterbot", + "Discourse", + + "zgrab", + } + + tmplVars.AllAgentMarks = map[string]string{ + "OPR": "opera", + "Chrome": "chrome", + "Firefox": "firefox", + "MSIE": "internetexplorer", + "Trident": "trident", // Hack to support IE11 + "Edge": "edge", + "Lynx": "lynx", // There's a rare android variant of lynx which isn't covered by this + "SamsungBrowser": "samsung", + "UCBrowser": "ucbrowser", + + "Google": "googlebot", + "Googlebot": "googlebot", + "yandex": "yandex", // from the URL + "DuckDuckBot": "duckduckgo", + "Baiduspider": "baidu", + "bingbot": "bing", + "BingPreview": "bing", + "SeznamBot": "seznambot", + "CloudFlare": "cloudflare", // Track alwayson specifically in case there are other bots? + "Uptimebot": "uptimebot", + "Slackbot": "slackbot", + "Discordbot": "discord", + "Twitterbot": "twitter", + "Discourse": "discourse", + + "zgrab": "zgrab", + } + var fileData = `// Code generated by Gosora's Router Generator. DO NOT EDIT. /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ package main @@ -234,6 +294,7 @@ package main import ( "log" "strings" + "bytes" "strconv" "compress/gzip" "sync" @@ -273,33 +334,8 @@ var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}} {{$index}}: "{{$element}}",{{end}} } -var markToAgent = map[string]string{ - "OPR":"opera", - "Chrome":"chrome", - "Firefox":"firefox", - "MSIE":"internetexplorer", - "Trident":"trident", // Hack to support IE11 - "Edge":"edge", - "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this - "SamsungBrowser":"samsung", - "UCBrowser":"ucbrowser", - - "Google":"googlebot", - "Googlebot":"googlebot", - "yandex": "yandex", // from the URL - "DuckDuckBot":"duckduckgo", - "Baiduspider":"baidu", - "bingbot":"bing", - "BingPreview":"bing", - "SeznamBot":"seznambot", - "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? - "Uptimebot":"uptimebot", - "Slackbot":"slackbot", - "Discordbot":"discord", - "Twitterbot":"twitter", - "Discourse":"discourse", - - "zgrab":"zgrab", +var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}} + "{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}} } /*var agentRank = map[string]int{ "opera":9, @@ -510,42 +546,49 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } r.DumpRequest(req,"Blank UA: " + prepend) } - } else { - var runeEquals = func(a []rune, b []rune) bool { - if len(a) != len(b) { - return false - } - for i, item := range a { - if item != b[i] { - return false - } - } - return true - } - + } else { // WIP UA Parser - var indices []int var items []string - var buffer []rune - for index, item := range ua { + var buffer []byte + var os string + for _, item := range StringToBytes(ua) { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && (runeEquals(buffer,[]rune("http")) || runeEquals(buffer,[]rune("rv")))) || item == ',' || item == '/' { + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) + if len(buffer) > 2 { + // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append + switch(BytesToString(buffer)) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + case "like": + // Skip this word + default: + items = append(items, string(buffer)) + } + } buffer = buffer[:0] } } else { // TODO: Test this items = items[:0] - indices = indices[:0] r.SuspiciousRequest(req,"Illegal char in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break } } + if os == "" { + os = "unknown" + } // Iterate over this in reverse as the real UA tends to be on the right side var agent string @@ -562,24 +605,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.requestLogger.Print("parsed agent: ", agent) } - var os string - for _, mark := range items { - switch(mark) { - case "Windows": - os = "windows" - case "Linux": - os = "linux" - case "Mac": - os = "mac" - case "iPhone": - os = "iphone" - case "Android": - os = "android" - } - } - if os == "" { - os = "unknown" - } if common.Dev.SuperDebug { r.requestLogger.Print("os: ", os) r.requestLogger.Printf("items: %+v\n",items) diff --git a/router_gen/routes.go b/router_gen/routes.go index 1aeb7d37..30fcdbc3 100644 --- a/router_gen/routes.go +++ b/router_gen/routes.go @@ -104,6 +104,8 @@ func replyRoutes() *RouteGroup { Action("routes.ReplyLikeSubmit", "/reply/like/submit/", "extraData"), //MemberView("routes.ReplyEdit","/reply/edit/","extraData"), // No js fallback //MemberView("routes.ReplyDelete","/reply/delete/","extraData"), // No js confirmation page? We could have a confirmation modal for the JS case + UploadAction("routes.AddAttachToReplySubmit", "/reply/attach/add/submit/", "extraData").MaxSizeVar("int(common.Config.MaxRequestSize)"), + Action("routes.RemoveAttachFromReplySubmit", "/reply/attach/remove/submit/", "extraData"), ) } diff --git a/routes/profile.go b/routes/profile.go index 7073101b..b63eb0fd 100644 --- a/routes/profile.go +++ b/routes/profile.go @@ -100,7 +100,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User, heade replyLikeCount := 0 // TODO: Add a hook here - replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""}) + replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, 0, "", "", nil}) } err = rows.Err() if err != nil { diff --git a/routes/reply.go b/routes/reply.go index 49e60b50..9a2b393a 100644 --- a/routes/reply.go +++ b/routes/reply.go @@ -3,12 +3,14 @@ package routes import ( "database/sql" "encoding/json" + "errors" "net/http" "strconv" "strings" "github.com/Azareal/Gosora/common" "github.com/Azareal/Gosora/common/counters" + "github.com/Azareal/Gosora/common/phrases" "github.com/Azareal/Gosora/query_gen" ) @@ -333,6 +335,106 @@ func ReplyDeleteSubmit(w http.ResponseWriter, r *http.Request, user common.User, return nil } +// TODO: Avoid uploading this again if the attachment already exists? They'll resolve to the same hash either way, but we could save on some IO / bandwidth here +// TODO: Enforce the max request limit on all of this topic's attachments +// TODO: Test this route +func AddAttachToReplySubmit(w http.ResponseWriter, r *http.Request, user common.User, srid string) common.RouteError { + rid, err := strconv.Atoi(srid) + if err != nil { + return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r) + } + + reply, err := common.Rstore.Get(rid) + if err == sql.ErrNoRows { + return common.PreErrorJS("You can't attach to something which doesn't exist!", w, r) + } else if err != nil { + return common.InternalErrorJS(err, w, r) + } + + topic, err := common.Topics.Get(reply.ParentID) + if err != nil { + return common.NotFoundJS(w, r) + } + + _, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID) + if ferr != nil { + return ferr + } + if !user.Perms.ViewTopic || !user.Perms.EditReply || !user.Perms.UploadFiles { + return common.NoPermissionsJS(w, r, user) + } + if topic.IsClosed && !user.Perms.CloseTopic { + return common.NoPermissionsJS(w, r, user) + } + + // Handle the file attachments + pathMap, rerr := uploadAttachment(w, r, user, topic.ParentID, "forums", rid, "replies") + if rerr != nil { + // TODO: This needs to be a JS error... + return rerr + } + if len(pathMap) == 0 { + return common.InternalErrorJS(errors.New("no paths for attachment add"), w, r) + } + + var elemStr string + for path, aids := range pathMap { + elemStr += "\"" + path + "\":\"" + aids + "\"," + } + if len(elemStr) > 1 { + elemStr = elemStr[:len(elemStr)-1] + } + + w.Write([]byte(`{"success":"1","elems":[{` + elemStr + `}]}`)) + return nil +} + +// TODO: Reduce the amount of duplication between this and RemoveAttachFromTopicSubmit +func RemoveAttachFromReplySubmit(w http.ResponseWriter, r *http.Request, user common.User, srid string) common.RouteError { + rid, err := strconv.Atoi(srid) + if err != nil { + return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r) + } + + reply, err := common.Rstore.Get(rid) + if err == sql.ErrNoRows { + return common.PreErrorJS("You can't attach from something which doesn't exist!", w, r) + } else if err != nil { + return common.InternalErrorJS(err, w, r) + } + + topic, err := common.Topics.Get(reply.ParentID) + if err != nil { + return common.NotFoundJS(w, r) + } + + _, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID) + if ferr != nil { + return ferr + } + if !user.Perms.ViewTopic || !user.Perms.EditReply { + return common.NoPermissionsJS(w, r, user) + } + if topic.IsClosed && !user.Perms.CloseTopic { + return common.NoPermissionsJS(w, r, user) + } + + for _, said := range strings.Split(r.PostFormValue("aids"), ",") { + aid, err := strconv.Atoi(said) + if err != nil { + return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r) + } + rerr := deleteAttachment(w, r, user, aid, true) + if rerr != nil { + // TODO: This needs to be a JS error... + return rerr + } + } + + w.Write(successJSONBytes) + return nil +} + // TODO: Move the profile reply routes to their own file? func ProfileReplyCreateSubmit(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { if !user.Perms.ViewTopic || !user.Perms.CreateReply { diff --git a/routes/topic.go b/routes/topic.go index a46dcfd9..6a45a4da 100644 --- a/routes/topic.go +++ b/routes/topic.go @@ -32,7 +32,7 @@ var topicStmts TopicStmts func init() { common.DbInits.Add(func(acc *qgen.Accumulator) error { topicStmts = TopicStmts{ - getReplies: acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"), + getReplies: acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.attachCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"), getLikedTopic: acc.Select("likes").Columns("targetItem").Where("sentBy = ? && targetItem = ? && targetType = 'topics'").Prepare(), // TODO: Less race-y attachment count updates updateAttachs: acc.Update("topics").Set("attachCount = ?").Where("tid = ?").Prepare(), @@ -109,7 +109,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header } if topic.AttachCount > 0 { - attachs, err := common.Attachments.MiniTopicGet(topic.ID) + attachs, err := common.Attachments.MiniGetList("topics", topic.ID) if err != nil { // TODO: We might want to be a little permissive here in-case of a desync? return common.InternalError(err, w, r) @@ -124,9 +124,18 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header // Get the replies if we have any... if topic.PostCount > 0 { - var likedMap = make(map[int]int) + var likedMap map[int]int + if user.Liked > 0 { + likedMap = make(map[int]int) + } var likedQueryList = []int{user.ID} + var attachMap map[int]int + if user.Perms.EditReply { + attachMap = make(map[int]int) + } + var attachQueryList = []int{} + rows, err := topicStmts.getReplies.Query(topic.ID, offset, common.Config.ItemsPerPage) if err == sql.ErrNoRows { return common.LocalError("Bad Page. Some of the posts may have been deleted or you got here by directly typing in the page number.", w, r, user) @@ -138,7 +147,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header // TODO: Factor the user fields out and embed a user struct instead replyItem := common.ReplyUser{ClassName: ""} for rows.Next() { - err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType) + err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.AttachCount, &replyItem.ActionType) if err != nil { return common.InternalError(err, w, r) } @@ -196,6 +205,10 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header likedMap[replyItem.ID] = len(tpage.ItemList) likedQueryList = append(likedQueryList, replyItem.ID) } + if user.Perms.EditReply && replyItem.AttachCount > 0 { + attachMap[replyItem.ID] = len(tpage.ItemList) + attachQueryList = append(attachQueryList, replyItem.ID) + } header.Hooks.VhookNoRet("topic_reply_row_assign", &tpage, &replyItem) // TODO: Use a pointer instead to make it easier to abstract this loop? What impact would this have on escape analysis? @@ -228,6 +241,16 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header return common.InternalError(err, w, r) } } + + if user.Perms.EditReply && len(attachQueryList) > 0 { + amap, err := common.Attachments.BulkMiniGetList("replies", attachQueryList) + if err != nil && err != sql.ErrNoRows { + return common.InternalError(err, w, r) + } + for id, attach := range amap { + tpage.ItemList[attachMap[id]].Attachments = attach + } + } } rerr := renderTemplate("topic", w, r, header, tpage) diff --git a/schema/mysql/inserts.sql b/schema/mysql/inserts.sql index 1436d023..a8d161cb 100644 --- a/schema/mysql/inserts.sql +++ b/schema/mysql/inserts.sql @@ -1,3 +1,11 @@ +ALTER TABLE `topics` ADD INDEX `parentID` (`parentID`);; +ALTER TABLE `replies` ADD INDEX `tid` (`tid`);; +ALTER TABLE `polls` ADD INDEX `parentID` (`parentID`);; +ALTER TABLE `likes` ADD INDEX `targetItem` (`targetItem`);; +ALTER TABLE `emails` ADD INDEX `uid` (`uid`);; +ALTER TABLE `attachments` ADD INDEX `originID` (`originID`);; +ALTER TABLE `attachments` ADD INDEX `path` (`path`);; +ALTER TABLE `activity_stream_matches` ADD INDEX `watcher` (`watcher`);; INSERT INTO `sync`(`last_update`) VALUES (UTC_TIMESTAMP()); INSERT INTO `settings`(`name`,`content`,`type`,`constraints`) VALUES ('activation_type','1','list','1-3'); INSERT INTO `settings`(`name`,`content`,`type`) VALUES ('bigpost_min_words','250','int'); diff --git a/templates/topic_alt.html b/templates/topic_alt.html index 6c3861c3..dfabe5b1 100644 --- a/templates/topic_alt.html +++ b/templates/topic_alt.html @@ -36,14 +36,7 @@ {{if .Poll.ID}}
- {{/** TODO: De-dupe userinfo with a common template **/}} -
-
 
-
- - {{if .Topic.Tag}}
{{.Topic.Tag}}
{{else}}
{{level .Topic.Level}}
{{end}} -
-
+ {{template "topic_alt_userinfo.html" .Topic }}
{{range .Poll.QuickOptions}} @@ -73,10 +66,10 @@
{{.Topic.ContentHTML}}
{{if .CurrentUser.Loggedin}}{{if .CurrentUser.Perms.EditTopic}} - {{if .Topic.Attachments}}
+ {{if .Topic.Attachments}}
{{range .Topic.Attachments}}
- {{if .Image}}{{end}} + {{if .Image}}{{end}} {{.Path}} diff --git a/templates/topic_alt_posts.html b/templates/topic_alt_posts.html index 7198472a..e954cc19 100644 --- a/templates/topic_alt_posts.html +++ b/templates/topic_alt_posts.html @@ -5,8 +5,28 @@ {{.ActionType}} {{else}} -
{{.Content}}
{{.ContentHtml}}
+ {{if $.CurrentUser.Loggedin}}{{if $.CurrentUser.Perms.EditReply}} +
{{.Content}}
+ + {{if .Attachments}}
+ {{range .Attachments}} +
+ {{if .Image}}{{end}} + {{.Path}} + + +
+ {{end}} +
+ {{if $.CurrentUser.Perms.UploadFiles}} + + {{end}} + +
+
{{end}} + {{end}}{{end}} +
{{if $.CurrentUser.Loggedin}} diff --git a/templates/topic_posts.html b/templates/topic_posts.html index a15130c0..0616ffb2 100644 --- a/templates/topic_posts.html +++ b/templates/topic_posts.html @@ -8,6 +8,7 @@
{{/** TODO: We might end up with
s in the inline editor, fix this **/}}

{{.ContentHtml}}

+ {{if $.CurrentUser.Loggedin}}{{if $.CurrentUser.Perms.EditReply}}
{{.Content}}
{{end}}{{end}} diff --git a/themes/nox/public/main.css b/themes/nox/public/main.css index df827d81..f47b34b6 100644 --- a/themes/nox/public/main.css +++ b/themes/nox/public/main.css @@ -616,7 +616,7 @@ button, .formbutton, .panel_right_button:not(.has_inner_button) { } .topic_item .topic_forum { font-size: 19px; - line-height: 31px; + line-height: 30px; color: #cccccc; } .topic_view_count { diff --git a/tickloop.go b/tickloop.go index 3a798633..32ba8029 100644 --- a/tickloop.go +++ b/tickloop.go @@ -45,7 +45,13 @@ func runHook(name string) { } } -func tickLoop(thumbChan chan bool, halfSecondTicker *time.Ticker, secondTicker *time.Ticker, fifteenMinuteTicker *time.Ticker, hourTicker *time.Ticker) { +func tickLoop(thumbChan chan bool) { + // TODO: Write tests for these + // Run this goroutine once every half second + halfSecondTicker := time.NewTicker(time.Second / 2) + secondTicker := time.NewTicker(time.Second) + fifteenMinuteTicker := time.NewTicker(15 * time.Minute) + hourTicker := time.NewTicker(time.Hour) for { select { case <-halfSecondTicker.C: diff --git a/tmpl_stub.go b/tmpl_stub.go index cc94bfce..855e1dfe 100644 --- a/tmpl_stub.go +++ b/tmpl_stub.go @@ -23,3 +23,12 @@ func StringToBytes(s string) (bytes []byte) { runtime.KeepAlive(&s) return bytes } + +func BytesToString(bytes []byte) (s string) { + slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes)) + str := (*reflect.StringHeader)(unsafe.Pointer(&s)) + str.Data = slice.Data + str.Len = slice.Len + runtime.KeepAlive(&bytes) + return s +}