From c2f2dd7f10acd51df65d8caa5baa1fd2089cdb38 Mon Sep 17 00:00:00 2001 From: Azareal Date: Sun, 12 May 2019 09:07:24 +1000 Subject: [PATCH] Reduce the amount of data sent for alert resumes in preparation for moving resumes to the WebSockets endpoint. Remove a bit of superfluous information from the avatar alt attributes. Improved upon the meta store tests. Move float: right; out of the footer and into the Tempra Simple and Shadow stylesheets. Added a missing level label to Shadow. --- common/alerts.go | 8 ++- go.mod | 2 +- go.sum | 2 + misc_test.go | 19 +++++- mysql.go | 2 +- public/global.js | 82 +++++++++++++------------ routes.go | 72 +++++++++++++--------- routes/reply.go | 4 +- routes/topic.go | 2 +- templates/footer.html | 2 +- templates/forum.html | 4 +- templates/forums.html | 2 +- templates/ip_search.html | 2 +- templates/panel_users.html | 2 +- templates/profile.html | 2 +- templates/profile_comments_row_alt.html | 2 +- templates/topics_topic.html | 6 +- templates/widget_online.html | 2 +- themes/nox/overrides/topics_topic.html | 4 +- themes/shadow/public/main.css | 6 ++ themes/tempra_simple/public/main.css | 3 + 21 files changed, 138 insertions(+), 92 deletions(-) diff --git a/common/alerts.go b/common/alerts.go index 2e92d80b..11e9cdaf 100644 --- a/common/alerts.go +++ b/common/alerts.go @@ -11,6 +11,7 @@ import ( "errors" "strconv" "strings" + "time" "github.com/Azareal/Gosora/common/phrases" "github.com/Azareal/Gosora/query_gen" @@ -23,6 +24,7 @@ type Alert struct { Event string ElementType string ElementID int + CreatedAt time.Time Actor *User } @@ -42,14 +44,14 @@ var alertStmts AlertStmts func init() { DbInits.Add(func(acc *qgen.Accumulator) error { alertStmts = AlertStmts{ - addActivity: acc.Insert("activity_stream").Columns("actor, targetUser, event, elementType, elementID").Fields("?,?,?,?,?").Prepare(), + addActivity: acc.Insert("activity_stream").Columns("actor, targetUser, event, elementType, elementID, createdAt").Fields("?,?,?,?,?,UTC_TIMESTAMP()").Prepare(), notifyWatchers: acc.SimpleInsertInnerJoin( qgen.DBInsert{"activity_stream_matches", "watcher, asid", ""}, qgen.DBJoin{"activity_stream", "activity_subscriptions", "activity_subscriptions.user, activity_stream.asid", "activity_subscriptions.targetType = activity_stream.elementType AND activity_subscriptions.targetID = activity_stream.elementID AND activity_subscriptions.user != activity_stream.actor", "asid = ?", "", ""}, ), notifyOne: acc.Insert("activity_stream_matches").Columns("watcher, asid").Fields("?,?").Prepare(), getWatchers: acc.SimpleInnerJoin("activity_stream", "activity_subscriptions", "activity_subscriptions.user", "activity_subscriptions.targetType = activity_stream.elementType AND activity_subscriptions.targetID = activity_stream.elementID AND activity_subscriptions.user != activity_stream.actor", "asid = ?", "", ""), - getActivityEntry: acc.Select("activity_stream").Columns("actor, targetUser, event, elementType, elementID").Where("asid = ?").Prepare(), + getActivityEntry: acc.Select("activity_stream").Columns("actor, targetUser, event, elementType, elementID, createdAt").Where("asid = ?").Prepare(), } return acc.FirstError() }) @@ -237,7 +239,7 @@ func notifyWatchers(asid int64) { } var alert = Alert{ASID: int(asid)} - err = alertStmts.getActivityEntry.QueryRow(asid).Scan(&alert.ActorID, &alert.TargetUserID, &alert.Event, &alert.ElementType, &alert.ElementID) + err = alertStmts.getActivityEntry.QueryRow(asid).Scan(&alert.ActorID, &alert.TargetUserID, &alert.Event, &alert.ElementType, &alert.ElementID, &alert.CreatedAt) if err != nil && err != ErrNoRows { LogError(err) return diff --git a/go.mod b/go.mod index 3c88a96f..9071bcab 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/go-sql-driver/mysql v1.4.0 github.com/gorilla/websocket v1.4.0 github.com/lib/pq v1.0.0 - github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 + github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 github.com/olivere/elastic v6.2.16+incompatible // indirect github.com/oschwald/geoip2-golang v1.2.1 github.com/oschwald/maxminddb-golang v1.3.0 // indirect diff --git a/go.sum b/go.sum index cca094e6..560e26ec 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg= +github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/olivere/elastic v6.2.16+incompatible h1:+mQIHbkADkOgq9tFqnbyg7uNFVV6swGU07EoK1u0nEQ= diff --git a/misc_test.go b/misc_test.go index 7542c246..7a8ace92 100644 --- a/misc_test.go +++ b/misc_test.go @@ -990,13 +990,26 @@ func TestPhrases(t *testing.T) { func TestMetaStore(t *testing.T) { m, err := c.Meta.Get("magic") - expect(t, m == "", "meta var should be empty") - recordMustNotExist(t, err, "meta var shouldn't exist") + expect(t, m == "", "meta var magic should be empty") + recordMustNotExist(t, err, "meta var magic should not exist") + err = c.Meta.Set("magic","lol") expectNilErr(t,err) + m, err = c.Meta.Get("magic") expectNilErr(t,err) - expect(t,m=="lol","meta var should be lol") + expect(t,m=="lol","meta var magic should be lol") + + err = c.Meta.Set("magic","wha") + expectNilErr(t,err) + + m, err = c.Meta.Get("magic") + expectNilErr(t,err) + expect(t,m=="wha","meta var magic should be wha") + + m, err = c.Meta.Get("giggle") + expect(t, m == "", "meta var giggle should be empty") + recordMustNotExist(t, err, "meta var giggle should not exist") } func TestWordFilters(t *testing.T) { diff --git a/mysql.go b/mysql.go index 8ab9f077..505a22d4 100644 --- a/mysql.go +++ b/mysql.go @@ -54,7 +54,7 @@ func initMySQL() (err error) { // TODO: Is there a less noisy way of doing this for tests? log.Print("Preparing getActivityFeedByWatcher statement.") - stmts.getActivityFeedByWatcher, err = db.Prepare("SELECT activity_stream_matches.asid, activity_stream.actor, activity_stream.targetUser, activity_stream.event, activity_stream.elementType, activity_stream.elementID FROM `activity_stream_matches` INNER JOIN `activity_stream` ON activity_stream_matches.asid = activity_stream.asid AND activity_stream_matches.watcher != activity_stream.actor WHERE `watcher` = ? ORDER BY activity_stream.asid DESC LIMIT 16") + stmts.getActivityFeedByWatcher, err = db.Prepare("SELECT activity_stream_matches.asid, activity_stream.actor, activity_stream.targetUser, activity_stream.event, activity_stream.elementType, activity_stream.elementID, activity_stream.createdAt FROM `activity_stream_matches` INNER JOIN `activity_stream` ON activity_stream_matches.asid = activity_stream.asid AND activity_stream_matches.watcher != activity_stream.actor WHERE `watcher` = ? ORDER BY activity_stream.asid DESC LIMIT 16") if err != nil { return errors.WithStack(err) } diff --git a/public/global.js b/public/global.js index 448ad1f0..a4f06788 100644 --- a/public/global.js +++ b/public/global.js @@ -49,7 +49,7 @@ function bindToAlerts() { type: "POST", dataType: "json", data: { id: $(this).attr("data-asid") }, - error: ajaxError, + //error: ajaxError, success: () => { window.location.href = this.getAttribute("href"); } @@ -134,12 +134,15 @@ function setAlertError(menuAlerts,msg) { } var alertsInitted = false; -function loadAlerts(menuAlerts) { +var lastTc = 0; +function loadAlerts(menuAlerts, eTc = false) { if(!alertsInitted) return; + let tc = ""; + if(eTc && lastTc != 0) tc = "&t=" + lastTc + "&c=" + alertCount; $.ajax({ type: 'get', dataType: 'json', - url:'/api/?module=alerts', + url:'/api/?module=alerts' + tc, success: (data) => { if("errmsg" in data) { setAlertError(menuAlerts,data.errmsg) @@ -147,11 +150,17 @@ function loadAlerts(menuAlerts) { } alertList = []; alertMapping = {}; - if(!data.hasOwnProperty("msgs")) data = {"msgs":[],"count":0}; - for(var i in data.msgs) addAlert(data.msgs[i]); - console.log("data.count:",data.count) - alertCount = data.count; - updateAlertList(menuAlerts) + if(!data.hasOwnProperty("msgs")) data = {"msgs":[],"count":alertCount,"tc":lastTc}; + /*else if(data.count != (alertCount + data.msgs.length)) tc = false; + if(eTc && lastTc != 0) { + for(var i in data.msgs) wsAlertEvent(data.msgs[i]); + } else {*/ + for(var i in data.msgs) addAlert(data.msgs[i]); + console.log("data.count:",data.count); + alertCount = data.count; + updateAlertList(menuAlerts); + //} + lastTc = data.tc; }, error: (magic,theStatus,error) => { let errtxt = "Unable to get the alerts"; @@ -196,11 +205,10 @@ function wsAlertEvent(data) { alertList = [alertList[alertList.length-1]]; aTmp = aTmp.slice(0,-1); for(let i = 0; i < aTmp.length; i++) alertList.push(aTmp[i]); - //var alist = ""; - //for (var i = 0; i < alertList.length; i++) alist += alertMapping[alertList[i]]; // TODO: Add support for other alert feeds like PM Alerts var generalAlerts = document.getElementById("general_alerts"); // TODO: Make sure we update alertCount here + lastTc = 0; updateAlertList(generalAlerts/*, alist*/); } @@ -259,28 +267,27 @@ function runWebSockets(resume = false) { else if("event" in data) { if(data.event == "dismiss-alert"){ Object.keys(alertMapping).forEach((key) => { - if(key==data.id) { - alertCount--; - let index = -1; - for(var i = 0; i < alertList.length; i++) { - if(alertList[i]==key) { - alertList[i] = 0; - index = i; - } + if(key!=data.id) return; + alertCount--; + let index = -1; + for(var i = 0; i < alertList.length; i++) { + if(alertList[i]==key) { + alertList[i] = 0; + index = i; } - if(index==-1) return; - - for(var i = index; (i+1) < alertList.length; i++) { - alertList[i] = alertList[i+1]; - } - alertList.splice(alertList.length-1,1); - delete alertMapping[key]; - - // TODO: Add support for other alert feeds like PM Alerts - var generalAlerts = document.getElementById("general_alerts"); - if(alertList.length < 8) loadAlerts(generalAlerts); - else updateAlertList(generalAlerts); } + if(index==-1) return; + + for(var i = index; (i+1) < alertList.length; i++) { + alertList[i] = alertList[i+1]; + } + alertList.splice(alertList.length-1,1); + delete alertMapping[key]; + + // TODO: Add support for other alert feeds like PM Alerts + var generalAlerts = document.getElementById("general_alerts"); + if(alertList.length < 8) loadAlerts(generalAlerts); + else updateAlertList(generalAlerts); }); } } else if("Topics" in data) { @@ -366,18 +373,15 @@ function runWebSockets(resume = false) { function PageOffset(count, page, perPage) { let offset = 0; let lastPage = LastPage(count, perPage) - if(page > 1) { - offset = (perPage * page) - perPage - } else if (page == -1) { - page = lastPage - offset = (perPage * page) - perPage - } else { - page = 1 - } + if(page > 1) offset = (perPage * page) - perPage; + else if (page == -1) { + page = lastPage; + offset = (perPage * page) - perPage; + } else page = 1; // We don't want the offset to overflow the slices, if everything's in memory //if(offset >= (count - 1)) offset = 0; - return {Offset:offset, Page:page, LastPage:lastPage} + return {Offset:offset, Page:page, LastPage:lastPage}; } function LastPage(count, perPage) { return (count / perPage) + 1 diff --git a/routes.go b/routes.go index 64b7bfb6..7df42919 100644 --- a/routes.go +++ b/routes.go @@ -10,12 +10,13 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "io" "errors" + "io" "log" "net/http" "strconv" "strings" + "time" "unicode" c "github.com/Azareal/Gosora/common" @@ -74,9 +75,9 @@ func routeAPI(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError var etag string _, ok := w.(c.GzipResponseWriter) if ok { - etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"-ng\"" + etag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-ng\"" } else { - etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"-n\"" + etag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-n\"" } w.Header().Set("ETag", etag) if match := r.Header.Get("If-None-Match"); match != "" { @@ -97,26 +98,43 @@ func routeAPI(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError return c.InternalErrorJS(err, w, r) } - rows, err := stmts.getActivityFeedByWatcher.Query(user.ID) - if err != nil { - return c.InternalErrorJS(err, w, r) - } - defer rows.Close() - + rCreatedAt, _ := strconv.ParseInt(r.FormValue("t"), 10, 64) + rCount, _ := strconv.Atoi(r.FormValue("c")) + //log.Print("rCreatedAt:", rCreatedAt) + //log.Print("rCount:", rCount) var actors []int var alerts []c.Alert - for rows.Next() { - var alert c.Alert - err = rows.Scan(&alert.ASID, &alert.ActorID, &alert.TargetUserID, &alert.Event, &alert.ElementType, &alert.ElementID) + var createdAt time.Time + var topCreatedAt int64 + + if count != 0 { + rows, err := stmts.getActivityFeedByWatcher.Query(user.ID) + if err != nil { + return c.InternalErrorJS(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var alert c.Alert + err = rows.Scan(&alert.ASID, &alert.ActorID, &alert.TargetUserID, &alert.Event, &alert.ElementType, &alert.ElementID, &createdAt) + if err != nil { + return c.InternalErrorJS(err, w, r) + } + + uCreatedAt := createdAt.Unix() + //log.Print("uCreatedAt", uCreatedAt) + //if rCreatedAt == 0 || rCreatedAt < uCreatedAt { + alerts = append(alerts, alert) + actors = append(actors, alert.ActorID) + //} + if uCreatedAt > topCreatedAt { + topCreatedAt = uCreatedAt + } + } + err = rows.Err() if err != nil { return c.InternalErrorJS(err, w, r) } - alerts = append(alerts, alert) - actors = append(actors, alert.ActorID) - } - err = rows.Err() - if err != nil { - return c.InternalErrorJS(err, w, r) } // Might not want to error here, if the account was deleted properly, we might want to figure out how we should handle deletions in general @@ -134,23 +152,21 @@ func routeAPI(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError if !ok { return c.InternalErrorJS(errors.New("No such actor"), w, r) } - res, err := c.BuildAlert(alert, user) if err != nil { return c.LocalErrorJS(err.Error(), w, r) } - //sb.Write(res) msglist += res + "," } - if len(msglist) != 0 { msglist = msglist[0 : len(msglist)-1] } - if count == 0 { + + if count == 0 || msglist == "" || (rCreatedAt != 0 && rCreatedAt >= topCreatedAt && count == rCount) { _, _ = io.WriteString(w, `{}`) } else { - _, _ = io.WriteString(w, `{"msgs":[` + msglist + `],"count":` + strconv.Itoa(count) + `}`) + _, _ = io.WriteString(w, `{"msgs":[`+msglist+`],"count":`+strconv.Itoa(count)+`,"tc":`+strconv.Itoa(int(topCreatedAt))+`}`) } default: return c.PreErrorJS("Invalid Module", w, r) @@ -218,18 +234,18 @@ func routeAPIPhrases(w http.ResponseWriter, r *http.Request, user c.User) c.Rout var etag string _, ok := w.(c.GzipResponseWriter) if ok { - etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"-g\"" + etag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-g\"" } else { - etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"\"" + etag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "\"" } - + var plist map[string]string var posLoop = func(positive string) c.RouteError { // ! Constrain it to a subset of phrases for now for _, item := range phraseWhitelist { if strings.HasPrefix(positive, item) { // TODO: Break this down into smaller security boundaries based on control panel sections? - if strings.HasPrefix(positive,"panel") { + if strings.HasPrefix(positive, "panel") { w.Header().Set("Cache-Control", "private") ok = user.IsSuperMod } else { @@ -306,7 +322,7 @@ func routeJSAntispam(w http.ResponseWriter, r *http.Request, user c.User) c.Rout jsToken := hex.EncodeToString(h.Sum(nil)) var innerCode = "`document.getElementByld('golden-watch').value = '" + jsToken + "';`" - io.WriteString(w, `let hihi = ` + innerCode + `; + io.WriteString(w, `let hihi = `+innerCode+`; hihi = hihi.replace('ld','Id'); eval(hihi);`) diff --git a/routes/reply.go b/routes/reply.go index 314a4bff..c2970d6e 100644 --- a/routes/reply.go +++ b/routes/reply.go @@ -487,7 +487,7 @@ func ProfileReplyCreateSubmit(w http.ResponseWriter, r *http.Request, user c.Use } // ! Be careful about leaking per-route permission state with &user - alert := c.Alert{0, user.ID, profileOwner.ID, "reply", "user", profileOwner.ID, &user} + alert := c.Alert{ActorID: user.ID, TargetUserID: profileOwner.ID, Event: "reply", ElementType: "user", ElementID: profileOwner.ID, Actor: &user} err = c.AddActivityAndNotifyTarget(alert) if err != nil { return c.InternalError(err, w, r) @@ -623,7 +623,7 @@ func ReplyLikeSubmit(w http.ResponseWriter, r *http.Request, user c.User, srid s } // ! Be careful about leaking per-route permission state with &user - alert := c.Alert{0, user.ID, reply.CreatedBy, "like", "post", rid, &user} + alert := c.Alert{ActorID: user.ID, TargetUserID: reply.CreatedBy, Event: "like", ElementType: "post", ElementID: rid, Actor: &user} err = c.AddActivityAndNotifyTarget(alert) if err != nil { return c.InternalErrorJSQ(err, w, r, isJs) diff --git a/routes/topic.go b/routes/topic.go index b71abeaf..32c2a6a5 100644 --- a/routes/topic.go +++ b/routes/topic.go @@ -1031,7 +1031,7 @@ func LikeTopicSubmit(w http.ResponseWriter, r *http.Request, user c.User, stid s } // ! Be careful about leaking per-route permission state with &user - alert := c.Alert{0, user.ID, topic.CreatedBy, "like", "topic", tid, &user} + alert := c.Alert{ActorID: user.ID, TargetUserID: topic.CreatedBy, Event: "like", ElementType: "topic", ElementID: tid, Actor: &user} err = c.AddActivityAndNotifyTarget(alert) if err != nil { return c.InternalErrorJSQ(err, w, r, isJs) diff --git a/templates/footer.html b/templates/footer.html index ace0807a..5bb17b82 100644 --- a/templates/footer.html +++ b/templates/footer.html @@ -11,7 +11,7 @@ {{if .CurrentUser.IsAdmin}}
{{.Header.Elapsed1}} | {{elapsed .Header.StartedAt}}
{{end}}
-
+