From 5a8b994877d81c3a9c0cf2d586953c46b3591d02 Mon Sep 17 00:00:00 2001 From: Azareal Date: Thu, 8 Mar 2018 03:59:47 +0000 Subject: [PATCH] Added support for phrases in templates. The language of the end-user is now tracked and presented in the Analytics Manager. Profile owners now get alerts when someone posts on their profiles. The login page is now transpiled, estimated to be sixty times faster. The registration page is now transpiled, estimated to be sixty times faster. The IP Search page is now transpiled, estimated to be sixty times faster. The error pages are now transpiled, estimated to be sixty times faster. The login page now uses phrases. The registration page now uses phrases. IP Search now uses phrases. Renamed the ip-search template to ip_search. Alerts are now held in an alertbox container div. Added ids for the main container divs for the account manager sections. Added an id to the main container for the topic list template. Added an id to the main container for the forum list template. Added an id to the main container for the forum template. Added an avatar box CSS class for the avatar box in the account manager's avatar page. Did a bit of renaming for a future refactor in the routes counter. Did a bit of renaming for a future refactor in the operating system counter. A notice is shown to the user now when their account is inactive. The account activation status is now fetched by the user store. We now track Slackbot. You can now prepend strings to the start of router.DumpRequest request dumps to avoid tearing these bits of contextual data away from the bodies. .action file extensions are now seen as suspicious by the router. Moved routeWebsockets to common.RouteWebsockets for now. Moved routeCreateReplySubmit to routes.CreateReplySubmit. Moved alert.go into common. Moved the WebSockets logic into common. Escape strings a little earlier in the analytics routes and use integers instead of strings where possible. We now show a success notification when you update a user via the User Manager. Split the configuration properties off from CTemplateSet into CTemplateConfig. Renamed some of the properties of CTemplateSet to make them easier to understand. Removed some obsolete properties from CTemplateSet. Did a bit of spring cleaning in the template transpiler to cut down on unneccessary lines and to reduce duplication. Fixed a double else bug in ranges over maps in the template transpiler. Split the minifiers off the main template transpilation file into their own file. Refactored some of the routes which rely on alerts to use shared functions rather than having unique implementations in the routes themselves. All Themes Except Cosora: Refactored the opt nodes to make it easier to roll out bulk moderation. Shadow: Improved the notice CSS. Tweaked the sticky border colour. Cosora: The theme JS file now uses strict mode. Notices are shunted under rowhead with JS now, although this change might be reverted soon. Added CSS for notices. Fixed the padding under the avatar box in the account manager avatar page. Schema: Added the viewchunks_langs table. --- alerts.go | 164 ------- common/alerts.go | 243 ++++++++++ common/counters/langs.go | 166 +++++++ common/counters/routes.go | 20 +- common/counters/systems.go | 32 +- common/errors.go | 2 +- no_websockets.go => common/no_websockets.go | 6 +- common/phrases.go | 43 +- common/routes_common.go | 3 + common/template_init.go | 99 +++- common/templates/minifiers.go | 54 +++ common/templates/templates.go | 238 ++++----- common/themes.go | 21 + common/user_store.go | 14 +- websockets.go => common/websockets.go | 118 ++--- gen_mssql.go | 45 -- gen_mysql.go | 40 -- gen_router.go | 481 ++++++++++--------- langs/english.json | 117 ++++- main.go | 8 +- member_routes.go | 226 +-------- panel_routes.go | 117 ++++- query_gen/main.go | 16 - query_gen/tables.go | 9 + router_gen/main.go | 58 ++- router_gen/routes.go | 6 +- routes.go | 2 +- routes/account.go | 4 +- routes/moderate.go | 8 +- routes/reply.go | 175 +++++++ schema/mssql/query_viewchunks_langs.sql | 5 + schema/mysql/query_viewchunks_langs.sql | 5 + schema/pgsql/query_viewchunks_langs.sql | 5 + template_error.go | 115 +++++ template_forum.go | 123 ++--- template_forums.go | 4 +- template_guilds_guild_list.go | 4 +- template_ip_search.go | 145 ++++++ template_list.go | 424 ++++++++++------ template_login.go | 128 +++++ template_profile.go | 4 +- template_register.go | 131 +++++ template_topic.go | 2 + template_topic_alt.go | 2 + template_topics.go | 153 +++--- templates/account_own_edit.html | 2 +- templates/account_own_edit_avatar.html | 4 +- templates/account_own_edit_email.html | 2 +- templates/account_own_edit_username.html | 2 +- templates/error.html | 2 +- templates/forum.html | 22 +- templates/forums.html | 2 +- templates/header.html | 4 +- templates/{ip-search.html => ip_search.html} | 4 +- templates/login.html | 12 +- templates/panel-inner-menu.html | 3 + templates/panel_analytics_lang_views.html | 34 ++ templates/panel_analytics_langs.html | 30 ++ templates/register.html | 14 +- templates/topics.html | 20 +- themes/cosora/public/main.css | 17 +- themes/cosora/public/misc.js | 11 +- themes/shadow/public/main.css | 37 +- themes/tempra-conflux/public/main.css | 16 +- themes/tempra-simple/public/main.css | 23 +- 65 files changed, 2701 insertions(+), 1345 deletions(-) delete mode 100644 alerts.go create mode 100644 common/alerts.go create mode 100644 common/counters/langs.go rename no_websockets.go => common/no_websockets.go (84%) create mode 100644 common/templates/minifiers.go rename websockets.go => common/websockets.go (81%) create mode 100644 schema/mssql/query_viewchunks_langs.sql create mode 100644 schema/mysql/query_viewchunks_langs.sql create mode 100644 schema/pgsql/query_viewchunks_langs.sql create mode 100644 template_error.go create mode 100644 template_ip_search.go create mode 100644 template_login.go create mode 100644 template_register.go rename templates/{ip-search.html => ip_search.html} (87%) create mode 100644 templates/panel_analytics_lang_views.html create mode 100644 templates/panel_analytics_langs.html diff --git a/alerts.go b/alerts.go deleted file mode 100644 index 07741251..00000000 --- a/alerts.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -* -* Gosora Alerts System -* Copyright Azareal 2017 - 2018 -* - */ -package main - -import ( - "errors" - "log" - "strconv" - "strings" - - "./common" -) - -// These notes are for me, don't worry about it too much ^_^ -/* -"You received a friend invite from {user}" -"{x}{mentioned you on}{user}{'s profile}" -"{x}{mentioned you in}{topic}" -"{x}{likes}{you}" -"{x}{liked}{your topic}{topic}" -"{x}{liked}{your post on}{user}{'s profile}" todo -"{x}{liked}{your post in}{topic}" -"{x}{replied to}{your post in}{topic}" todo -"{x}{replied to}{topic}" -"{x}{replied to}{your topic}{topic}" -"{x}{created a new topic}{topic}" -*/ - -func buildAlert(asid int, event string, elementType string, actorID int, targetUserID int, elementID int, user common.User /* The current user */) (string, error) { - var targetUser *common.User - - actor, err := common.Users.Get(actorID) - if err != nil { - return "", errors.New("Unable to find the actor") - } - - /*if elementType != "forum" { - targetUser, err = users.Get(targetUser_id) - if err != nil { - LocalErrorJS("Unable to find the target user",w,r) - return - } - }*/ - - if event == "friend_invite" { - return `{"msg":"You received a friend invite from {0}","sub":["` + actor.Name + `"],"path":"` + actor.Link + `","avatar":"` + strings.Replace(actor.Avatar, "/", "\\/", -1) + `","asid":"` + strconv.Itoa(asid) + `"}`, nil - } - - var act, postAct, url, area string - var startFrag, endFrag string - switch elementType { - case "forum": - if event == "reply" { - act = "created a new topic" - topic, err := common.Topics.Get(elementID) - if err != nil { - common.DebugLogf("Unable to find linked topic %d", elementID) - return "", errors.New("Unable to find the linked topic") - } - url = topic.Link - area = topic.Title - // Store the forum ID in the targetUser column instead of making a new one? o.O - // Add an additional column for extra information later on when we add the ability to link directly to posts. We don't need the forum data for now... - } else { - act = "did something in a forum" - } - case "topic": - topic, err := common.Topics.Get(elementID) - if err != nil { - common.DebugLogf("Unable to find linked topic %d", elementID) - return "", errors.New("Unable to find the linked topic") - } - url = topic.Link - area = topic.Title - - if targetUserID == user.ID { - postAct = " your topic" - } - case "user": - targetUser, err = common.Users.Get(elementID) - if err != nil { - common.DebugLogf("Unable to find target user %d", elementID) - return "", errors.New("Unable to find the target user") - } - area = targetUser.Name - endFrag = "'s profile" - url = targetUser.Link - case "post": - topic, err := common.TopicByReplyID(elementID) - if err != nil { - return "", errors.New("Unable to find the linked reply or parent topic") - } - url = topic.Link - area = topic.Title - if targetUserID == user.ID { - postAct = " your post in" - } - default: - return "", errors.New("Invalid elementType") - } - - switch event { - case "like": - if elementType == "user" { - act = "likes" - endFrag = "" - if targetUser.ID == user.ID { - area = "you" - } - } else { - act = "liked" - } - case "mention": - if elementType == "user" { - act = "mentioned you on" - } else { - act = "mentioned you in" - postAct = "" - } - case "reply": - act = "replied to" - } - - return `{"msg":"{0} ` + startFrag + act + postAct + ` {1}` + endFrag + `","sub":["` + actor.Name + `","` + area + `"],"path":"` + url + `","avatar":"` + actor.Avatar + `","asid":"` + strconv.Itoa(asid) + `"}`, nil -} - -func notifyWatchers(asid int64) { - rows, err := stmts.getWatchers.Query(asid) - if err != nil && err != ErrNoRows { - log.Fatal(err.Error()) - return - } - defer rows.Close() - - var uid int - var uids []int - for rows.Next() { - err := rows.Scan(&uid) - if err != nil { - log.Fatal(err.Error()) - return - } - uids = append(uids, uid) - } - err = rows.Err() - if err != nil { - log.Fatal(err.Error()) - return - } - - var actorID, targetUserID, elementID int - var event, elementType string - err = stmts.getActivityEntry.QueryRow(asid).Scan(&actorID, &targetUserID, &event, &elementType, &elementID) - if err != nil && err != ErrNoRows { - log.Fatal(err.Error()) - return - } - - _ = wsHub.pushAlerts(uids, int(asid), event, elementType, actorID, targetUserID, elementID) -} diff --git a/common/alerts.go b/common/alerts.go new file mode 100644 index 00000000..3b8d0e8c --- /dev/null +++ b/common/alerts.go @@ -0,0 +1,243 @@ +/* +* +* Gosora Alerts System +* Copyright Azareal 2017 - 2018 +* + */ +package common + +import ( + "database/sql" + "errors" + "log" + "strconv" + "strings" + + "../query_gen/lib" +) + +type AlertStmts struct { + addActivity *sql.Stmt + notifyWatchers *sql.Stmt + notifyOne *sql.Stmt + getWatchers *sql.Stmt + getActivityEntry *sql.Stmt +} + +var alertStmts AlertStmts + +// TODO: Move these statements into some sort of activity abstraction +// TODO: Rewrite the alerts logic +func init() { + DbInits.Add(func(acc *qgen.Accumulator) error { + alertStmts = AlertStmts{ + addActivity: acc.Insert("activity_stream").Columns("actor, targetUser, event, elementType, elementID").Fields("?,?,?,?,?").Prepare(), + notifyWatchers: acc.SimpleInsertInnerJoin( + qgen.DBInsert{"activity_stream_matches", "watcher, asid", ""}, + qgen.DBJoin{"activity_stream", "activity_subscriptions", "activity_subscriptions.user, activity_stream.asid", "activity_subscriptions.targetType = activity_stream.elementType AND activity_subscriptions.targetID = activity_stream.elementID AND activity_subscriptions.user != activity_stream.actor", "asid = ?", "", ""}, + ), + notifyOne: acc.Insert("activity_stream_matches").Columns("watcher, asid").Fields("?,?").Prepare(), + getWatchers: acc.SimpleInnerJoin("activity_stream", "activity_subscriptions", "activity_subscriptions.user", "activity_subscriptions.targetType = activity_stream.elementType AND activity_subscriptions.targetID = activity_stream.elementID AND activity_subscriptions.user != activity_stream.actor", "asid = ?", "", ""), + getActivityEntry: acc.Select("activity_stream").Columns("actor, targetUser, event, elementType, elementID").Where("asid = ?").Prepare(), + } + return acc.FirstError() + }) +} + +// These notes are for me, don't worry about it too much ^_^ +/* +"You received a friend invite from {user}" +"{x}{mentioned you on}{user}{'s profile}" +"{x}{mentioned you in}{topic}" +"{x}{likes}{you}" +"{x}{liked}{your topic}{topic}" +"{x}{liked}{your post on}{user}{'s profile}" todo +"{x}{liked}{your post in}{topic}" +"{x}{replied to}{your post in}{topic}" todo +"{x}{replied to}{topic}" +"{x}{replied to}{your topic}{topic}" +"{x}{created a new topic}{topic}" +*/ + +func BuildAlert(asid int, event string, elementType string, actorID int, targetUserID int, elementID int, user User /* The current user */) (string, error) { + var targetUser *User + + actor, err := Users.Get(actorID) + if err != nil { + return "", errors.New("Unable to find the actor") + } + + /*if elementType != "forum" { + targetUser, err = users.Get(targetUser_id) + if err != nil { + LocalErrorJS("Unable to find the target user",w,r) + return + } + }*/ + + if event == "friend_invite" { + return `{"msg":"You received a friend invite from {0}","sub":["` + actor.Name + `"],"path":"` + actor.Link + `","avatar":"` + strings.Replace(actor.Avatar, "/", "\\/", -1) + `","asid":"` + strconv.Itoa(asid) + `"}`, nil + } + + var act, postAct, url, area string + var startFrag, endFrag string + switch elementType { + case "forum": + if event == "reply" { + act = "created a new topic" + topic, err := Topics.Get(elementID) + if err != nil { + DebugLogf("Unable to find linked topic %d", elementID) + return "", errors.New("Unable to find the linked topic") + } + url = topic.Link + area = topic.Title + // Store the forum ID in the targetUser column instead of making a new one? o.O + // Add an additional column for extra information later on when we add the ability to link directly to posts. We don't need the forum data for now... + } else { + act = "did something in a forum" + } + case "topic": + topic, err := Topics.Get(elementID) + if err != nil { + DebugLogf("Unable to find linked topic %d", elementID) + return "", errors.New("Unable to find the linked topic") + } + url = topic.Link + area = topic.Title + + if targetUserID == user.ID { + postAct = " your topic" + } + case "user": + targetUser, err = Users.Get(elementID) + if err != nil { + DebugLogf("Unable to find target user %d", elementID) + return "", errors.New("Unable to find the target user") + } + area = targetUser.Name + endFrag = "'s profile" + url = targetUser.Link + case "post": + topic, err := TopicByReplyID(elementID) + if err != nil { + return "", errors.New("Unable to find the linked reply or parent topic") + } + url = topic.Link + area = topic.Title + if targetUserID == user.ID { + postAct = " your post in" + } + default: + return "", errors.New("Invalid elementType") + } + + switch event { + case "like": + if elementType == "user" { + act = "likes" + endFrag = "" + if targetUser.ID == user.ID { + area = "you" + } + } else { + act = "liked" + } + case "mention": + if elementType == "user" { + act = "mentioned you on" + } else { + act = "mentioned you in" + postAct = "" + } + case "reply": + act = "replied to" + } + + return `{"msg":"{0} ` + startFrag + act + postAct + ` {1}` + endFrag + `","sub":["` + actor.Name + `","` + area + `"],"path":"` + url + `","avatar":"` + actor.Avatar + `","asid":"` + strconv.Itoa(asid) + `"}`, nil +} + +func AddActivityAndNotifyAll(actor int, targetUser int, event string, elementType string, elementID int) error { + res, err := alertStmts.addActivity.Exec(actor, targetUser, event, elementType, elementID) + if err != nil { + return err + } + lastID, err := res.LastInsertId() + if err != nil { + return err + } + return NotifyWatchers(lastID) +} + +func AddActivityAndNotifyTarget(actor int, targetUser int, event string, elementType string, elementID int) error { + res, err := alertStmts.addActivity.Exec(actor, targetUser, event, elementType, elementID) + if err != nil { + return err + } + lastID, err := res.LastInsertId() + if err != nil { + return err + } + err = NotifyOne(targetUser, lastID) + if err != nil { + return err + } + + // Live alerts, if the target is online and WebSockets is enabled + _ = WsHub.pushAlert(targetUser, int(lastID), event, elementType, actor, targetUser, elementID) + return nil +} + +func NotifyOne(watcher int, asid int64) error { + _, err := alertStmts.notifyOne.Exec(watcher, asid) + return err +} + +func NotifyWatchers(asid int64) error { + _, err := alertStmts.notifyWatchers.Exec(asid) + if err != nil { + return err + } + + // Alert the subscribers about this without blocking us from doing something else + if EnableWebsockets { + go notifyWatchers(asid) + } + + return nil +} + +func notifyWatchers(asid int64) { + rows, err := alertStmts.getWatchers.Query(asid) + if err != nil && err != ErrNoRows { + log.Fatal(err.Error()) + return + } + defer rows.Close() + + var uid int + var uids []int + for rows.Next() { + err := rows.Scan(&uid) + if err != nil { + log.Fatal(err.Error()) + return + } + uids = append(uids, uid) + } + err = rows.Err() + if err != nil { + log.Fatal(err.Error()) + return + } + + var actorID, targetUserID, elementID int + var event, elementType string + err = alertStmts.getActivityEntry.QueryRow(asid).Scan(&actorID, &targetUserID, &event, &elementType, &elementID) + if err != nil && err != ErrNoRows { + log.Fatal(err.Error()) + return + } + + _ = WsHub.pushAlerts(uids, int(asid), event, elementType, actorID, targetUserID, elementID) +} diff --git a/common/counters/langs.go b/common/counters/langs.go new file mode 100644 index 00000000..a5312b39 --- /dev/null +++ b/common/counters/langs.go @@ -0,0 +1,166 @@ +package counters + +import "database/sql" +import ".." +import "../../query_gen/lib" + +var LangViewCounter *DefaultLangViewCounter + +var langCodes = []string{ + "unknown", + "af", + "ar", + "az", + "be", + "bg", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "dv", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "fi", + "fo", + "fr", + "gl", + "gu", + "he", + "hi", + "hr", + "hu", + "hy", + "id", + "is", + "it", + "ja", + "ka", + "kk", + "kn", + "ko", + "kok", + "ky", + "lt", + "lv", + "mi", + "mk", + "mn", + "mr", + "ms", + "mt", + "nb", + "nl", + "nn", + "ns", + "pa", + "pl", + "ps", + "pt", + "qu", + "ro", + "ru", + "sa", + "se", + "sk", + "sl", + "sq", + "sr", + "sv", + "sw", + "syr", + "ta", + "te", + "th", + "tl", + "tn", + "tr", + "tt", + "ts", + "uk", + "ur", + "uz", + "vi", + "xh", + "zh", + "zu", +} + +type DefaultLangViewCounter struct { + buckets []*RWMutexCounterBucket //[OSID]count + codesToIndices map[string]int + insert *sql.Stmt +} + +func NewDefaultLangViewCounter() (*DefaultLangViewCounter, error) { + acc := qgen.Builder.Accumulator() + + var langBuckets = make([]*RWMutexCounterBucket, len(langCodes)) + for bucketID, _ := range langBuckets { + langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} + } + var codesToIndices = make(map[string]int) + for index, code := range langCodes { + codesToIndices[code] = index + } + + counter := &DefaultLangViewCounter{ + buckets: langBuckets, + codesToIndices: codesToIndices, + insert: acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + } + + common.AddScheduledFifteenMinuteTask(counter.Tick) + //common.AddScheduledSecondTask(counter.Tick) + common.AddShutdownTask(counter.Tick) + return counter, acc.FirstError() +} + +func (counter *DefaultLangViewCounter) Tick() error { + for id, bucket := range counter.buckets { + var count int + bucket.RLock() + count = bucket.counter + bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? + bucket.RUnlock() + + err := counter.insertChunk(count, id) // TODO: Bulk insert for speed? + if err != nil { + return err + } + } + return nil +} + +func (counter *DefaultLangViewCounter) insertChunk(count int, id int) error { + if count == 0 { + return nil + } + var langCode = langCodes[id] + common.DebugLogf("Inserting a viewchunk with a count of %d for lang %s (%d)", count, langCode, id) + _, err := counter.insert.Exec(count, langCode) + return err +} + +func (counter *DefaultLangViewCounter) Bump(langCode string) { + id, ok := counter.codesToIndices[langCode] + if !ok { + // TODO: Tell the caller that the code's invalid + id = 0 // Unknown + } + + // TODO: Test this check + common.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id]) + if len(counter.buckets) <= id || id < 0 { + return + } + counter.buckets[id].Lock() + counter.buckets[id].counter++ + counter.buckets[id].Unlock() +} diff --git a/common/counters/routes.go b/common/counters/routes.go index 22fe2566..ee399845 100644 --- a/common/counters/routes.go +++ b/common/counters/routes.go @@ -8,8 +8,8 @@ var RouteViewCounter *DefaultRouteViewCounter // TODO: Make this lockless? type DefaultRouteViewCounter struct { - routeBuckets []*RWMutexCounterBucket //[RouteID]count - insert *sql.Stmt + buckets []*RWMutexCounterBucket //[RouteID]count + insert *sql.Stmt } func NewDefaultRouteViewCounter() (*DefaultRouteViewCounter, error) { @@ -19,8 +19,8 @@ func NewDefaultRouteViewCounter() (*DefaultRouteViewCounter, error) { routeBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} } counter := &DefaultRouteViewCounter{ - routeBuckets: routeBuckets, - insert: acc.Insert("viewchunks").Columns("count, createdAt, route").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + buckets: routeBuckets, + insert: acc.Insert("viewchunks").Columns("count, createdAt, route").Fields("?,UTC_TIMESTAMP(),?").Prepare(), } common.AddScheduledFifteenMinuteTask(counter.Tick) // There could be a lot of routes, so we don't want to be running this every second //common.AddScheduledSecondTask(counter.Tick) @@ -29,7 +29,7 @@ func NewDefaultRouteViewCounter() (*DefaultRouteViewCounter, error) { } func (counter *DefaultRouteViewCounter) Tick() error { - for routeID, routeBucket := range counter.routeBuckets { + for routeID, routeBucket := range counter.buckets { var count int routeBucket.RLock() count = routeBucket.counter @@ -56,11 +56,11 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error func (counter *DefaultRouteViewCounter) Bump(route int) { // TODO: Test this check - common.DebugDetail("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) - if len(counter.routeBuckets) <= route || route < 0 { + common.DebugDetail("counter.buckets[", route, "]: ", counter.buckets[route]) + if len(counter.buckets) <= route || route < 0 { return } - counter.routeBuckets[route].Lock() - counter.routeBuckets[route].counter++ - counter.routeBuckets[route].Unlock() + counter.buckets[route].Lock() + counter.buckets[route].counter++ + counter.buckets[route].Unlock() } diff --git a/common/counters/systems.go b/common/counters/systems.go index b24aa364..c25f5734 100644 --- a/common/counters/systems.go +++ b/common/counters/systems.go @@ -7,8 +7,8 @@ import "../../query_gen/lib" var OSViewCounter *DefaultOSViewCounter type DefaultOSViewCounter struct { - osBuckets []*RWMutexCounterBucket //[OSID]count - insert *sql.Stmt + buckets []*RWMutexCounterBucket //[OSID]count + insert *sql.Stmt } func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) { @@ -18,8 +18,8 @@ func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) { osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} } counter := &DefaultOSViewCounter{ - osBuckets: osBuckets, - insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + buckets: osBuckets, + insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(), } common.AddScheduledFifteenMinuteTask(counter.Tick) //common.AddScheduledSecondTask(counter.Tick) @@ -28,14 +28,14 @@ func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) { } func (counter *DefaultOSViewCounter) Tick() error { - for osID, osBucket := range counter.osBuckets { + for id, bucket := range counter.buckets { var count int - osBucket.RLock() - count = osBucket.counter - osBucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? - osBucket.RUnlock() + bucket.RLock() + count = bucket.counter + bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? + bucket.RUnlock() - err := counter.insertChunk(count, osID) // TODO: Bulk insert for speed? + err := counter.insertChunk(count, id) // TODO: Bulk insert for speed? if err != nil { return err } @@ -53,13 +53,13 @@ func (counter *DefaultOSViewCounter) insertChunk(count int, os int) error { return err } -func (counter *DefaultOSViewCounter) Bump(os int) { +func (counter *DefaultOSViewCounter) Bump(id int) { // TODO: Test this check - common.DebugDetail("counter.osBuckets[", os, "]: ", counter.osBuckets[os]) - if len(counter.osBuckets) <= os || os < 0 { + common.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id]) + if len(counter.buckets) <= id || id < 0 { return } - counter.osBuckets[os].Lock() - counter.osBuckets[os].counter++ - counter.osBuckets[os].Unlock() + counter.buckets[id].Lock() + counter.buckets[id].counter++ + counter.buckets[id].Unlock() } diff --git a/common/errors.go b/common/errors.go index 245ac182..cd6b7cf1 100644 --- a/common/errors.go +++ b/common/errors.go @@ -290,7 +290,7 @@ func handleErrorTemplate(w http.ResponseWriter, r *http.Request, pi Page) { if RunPreRenderHook("pre_render_error", w, r, &pi.CurrentUser, &pi) { return } - err := Templates.ExecuteTemplate(w, "error.html", pi) + err := RunThemeTemplate(pi.Header.Theme.Name, "error", pi, w) if err != nil { LogError(err) } diff --git a/no_websockets.go b/common/no_websockets.go similarity index 84% rename from no_websockets.go rename to common/no_websockets.go index 8f8d30a4..28bb37fa 100644 --- a/no_websockets.go +++ b/common/no_websockets.go @@ -1,12 +1,12 @@ // +build no_ws -package main +package common import "errors" import "net/http" // TODO: Disable WebSockets on high load? Add a Control Panel interface for disabling it? -var enableWebsockets = false // Put this in caps for consistency with the other constants? +var EnableWebsockets = false // Put this in caps for consistency with the other constants? var wsHub WSHub var errWsNouser = errors.New("This user isn't connected via WebSockets") @@ -38,5 +38,5 @@ func (hub *WSHub) pushAlerts(_ []int, _ int, _ string, _ string, _ int, _ int, _ return errWsNouser } -func routeWebsockets(_ http.ResponseWriter, _ *http.Request, _ User) { +func RouteWebsockets(_ http.ResponseWriter, _ *http.Request, _ User) { } diff --git a/common/phrases.go b/common/phrases.go index 558aa5ea..9caa4348 100644 --- a/common/phrases.go +++ b/common/phrases.go @@ -44,12 +44,17 @@ type LanguagePack struct { Accounts map[string]string // TODO: Apply these phrases in the software proper UserAgents map[string]string OperatingSystems map[string]string + HumanLanguages map[string]string Errors map[string]map[string]string // map[category]map[name]value PageTitles map[string]string + TmplPhrases map[string]string + + TmplIndicesToPhrases [][][]byte // [tmplID][index]phrase } // TODO: Add the ability to edit language JSON files from the Control Panel and automatically scan the files for changes -var langPacks sync.Map // nolint it is used +var langPacks sync.Map // nolint it is used +var langTmplIndicesToNames [][]string // [tmplID][index]phraseName func InitPhrases() error { log.Print("Loading the language packs") @@ -75,6 +80,15 @@ func InitPhrases() error { return err } + langPack.TmplIndicesToPhrases = make([][][]byte, len(langTmplIndicesToNames)) + for tmplID, phraseNames := range langTmplIndicesToNames { + var phraseSet = make([][]byte, len(phraseNames)) + for index, phraseName := range phraseNames { + phraseSet[index] = []byte(langPack.TmplPhrases[phraseName]) + } + langPack.TmplIndicesToPhrases[tmplID] = phraseSet + } + log.Print("Adding the '" + langPack.Name + "' language pack") langPacks.Store(langPack.Name, &langPack) langPackCount++ @@ -170,6 +184,14 @@ func GetOSPhrase(name string) (string, bool) { return res, true } +func GetHumanLangPhrase(name string) (string, bool) { + res, ok := currentLangPack.Load().(*LanguagePack).HumanLanguages[name] + if !ok { + return "", false + } + return res, true +} + // TODO: Does comma ok work with multi-dimensional maps? func GetErrorPhrase(category string, name string) string { res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name] @@ -187,6 +209,14 @@ func GetTitlePhrase(name string) string { return res } +func GetTmplPhrase(name string) string { + res, ok := currentLangPack.Load().(*LanguagePack).TmplPhrases[name] + if !ok { + return getPhrasePlaceholder() + } + return res +} + func getPhrasePlaceholder() string { return "{name}" } @@ -213,3 +243,14 @@ func ChangeLanguagePack(name string) (exists bool) { currentLangPack.Store(pack) return true } + +// Template Transpiler Stuff + +func RegisterTmplPhraseNames(phraseNames []string) (tmplID int) { + langTmplIndicesToNames = append(langTmplIndicesToNames, phraseNames) + return len(langTmplIndicesToNames) - 1 +} + +func GetTmplPhrasesBytes(tmplID int) [][]byte { + return currentLangPack.Load().(*LanguagePack).TmplIndicesToPhrases[tmplID] +} diff --git a/common/routes_common.go b/common/routes_common.go index ace95c1f..ab1e93f8 100644 --- a/common/routes_common.go +++ b/common/routes_common.go @@ -217,6 +217,9 @@ func userCheck(w http.ResponseWriter, r *http.Request, user *User) (headerVars * if user.IsBanned { headerVars.NoticeList = append(headerVars.NoticeList, "Your account has been suspended. Some of your permissions may have been revoked.") } + if user.Loggedin && !user.Active { + headerVars.NoticeList = append(headerVars.NoticeList, "Your account hasn't been activated yet. Some features may remain unavailable until it is.") + } if len(theme.Resources) > 0 { rlist := theme.Resources diff --git a/common/template_init.go b/common/template_init.go index 30a98dbe..0d98e59f 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -22,6 +22,7 @@ type CTmpl struct { Imports []string } +// TODO: Stop duplicating these bits of code // nolint func interpreted_topic_template(pi TopicPage, w http.ResponseWriter) error { mapping, ok := Themes[DefaultThemeBox.Load().(string)].TemplatesMap["topic"] @@ -80,18 +81,57 @@ var Template_create_topic_handle func(CreateTopicPage, http.ResponseWriter) erro return Templates.ExecuteTemplate(w, mapping+".html", pi) } +// nolint +var Template_login_handle func(Page, http.ResponseWriter) error = func(pi Page, w http.ResponseWriter) error { + mapping, ok := Themes[DefaultThemeBox.Load().(string)].TemplatesMap["login"] + if !ok { + mapping = "login" + } + return Templates.ExecuteTemplate(w, mapping+".html", pi) +} + +// nolint +var Template_register_handle func(Page, http.ResponseWriter) error = func(pi Page, w http.ResponseWriter) error { + mapping, ok := Themes[DefaultThemeBox.Load().(string)].TemplatesMap["register"] + if !ok { + mapping = "register" + } + return Templates.ExecuteTemplate(w, mapping+".html", pi) +} + +// nolint +var Template_error_handle func(Page, http.ResponseWriter) error = func(pi Page, w http.ResponseWriter) error { + mapping, ok := Themes[DefaultThemeBox.Load().(string)].TemplatesMap["error"] + if !ok { + mapping = "error" + } + return Templates.ExecuteTemplate(w, mapping+".html", pi) +} + +// nolint +var Template_ip_search_handle func(IPSearchPage, http.ResponseWriter) error = func(pi IPSearchPage, w http.ResponseWriter) error { + mapping, ok := Themes[DefaultThemeBox.Load().(string)].TemplatesMap["ip_search"] + if !ok { + mapping = "ip_search" + } + return Templates.ExecuteTemplate(w, mapping+".html", pi) +} + // ? - Add template hooks? func compileTemplates() error { + var config tmpl.CTemplateConfig + config.Minify = Config.MinifyTemplates + config.SuperDebug = Dev.TemplateDebug + var c tmpl.CTemplateSet - c.Minify(Config.MinifyTemplates) - c.SuperDebug(Dev.TemplateDebug) + c.SetConfig(config) // Schemas to train the template compiler on what to expect // TODO: Add support for interface{}s - user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, "", "", "", "", "", 0, 0, "0.0.0.0.0", 0} + user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, BuildAvatar(62, ""), "", "", "", "", 0, 0, "0.0.0.0.0", 0} // TODO: Do a more accurate level calculation for this? - user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", "", "", "", "", 58, 1000, "127.0.0.1", 0} - user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", "", "", "", "", 42, 900, "::1", 0} + user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(1, ""), "", "", "", "", 58, 1000, "127.0.0.1", 0} + user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(2, ""), "", "", "", "", 42, 900, "::1", 0} headerVars := &HeaderVars{ Site: Site, Settings: SettingBox.Load().(SettingMap), @@ -168,12 +208,37 @@ func compileTemplates() error { return err } - // Let plugins register their own templates - if Dev.DebugMode { - log.Print("Registering the templates for the plugins") + loginPage := Page{"Login Page", user, headerVars, tList, nil} + loginTmpl, err := c.Compile("login.html", "templates/", "common.Page", loginPage, varList) + if err != nil { + return err } - c.SkipHandles(true) + registerPage := Page{"Registration Page", user, headerVars, tList, nil} + registerTmpl, err := c.Compile("register.html", "templates/", "common.Page", registerPage, varList) + if err != nil { + return err + } + + errorPage := Page{"Error", user, headerVars, tList, "A problem has occurred in the system."} + errorTmpl, err := c.Compile("error.html", "templates/", "common.Page", errorPage, varList) + if err != nil { + return err + } + + var ipUserList = make(map[int]*User) + ipUserList[1] = &user2 + ipSearchPage := IPSearchPage{"IP Search", user2, headerVars, ipUserList, "::1"} + ipSearchTmpl, err := c.Compile("ip_search.html", "templates/", "common.IPSearchPage", ipSearchPage, varList) + if err != nil { + return err + } + + // Let plugins register their own templates + DebugLog("Registering the templates for the plugins") + config = c.GetConfig() + config.SkipHandles = true + c.SetConfig(config) for _, tmplfunc := range PrebuildTmplList { tmplItem := tmplfunc(user, headerVars) varList = make(map[string]tmpl.VarItem) @@ -191,6 +256,10 @@ func compileTemplates() error { go writeTemplate("forums", forumsTmpl) go writeTemplate("topics", topicsTmpl) go writeTemplate("forum", forumTmpl) + go writeTemplate("login", loginTmpl) + go writeTemplate("register", registerTmpl) + go writeTemplate("ip_search", ipSearchTmpl) + go writeTemplate("error", errorTmpl) go func() { err := writeFile("./template_list.go", "package main\n\n// nolint\n"+c.FragOut) if err != nil { @@ -279,10 +348,16 @@ func InitTemplates() error { return template.HTML(BuildWidget(dock.(string), headerVarInt.(*HeaderVars))) } - // The interpreted templates... - if Dev.DebugMode { - log.Print("Loading the template files...") + fmap["lang"] = func(phraseNameInt interface{}) interface{} { + phraseName, ok := phraseNameInt.(string) + if !ok { + panic("phraseNameInt is not a string") + } + return GetTmplPhrase(phraseName) } + + // The interpreted templates... + DebugLog("Loading the template files...") Templates.Funcs(fmap) template.Must(Templates.ParseGlob("templates/*")) template.Must(Templates.ParseGlob("pages/*")) diff --git a/common/templates/minifiers.go b/common/templates/minifiers.go new file mode 100644 index 00000000..58cbf53e --- /dev/null +++ b/common/templates/minifiers.go @@ -0,0 +1,54 @@ +package tmpl + +import ( + "strconv" + "strings" +) + +// TODO: Write unit tests for this +func minify(data string) string { + data = strings.Replace(data, "\t", "", -1) + data = strings.Replace(data, "\v", "", -1) + data = strings.Replace(data, "\n", "", -1) + data = strings.Replace(data, "\r", "", -1) + data = strings.Replace(data, " ", " ", -1) + return data +} + +// TODO: Strip comments +// TODO: Handle CSS nested in