From a8e1076f7cb3505fa25d1e8f3d1aa9d235f46955 Mon Sep 17 00:00:00 2001 From: Azareal Date: Thu, 2 May 2019 09:14:07 +1000 Subject: [PATCH] Do proper averaging and collect more samples for the memory analytics pane. Added an ETag for the alerts endpoint when you're not logged to save bandwidth. The Page Manager now uses dyntmpl. The Setting Manager now uses dyntmpl. The Word Filter Manager now uses dyntmpl. Fixed the padding on the noavatar alerts for Nox. Tweaked the panel_word_filters_to phrase. Tweaked the panel_statistics_memory_head phrase. Added the panel_statistics_memory_chart_aria phrase. Added the panel_statistics_memory_table_aria phrase. Added the panel_statistics_memory_no_memory phrase. --- common/counters/memory.go | 31 +++++++++++- common/tasks.go | 2 +- langs/english.json | 7 ++- routes.go | 14 +++++ routes/panel/analytics.go | 73 +++++++++++++++++---------- routes/panel/pages.go | 4 +- routes/panel/settings.go | 4 +- routes/panel/word_filters.go | 4 +- templates/panel_analytics_memory.html | 10 ++-- templates/panel_pages.html | 10 +--- templates/panel_pages_edit.html | 11 +--- templates/panel_setting.html | 12 +---- templates/panel_settings.html | 10 +--- templates/panel_word_filters.html | 12 +---- themes/nox/public/main.css | 9 ++++ 15 files changed, 121 insertions(+), 92 deletions(-) diff --git a/common/counters/memory.go b/common/counters/memory.go index b35319c7..890ba646 100644 --- a/common/counters/memory.go +++ b/common/counters/memory.go @@ -1,6 +1,8 @@ package counters import ( + "time" + "sync" "runtime" "database/sql" @@ -12,6 +14,9 @@ var MemoryCounter *DefaultMemoryCounter type DefaultMemoryCounter struct { insert *sql.Stmt + totMem uint64 + totCount uint64 + sync.Mutex } func NewMemoryCounter(acc *qgen.Accumulator) (*DefaultMemoryCounter, error) { @@ -21,13 +26,35 @@ func NewMemoryCounter(acc *qgen.Accumulator) (*DefaultMemoryCounter, error) { c.AddScheduledFifteenMinuteTask(co.Tick) //c.AddScheduledSecondTask(co.Tick) c.AddShutdownTask(co.Tick) + ticker := time.NewTicker(time.Minute) + go func() { + for { + select { + case <-ticker.C: + var m runtime.MemStats + runtime.ReadMemStats(&m) + co.Lock() + co.totCount++ + co.totMem += m.Sys + co.Unlock() + } + } + }() return co, acc.FirstError() } func (co *DefaultMemoryCounter) Tick() (err error) { var m runtime.MemStats runtime.ReadMemStats(&m) - c.DebugLogf("Inserting a memchunk with a value of %d", m.Sys) - _, err = co.insert.Exec(m.Sys) + var avgMem uint64 + co.Lock() + co.totCount++ + co.totMem += m.Sys + avgMem = co.totMem / co.totCount + co.totMem = 0 + co.totCount = 0 + co.Unlock() + c.DebugLogf("Inserting a memchunk with a value of %d", avgMem) + _, err = co.insert.Exec(avgMem) return err } \ No newline at end of file diff --git a/common/tasks.go b/common/tasks.go index 5416a60e..b57c4b5e 100644 --- a/common/tasks.go +++ b/common/tasks.go @@ -1,7 +1,7 @@ /* * * Gosora Task System -* Copyright Azareal 2017 - 2019 +* Copyright Azareal 2017 - 2020 * */ package common diff --git a/langs/english.json b/langs/english.json index f0be358a..f3a4408b 100644 --- a/langs/english.json +++ b/langs/english.json @@ -844,7 +844,7 @@ "panel_group_extended_permissions":"Extended Permissions", "panel_word_filters_head":"Word Filters", - "panel_word_filters_to":"...to...", + "panel_word_filters_to":"to", "panel_word_filters_edit_button_aria":"Edit Word Filter", "panel_word_filters_update_button":"Update", "panel_word_filters_delete_button_aria":"Delete Word Filter", @@ -883,7 +883,7 @@ "panel_statistics_operating_systems_head":"Operating Systems", "panel_statistics_topic_counts_head":"Topic Counts", "panel_statistics_requests_head":"Requests", - "panel_statistics_memory_head":"Memory", + "panel_statistics_memory_head":"Memory Usage", "panel_statistics_time_range_one_year":"1 year", "panel_statistics_time_range_three_months":"3 months", @@ -897,11 +897,13 @@ "panel_statistics_post_counts_chart_aria":"Post Chart", "panel_statistics_topic_counts_chart_aria":"Topic Chart", "panel_statistics_requests_chart_aria":"Requests Chart", + "panel_statistics_memory_chart_aria":"Memory Use Chart", "panel_statistics_details_head":"Details", "panel_statistics_post_counts_table_aria":"Post Table, this has the same information as the post chart", "panel_statistics_topic_counts_table_aria":"Topic Table, this has the same information as the topic chart", "panel_statistics_route_views_table_aria":"View Table, this has the same information as the view chart", "panel_statistics_requests_table_aria":"View Table, this has the same information as the view chart", + "panel_statistics_memory_table_aria":"Memory Use Table, this has the same information as the memory use chart", "panel_statistics_views_suffix":" views", "panel_statistics_posts_suffix":" posts", "panel_statistics_topics_suffix":" topics", @@ -913,6 +915,7 @@ "panel_statistics_referrers_no_referrers":"No referrers could be found in the selected time range", "panel_statistics_routes_no_routes":"No route view counts could be found in the selected time range", "panel_statistics_operating_systems_no_operating_systems":"No operating systems could be found in the selected time range", + "panel_statistics_memory_no_memory":"No memory chunks could be found in the selected time range", "panel_logs_menu_head":"Logs", "panel_logs_registration_head":"Registrations", diff --git a/routes.go b/routes.go index 56656891..fb5078df 100644 --- a/routes.go +++ b/routes.go @@ -67,6 +67,20 @@ func routeAPI(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError // TODO: Split this into it's own function case "alerts": // A feed of events tailored for a specific user if !user.Loggedin { + var etag string + _, ok := w.(c.GzipResponseWriter) + if ok { + etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"-ng\"" + } else { + etag = "\""+strconv.FormatInt(c.StartTime.Unix(), 10)+"-n\"" + } + w.Header().Set("ETag", etag) + if match := r.Header.Get("If-None-Match"); match != "" { + if strings.Contains(match, etag) { + w.WriteHeader(http.StatusNotModified) + return nil + } + } w.Write(phraseLoginAlerts) return nil } diff --git a/routes/panel/analytics.go b/routes/panel/analytics.go index b3d9d712..26a1c58a 100644 --- a/routes/panel/analytics.go +++ b/routes/panel/analytics.go @@ -119,6 +119,44 @@ func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64 return viewMap, rows.Err() } +type pAvg struct { + Avg int64 + Tot int64 +} + +func analyticsRowsToAverageMap(rows *sql.Rows, labelList []int64, avgMap map[int64]int64) (map[int64]int64, error) { + defer rows.Close() + for rows.Next() { + var count int64 + var createdAt time.Time + err := rows.Scan(&count, &createdAt) + if err != nil { + return avgMap, err + } + var unixCreatedAt = createdAt.Unix() + // TODO: Bulk log this + if c.Dev.SuperDebug { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + var pAvgMap = make(map[int64]pAvg) + for _, value := range labelList { + if unixCreatedAt > value { + prev := pAvgMap[value] + prev.Avg += count + prev.Tot++ + pAvgMap[value] = prev + break + } + } + for key, pAvg := range pAvgMap { + avgMap[key] = pAvg.Avg / pAvg.Tot + } + } + return avgMap, rows.Err() +} + func PreAnalyticsDetail(w http.ResponseWriter, r *http.Request, user *c.User) (*c.BasePanelPage, c.RouteError) { basePage, ferr := buildBasePage(w, r, user, "analytics", "analytics") if ferr != nil { @@ -473,46 +511,29 @@ func AnalyticsMemory(w http.ResponseWriter, r *http.Request, user c.User) c.Rout if err != nil { return c.LocalError(err.Error(), w, r, user) } - revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) + revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsMemory") rows, err := qgen.NewAcc().Select("memchunks").Columns("count, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) + avgMap, err = analyticsRowsToAverageMap(rows, labelList, avgMap) if err != nil { return c.InternalError(err, w, r) } - var divBy int64 = 1 - switch timeRange.Range { - case "one-year": - divBy = 2 * 30 * 12 - case "three-months": - divBy = 2 * 30 * 3 - case "one-month": - divBy = 2 * 30 - case "one-week": - divBy = 1 * 7 - case "two-days": - divBy = 4 - case "one-day": - divBy = 2 - } - // TODO: Adjust for the missing chunks in week and month - var viewList []int64 - var viewItems []c.PanelAnalyticsItemUnit + var avgList []int64 + var avgItems []c.PanelAnalyticsItemUnit for _, value := range revLabelList { - viewMap[value] = viewMap[value] / divBy - viewList = append(viewList, viewMap[value]) - cv, cu := c.ConvertByteUnit(float64(viewMap[value])) - viewItems = append(viewItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)}) + avgList = append(avgList, avgMap[value]) + cv, cu := c.ConvertByteUnit(float64(avgMap[value])) + avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)}) } - graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} + graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) - pi := c.PanelAnalyticsStdUnit{graph, viewItems, timeRange.Range, timeRange.Unit, "time"} + pi := c.PanelAnalyticsStdUnit{graph, avgItems, timeRange.Range, timeRange.Unit, "time"} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right","analytics","panel_analytics_memory", pi}) } diff --git a/routes/panel/pages.go b/routes/panel/pages.go index 70ea64d9..54324f2c 100644 --- a/routes/panel/pages.go +++ b/routes/panel/pages.go @@ -33,7 +33,7 @@ func Pages(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { pageList := c.Paginate(pageCount, perPage, 5) pi := c.PanelCustomPagesPage{basePage, cPages, c.Paginator{pageList, page, lastPage}} - return renderTemplate("panel_pages", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"panel_page_list","","panel_pages",&pi}) } func PagesCreateSubmit(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { @@ -90,7 +90,7 @@ func PagesEdit(w http.ResponseWriter, r *http.Request, user c.User, spid string) } pi := c.PanelCustomPageEditPage{basePage, page} - return renderTemplate("panel_pages_edit", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"panel_page_edit","","panel_pages_edit",&pi}) } func PagesEditSubmit(w http.ResponseWriter, r *http.Request, user c.User, spid string) c.RouteError { diff --git a/routes/panel/settings.go b/routes/panel/settings.go index f31bfaad..fc90a635 100644 --- a/routes/panel/settings.go +++ b/routes/panel/settings.go @@ -50,7 +50,7 @@ func Settings(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError } pi := c.PanelPage{basePage, tList, settingList} - return renderTemplate("panel_settings", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"","","panel_settings",&pi}) } func SettingEdit(w http.ResponseWriter, r *http.Request, user c.User, sname string) c.RouteError { @@ -90,7 +90,7 @@ func SettingEdit(w http.ResponseWriter, r *http.Request, user c.User, sname stri pSetting := &c.PanelSetting{setting, phrases.GetSettingPhrase(setting.Name)} pi := c.PanelSettingPage{basePage, itemList, pSetting} - return renderTemplate("panel_setting", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"","","panel_setting",&pi}) } func SettingEditSubmit(w http.ResponseWriter, r *http.Request, user c.User, sname string) c.RouteError { diff --git a/routes/panel/word_filters.go b/routes/panel/word_filters.go index 733b5efa..6c11f556 100644 --- a/routes/panel/word_filters.go +++ b/routes/panel/word_filters.go @@ -25,7 +25,7 @@ func WordFilters(w http.ResponseWriter, r *http.Request, user c.User) c.RouteErr } pi := c.PanelPage{basePage, tList, filterList} - return renderTemplate("panel_word_filters", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"","","panel_word_filters",&pi}) } func WordFiltersCreateSubmit(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { @@ -67,7 +67,7 @@ func WordFiltersEdit(w http.ResponseWriter, r *http.Request, user c.User, wfid s _ = wfid pi := c.PanelPage{basePage, tList, nil} - return renderTemplate("panel_word_filters_edit", w, r, basePage.Header, &pi) + return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage,"","","panel_word_filters_edit",&pi}) } func WordFiltersEditSubmit(w http.ResponseWriter, r *http.Request, user c.User, wfid string) c.RouteError { diff --git a/templates/panel_analytics_memory.html b/templates/panel_analytics_memory.html index c7d6e8d0..6bb709b4 100644 --- a/templates/panel_analytics_memory.html +++ b/templates/panel_analytics_memory.html @@ -13,20 +13,20 @@
-
+

{{lang "panel_statistics_details_head"}}

-
+
{{range .ViewItems}}
- {{.Time}} + {{.Time}} {{.Count}}{{.Unit}}
- {{else}}
{{lang "panel_statistics_post_counts_no_post_counts"}}
{{end}} + {{else}}
{{lang "panel_statistics_memory_no_memory"}}
{{end}}