experimental route perf pane

add 1.14 to travis test list
more qutils tests

add panel_menu_stats_routes_perf phrase

.travis.yml

@@ -4,6 +4,7 @@ env:
 language: go
 go:
   - "1.13"
+  - "1.14"
   - master
 before_install:
   - cd $HOME

common/pages.go

@@ -472,6 +472,19 @@ type PanelAnalyticsRoutesPage struct {
 	TimeRange string
 }
 
+type PanelAnalyticsRoutesPerfItem struct {
+	Route string
+	Count int
+	Unit string
+}
+
+type PanelAnalyticsRoutesPerfPage struct {
+	*BasePanelPage
+	ItemList  []PanelAnalyticsRoutesPerfItem
+	Graph     PanelTimeGraph
+	TimeRange string
+}
+
 // TODO: Rename the fields as this structure is being used in a generic way now
 type PanelAnalyticsAgentsItem struct {
 	Agent         string

common/routes_common.go

@@ -98,9 +98,9 @@ func cascadeForumPerms(fp *ForumPerms, u *User) {
 
 // Even if they have the right permissions, the control panel is only open to supermods+. There are many areas without subpermissions which assume that the current user is a supermod+ and admins are extremely unlikely to give these permissions to someone who isn't at-least a supermod to begin with
 // TODO: Do a panel specific theme?
-func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (header *Header, stats PanelStats, rerr RouteError) {
+func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (h *Header, stats PanelStats, rerr RouteError) {
 	theme := GetThemeByReq(r)
-	header = &Header{
+	h = &Header{
 		Site:        Site,
 		Settings:    SettingBox.Load().(SettingMap),
 		Themes:      Themes,
@@ -110,15 +110,16 @@ func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (header
 		Zone:        "panel",
 		Writer:      w,
 		IsoCode:     phrases.GetLangPack().IsoCode,
+		StartedAt:   time.Now(),
 	}
 	// TODO: We should probably initialise header.ExtData
 	// ? - Should we only show this in debug mode? It might be useful for detecting issues in production, if we show it there as-well
 	//if user.IsAdmin {
-		header.StartedAt = time.Now()
+	//h.StartedAt = time.Now()
 	//}
 
-	header.AddSheet(theme.Name + "/main.css")
-	header.AddSheet(theme.Name + "/panel.css")
+	h.AddSheet(theme.Name + "/main.css")
+	h.AddSheet(theme.Name + "/panel.css")
 	if len(theme.Resources) > 0 {
 		rlist := theme.Resources
 		for _, res := range rlist {
@@ -126,12 +127,12 @@ func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (header
 				extarr := strings.Split(res.Name, ".")
 				ext := extarr[len(extarr)-1]
 				if ext == "css" {
-					header.AddSheet(res.Name)
+					h.AddSheet(res.Name)
 				} else if ext == "js" {
 					if res.Async {
-						header.AddScriptAsync(res.Name)
+						h.AddScriptAsync(res.Name)
 					} else {
-						header.AddScript(res.Name)
+						h.AddScript(res.Name)
 					}
 				}
 			}
@@ -146,7 +147,7 @@ func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (header
 	stats.Groups = Groups.Count()
 	stats.Forums = Forums.Count()
 	stats.Pages = Pages.Count()
-	stats.Settings = len(header.Settings)
+	stats.Settings = len(h.Settings)
 	stats.WordFilters = WordFilters.EstCount()
 	stats.Themes = len(Themes)
 	stats.Reports = 0 // TODO: Do the report count. Only show open threads?
@@ -160,12 +161,12 @@ func panelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (header
 				tname = "_" + theme.Name
 			}
 		}
-		header.AddPreScriptAsync("template_" + name + tname + ".js")
+		h.AddPreScriptAsync("template_" + name + tname + ".js")
 	}
 	addPreScript("alert")
 	addPreScript("notice")
 
-	return header, stats, nil
+	return h, stats, nil
 }
 
 func simplePanelUserCheck(w http.ResponseWriter, r *http.Request, user *User) (headerLite *HeaderLite, rerr RouteError) {
@@ -225,7 +226,7 @@ func userCheck(w http.ResponseWriter, r *http.Request, user *User) (header *Head
 	// An optimisation so we don't populate StartedAt for users who shouldn't see the stat anyway
 	// ? - Should we only show this in debug mode? It might be useful for detecting issues in production, if we show it there as-well
 	//if user.IsAdmin {
-		header.StartedAt = time.Now()
+	header.StartedAt = time.Now()
 	//}
 
 	//PrepResources(user,header,theme)

langs/english.json

@@ -800,6 +800,7 @@
 		"panel_menu_stats_topics":"Topics",
 		"panel_menu_stats_forums":"Forums",
 		"panel_menu_stats_routes":"Routes",
+		"panel_menu_stats_routes_perf":"Routes Perf",
 		"panel_menu_stats_agents":"Agents",
 		"panel_menu_stats_systems":"Systems",
 		"panel_menu_stats_languages":"Languages",

query_gen/utils_test.go

@@ -46,6 +46,8 @@ func TestProcessWhere(t *testing.T) {
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
 	whs = processWhere("uid=20")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "20"})
+	whs = processWhere("uid=uid+1")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenColumn, "uid"}, MT{TokenOp, "+"}, MT{TokenNumber, "1"})
 	whs = processWhere("uid='1'")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
 	whs = processWhere("uid='t'")

router_gen/routes.go

@@ -243,6 +243,7 @@ func panelRoutes() *RouteGroup {
 
 		View("panel.AnalyticsViews", "/panel/analytics/views/").Before("ParseForm"),
 		View("panel.AnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"),
+		View("panel.AnalyticsRoutesPerf", "/panel/analytics/routes-perf/").Before("ParseForm"),
 		View("panel.AnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"),
 		View("panel.AnalyticsSystems", "/panel/analytics/systems/").Before("ParseForm"),
 		View("panel.AnalyticsLanguages", "/panel/analytics/langs/").Before("ParseForm"),

routes/panel/analytics.go

@@ -81,6 +81,7 @@ func analyticsTimeRange(rawTimeRange string) (*AnalyticsTimeRange, error) {
 	return tRange, nil
 }
 
+// TODO: Clamp it rather than using an offset off the current time to avoid chaotic changes in stats as adjacent sets converge and diverge?
 func analyticsTimeRangeToLabelList(timeRange *AnalyticsTimeRange) (revLabelList []int64, labelList []int64, viewMap map[int64]int64) {
 	viewMap = make(map[int64]int64)
 	currentTime := time.Now().Unix()
@@ -715,6 +716,171 @@ func AnalyticsPerf(w http.ResponseWriter, r *http.Request, user c.User) c.RouteE
 	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_performance", pi})
 }
 
+func analyticsRowsToAvgDuoMap(rows *sql.Rows, labelList []int64, avgMap map[int64]int64) (map[string]map[int64]int64, map[string]int, error) {
+	aMap := make(map[string]map[int64]int64)
+	nameMap := make(map[string]int)
+	defer rows.Close()
+	for rows.Next() {
+		var count int64
+		var name string
+		var createdAt time.Time
+		err := rows.Scan(&count, &name, &createdAt)
+		if err != nil {
+			return aMap, nameMap, err
+		}
+
+		// TODO: Bulk log this
+		unixCreatedAt := createdAt.Unix()
+		if c.Dev.SuperDebug {
+			log.Print("count: ", count)
+			log.Print("name: ", name)
+			log.Print("createdAt: ", createdAt)
+			log.Print("unixCreatedAt: ", unixCreatedAt)
+		}
+
+		vvMap, ok := aMap[name]
+		if !ok {
+			vvMap = make(map[int64]int64)
+			for key, val := range avgMap {
+				vvMap[key] = val
+			}
+			aMap[name] = vvMap
+		}
+		for _, value := range labelList {
+			if unixCreatedAt > value {
+				vvMap[value] = (vvMap[value] + count) / 2
+				break
+			}
+		}
+		nameMap[name] = (nameMap[name] + int(count)) / 2
+	}
+	return aMap, nameMap, rows.Err()
+}
+
+func sortOVList(ovList []OVItem) (tOVList []OVItem) {
+	// Use bubble sort for now as there shouldn't be too many items
+	for i := 0; i < len(ovList)-1; i++ {
+		for j := 0; j < len(ovList)-1; j++ {
+			if ovList[j].count > ovList[j+1].count {
+				temp := ovList[j]
+				ovList[j] = ovList[j+1]
+				ovList[j+1] = temp
+			}
+		}
+	}
+
+	// Invert the direction
+	for i := len(ovList) - 1; i >= 0; i-- {
+		tOVList = append(tOVList, ovList[i])
+	}
+	return tOVList
+}
+
+func analyticsAMapToOVList(aMap map[string]map[int64]int64) (ovList []OVItem) {
+	// Order the map
+	for name, avgMap := range aMap {
+		var totcount int
+		for _, count := range avgMap {
+			totcount = (totcount + int(count)) / 2
+		}
+		ovList = append(ovList, OVItem{name, totcount, avgMap})
+	}
+
+	return sortOVList(ovList)
+}
+
+func AnalyticsRoutesPerf(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
+	basePage, ferr := PreAnalyticsDetail(w, r, &user)
+	if ferr != nil {
+		return ferr
+	}
+	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
+	basePage.AddSheet("chartist/chartist-plugin-legend.css")
+
+	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
+	if err != nil {
+		return c.LocalError(err.Error(), w, r, user)
+	}
+	// avgMap contains timestamps but not the averages for those stamps
+	revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange)
+
+	rows, err := qgen.NewAcc().Select("viewchunks").Columns("avg,route,createdAt").Where("count!=0 AND route!=''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
+	if err != nil && err != sql.ErrNoRows {
+		return c.InternalError(err, w, r)
+	}
+	aMap, routeMap, err := analyticsRowsToAvgDuoMap(rows, labelList, avgMap)
+	if err != nil {
+		return c.InternalError(err, w, r)
+	}
+	//c.DebugLogf("aMap: %+v\n", aMap)
+	//c.DebugLogf("routeMap: %+v\n", routeMap)
+	ovList := analyticsAMapToOVList(aMap)
+	//c.DebugLogf("ovList: %+v\n", ovList)
+
+	ex := strings.Split(r.FormValue("ex"), ",")
+	inEx := func(name string) bool {
+		for _, e := range ex {
+			if e == name {
+				return true
+			}
+		}
+		return false
+	}
+
+	/*
+	// TODO: Adjust for the missing chunks in week and month
+	var avgList []int64
+	var avgItems []c.PanelAnalyticsItemUnit
+	for _, value := range revLabelList {
+		avgList = append(avgList, avgMap[value])
+		cv, cu := c.ConvertPerfUnit(float64(avgMap[value]))
+		avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)})
+	}
+	graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList}
+	c.DebugLogf("graph: %+v\n", graph)
+	pi := c.PanelAnalyticsPerf{graph, avgItems, timeRange.Range, timeRange.Unit, "time", typ}
+	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_performance", pi})
+	*/
+
+	var vList [][]int64
+	var legendList []string
+	var i int
+	for _, ovitem := range ovList {
+		if inEx(ovitem.name) {
+			continue
+		}
+		var viewList []int64
+		for _, value := range revLabelList {
+			viewList = append(viewList, ovitem.viewMap[value])
+		}
+		vList = append(vList, viewList)
+		legendList = append(legendList, ovitem.name)
+		if i >= 6 {
+			break
+		}
+		i++
+	}
+	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
+	c.DebugLogf("graph: %+v\n", graph)
+
+	// TODO: Sort this slice
+	var routeItems []c.PanelAnalyticsRoutesPerfItem
+	for route, count := range routeMap {
+		if inEx(route) {
+			continue
+		}
+		cv, cu := c.ConvertPerfUnit(float64(count))
+		routeItems = append(routeItems, c.PanelAnalyticsRoutesPerfItem{
+			Route: route,
+			Unit: cu,
+			Count: int(cv),
+		})
+	}
+
+	pi := c.PanelAnalyticsRoutesPerfPage{basePage, routeItems, graph, timeRange.Range}
+	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_routes_perf", pi})
+}
+
 func analyticsRowsToRefMap(rows *sql.Rows) (map[string]int, error) {
 	nameMap := make(map[string]int)
 	defer rows.Close()
@@ -792,23 +958,7 @@ func analyticsVMapToOVList(vMap map[string]map[int64]int64) (ovList []OVItem) {
 		ovList = append(ovList, OVItem{name, totcount, viewMap})
 	}
 
-	// Use bubble sort for now as there shouldn't be too many items
-	for i := 0; i < len(ovList)-1; i++ {
-		for j := 0; j < len(ovList)-1; j++ {
-			if ovList[j].count > ovList[j+1].count {
-				temp := ovList[j]
-				ovList[j] = ovList[j+1]
-				ovList[j+1] = temp
-			}
-		}
-	}
-
-	// Invert the direction
-	var tOVList []OVItem
-	for i := len(ovList) - 1; i >= 0; i-- {
-		tOVList = append(tOVList, ovList[i])
-	}
-	return tOVList
+	return sortOVList(ovList)
 }
 
 func AnalyticsForums(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
@@ -917,7 +1067,10 @@ func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user c.User) c.Rout
 	if err != nil {
 		return c.InternalError(err, w, r)
 	}
+	//c.DebugLogf("vMap: %+v\n", vMap)
+	//c.DebugLogf("routeMap: %+v\n", routeMap)
 	ovList := analyticsVMapToOVList(vMap)
+	//c.DebugLogf("ovList: %+v\n", ovList)
 
 	ex := strings.Split(r.FormValue("ex"), ",")
 	inEx := func(name string) bool {

templates/panel_analytics_routes_perf.html

@@ -0,0 +1,19 @@
+<div class="colstack_item colstack_head">
+	<div class="rowitem">
+		<h1>{{lang "panel_stats_routes_head"}}</h1>
+		{{template "panel_analytics_time_range.html" . }}
+	</div>
+</div>
+<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/routes-perf/" method="get"></form>
+<div id="panel_analytics_routes_chart" class="colstack_graph_holder">
+	<div class="ct_chart"></div>
+</div>
+<div id="panel_analytics_routes" class="colstack_item rowlist">
+	{{range .ItemList}}
+	<div class="rowitem panel_compactrow editable_parent">
+		<a href="/panel/analytics/route/{{.Route}}" class="panel_upshift">{{.Route}}</a>
+		<span class="panel_compacttext to_right">{{.Count}}{{.Unit}}</span>
+	</div>
+	{{else}}<div class="rowitem passive rowmsg">{{lang "panel_stats_routes_no_routes"}}</div>{{end}}
+</div>
+{{template "panel_analytics_script_perf.html" . }}

templates/panel_inner_menu.html

@@ -50,6 +50,9 @@
 		<div class="rowitem passive submenu">
 			<a href="/panel/analytics/routes/">{{lang "panel_menu_stats_routes"}}</a>
 		</div>
+		<div class="rowitem passive submenu">
+			<a href="/panel/analytics/routes-perf/">{{lang "panel_menu_stats_routes_perf"}}</a>
+		</div>
 		<div class="rowitem passive submenu">
 			<a href="/panel/analytics/agents/">{{lang "panel_menu_stats_agents"}}</a>
 		</div>