package panel import ( "database/sql" "errors" "log" "net/http" "strconv" "strings" "time" c "github.com/Azareal/Gosora/common" "github.com/Azareal/Gosora/common/phrases" "github.com/Azareal/Gosora/query_gen" ) // TODO: Move this to another file, probably common/pages.go type AnalyticsTimeRange struct { Quantity int Unit string Slices int SliceWidth int Range string } func analyticsTimeRange(rawTimeRange string) (*AnalyticsTimeRange, error) { timeRange := &AnalyticsTimeRange{} timeRange.Quantity = 6 timeRange.Unit = "hour" timeRange.Slices = 12 timeRange.SliceWidth = 60 * 30 timeRange.Range = "six-hours" switch rawTimeRange { // This might be pushing it, we might want to come up with a more efficient scheme for dealing with large timeframes like this case "one-year": timeRange.Quantity = 12 timeRange.Unit = "month" timeRange.Slices = 12 timeRange.SliceWidth = 60 * 60 * 24 * 30 timeRange.Range = "one-year" case "three-months": timeRange.Quantity = 90 timeRange.Unit = "day" timeRange.Slices = 30 timeRange.SliceWidth = 60 * 60 * 24 * 3 timeRange.Range = "three-months" case "one-month": timeRange.Quantity = 30 timeRange.Unit = "day" timeRange.Slices = 30 timeRange.SliceWidth = 60 * 60 * 24 timeRange.Range = "one-month" case "one-week": timeRange.Quantity = 7 timeRange.Unit = "day" timeRange.Slices = 14 timeRange.SliceWidth = 60 * 60 * 12 timeRange.Range = "one-week" case "two-days": // Two days is experimental timeRange.Quantity = 2 timeRange.Unit = "day" timeRange.Slices = 24 timeRange.SliceWidth = 60 * 60 * 2 timeRange.Range = "two-days" case "one-day": timeRange.Quantity = 1 timeRange.Unit = "day" timeRange.Slices = 24 timeRange.SliceWidth = 60 * 60 timeRange.Range = "one-day" case "twelve-hours": timeRange.Quantity = 12 timeRange.Slices = 24 timeRange.Range = "twelve-hours" case "six-hours", "": default: return timeRange, errors.New("Unknown time range") } return timeRange, nil } func analyticsTimeRangeToLabelList(timeRange *AnalyticsTimeRange) (revLabelList []int64, labelList []int64, viewMap map[int64]int64) { viewMap = make(map[int64]int64) var currentTime = time.Now().Unix() for i := 1; i <= timeRange.Slices; i++ { var label = currentTime - int64(i*timeRange.SliceWidth) revLabelList = append(revLabelList, label) viewMap[label] = 0 } for _, value := range revLabelList { labelList = append(labelList, value) } return revLabelList, labelList, viewMap } func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[int64]int64, error) { defer rows.Close() for rows.Next() { var count int64 var createdAt time.Time err := rows.Scan(&count, &createdAt) if err != nil { return viewMap, err } var unixCreatedAt = createdAt.Unix() // TODO: Bulk log this if c.Dev.SuperDebug { log.Print("count: ", count) log.Print("createdAt: ", createdAt) log.Print("unixCreatedAt: ", unixCreatedAt) } for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count break } } } return viewMap, rows.Err() } type pAvg struct { Avg int64 Tot int64 } func analyticsRowsToAverageMap(rows *sql.Rows, labelList []int64, avgMap map[int64]int64) (map[int64]int64, error) { defer rows.Close() for rows.Next() { var count int64 var createdAt time.Time err := rows.Scan(&count, &createdAt) if err != nil { return avgMap, err } var unixCreatedAt = createdAt.Unix() // TODO: Bulk log this if c.Dev.SuperDebug { log.Print("count: ", count) log.Print("createdAt: ", createdAt) log.Print("unixCreatedAt: ", unixCreatedAt) } var pAvgMap = make(map[int64]pAvg) for _, value := range labelList { if unixCreatedAt > value { prev := pAvgMap[value] prev.Avg += count prev.Tot++ pAvgMap[value] = prev break } } for key, pAvg := range pAvgMap { avgMap[key] = pAvg.Avg / pAvg.Tot } } return avgMap, rows.Err() } func analyticsRowsToAverageMap2(rows *sql.Rows, labelList []int64, avgMap map[int64]int64, typ int) (map[int64]int64, error) { defer rows.Close() for rows.Next() { var stack, heap int64 var createdAt time.Time err := rows.Scan(&stack, &heap, &createdAt) if err != nil { return avgMap, err } var unixCreatedAt = createdAt.Unix() // TODO: Bulk log this if c.Dev.SuperDebug { log.Print("stack: ", stack) log.Print("heap: ", heap) log.Print("createdAt: ", createdAt) log.Print("unixCreatedAt: ", unixCreatedAt) } if typ == 1 { heap = 0 } else if typ == 2 { stack = 0 } var pAvgMap = make(map[int64]pAvg) for _, value := range labelList { if unixCreatedAt > value { prev := pAvgMap[value] prev.Avg += stack + heap prev.Tot++ pAvgMap[value] = prev break } } for key, pAvg := range pAvgMap { avgMap[key] = pAvg.Avg / pAvg.Tot } } return avgMap, rows.Err() } func PreAnalyticsDetail(w http.ResponseWriter, r *http.Request, user *c.User) (*c.BasePanelPage, c.RouteError) { basePage, ferr := buildBasePage(w, r, user, "analytics", "analytics") if ferr != nil { return nil, ferr } basePage.AddSheet("chartist/chartist.min.css") basePage.AddScript("chartist/chartist.min.js") basePage.AddScriptAsync("analytics.js") return basePage, nil } func AnalyticsViews(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsViews") // TODO: Add some sort of analytics store / iterator? rows, err := qgen.NewAcc().Select("viewchunks").Columns("count, createdAt").Where("route = ''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 var viewItems []c.PanelAnalyticsItem for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) var ttime string if timeRange.Range == "six-hours" || timeRange.Range == "twelve-hours" || timeRange.Range == "one-day" { ttime = "time" } pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, ttime} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_views", pi}) } func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user c.User, route string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsRouteViews") // TODO: Validate the route is valid rows, err := qgen.NewAcc().Select("viewchunks").Columns("count, createdAt").Where("route = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(route) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 var viewItems []c.PanelAnalyticsItem for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsRoutePage{basePage, c.SanitiseSingleLine(route), graph, viewItems, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_route_views", pi}) } func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user c.User, agent string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) // ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff agent = c.SanitiseSingleLine(agent) c.DebugLog("in panel.AnalyticsAgentViews") // TODO: Verify the agent is valid rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, createdAt").Where("browser = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(agent) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) friendlyAgent, ok := phrases.GetUserAgentPhrase(agent) if !ok { friendlyAgent = agent } pi := c.PanelAnalyticsAgentPage{basePage, agent, friendlyAgent, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_agent_views", pi}) } func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user c.User, sfid string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) fid, err := strconv.Atoi(sfid) if err != nil { return c.LocalError("Invalid integer", w, r, user) } c.DebugLog("in panel.AnalyticsForumViews") // TODO: Verify the agent is valid rows, err := qgen.NewAcc().Select("viewchunks_forums").Columns("count, createdAt").Where("forum = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(fid) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) forum, err := c.Forums.Get(fid) if err != nil { return c.InternalError(err, w, r) } pi := c.PanelAnalyticsAgentPage{basePage, sfid, forum.Name, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_forum_views", pi}) } func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user c.User, system string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) system = c.SanitiseSingleLine(system) c.DebugLog("in panel.AnalyticsSystemViews") // TODO: Verify the OS name is valid rows, err := qgen.NewAcc().Select("viewchunks_systems").Columns("count, createdAt").Where("system = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) friendlySystem, ok := phrases.GetOSPhrase(system) if !ok { friendlySystem = system } pi := c.PanelAnalyticsAgentPage{basePage, system, friendlySystem, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_system_views", pi}) } func AnalyticsLanguageViews(w http.ResponseWriter, r *http.Request, user c.User, lang string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) lang = c.SanitiseSingleLine(lang) c.DebugLog("in panel.AnalyticsLanguageViews") // TODO: Verify the language code is valid rows, err := qgen.NewAcc().Select("viewchunks_langs").Columns("count, createdAt").Where("lang = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(lang) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) friendlyLang, ok := phrases.GetHumanLangPhrase(lang) if !ok { friendlyLang = lang } pi := c.PanelAnalyticsAgentPage{basePage, lang, friendlyLang, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_lang_views", pi}) } func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user c.User, domain string) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsReferrerViews") // TODO: Verify the agent is valid rows, err := qgen.NewAcc().Select("viewchunks_referrers").Columns("count, createdAt").Where("domain = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(domain) if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsAgentPage{basePage, c.SanitiseSingleLine(domain), "", graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_referrer_views", pi}) } func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsTopics") rows, err := qgen.NewAcc().Select("topicchunks").Columns("count, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 var viewItems []c.PanelAnalyticsItem for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, "time"} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_topics", pi}) } func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsPosts") rows, err := qgen.NewAcc().Select("postchunks").Columns("count, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } var viewList []int64 var viewItems []c.PanelAnalyticsItem for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, "time"} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_posts", pi}) } func AnalyticsMemory(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsMemory") rows, err := qgen.NewAcc().Select("memchunks").Columns("count, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } avgMap, err = analyticsRowsToAverageMap(rows, labelList, avgMap) if err != nil { return c.InternalError(err, w, r) } // TODO: Adjust for the missing chunks in week and month var avgList []int64 var avgItems []c.PanelAnalyticsItemUnit for _, value := range revLabelList { avgList = append(avgList, avgMap[value]) cv, cu := c.ConvertByteUnit(float64(avgMap[value])) avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)}) } graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsStdUnit{graph, avgItems, timeRange.Range, timeRange.Unit, "time"} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_memory", pi}) } // TODO: Show stack and heap memory separately on the chart func AnalyticsActiveMemory(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange) c.DebugLog("in panel.AnalyticsActiveMemory") rows, err := qgen.NewAcc().Select("memchunks").Columns("stack, heap, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } var typ int switch r.FormValue("mtype") { case "1": typ = 1 case "2": typ = 2 default: typ = 0 } avgMap, err = analyticsRowsToAverageMap2(rows, labelList, avgMap, typ) if err != nil { return c.InternalError(err, w, r) } // TODO: Adjust for the missing chunks in week and month var avgList []int64 var avgItems []c.PanelAnalyticsItemUnit for _, value := range revLabelList { avgList = append(avgList, avgMap[value]) cv, cu := c.ConvertByteUnit(float64(avgMap[value])) avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)}) } graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList} c.DebugLogf("graph: %+v\n", graph) pi := c.PanelAnalyticsActiveMemory{graph, avgItems, timeRange.Range, timeRange.Unit, "time", typ} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_active_memory", pi}) } func analyticsRowsToRefMap(rows *sql.Rows) (map[string]int, error) { nameMap := make(map[string]int) defer rows.Close() for rows.Next() { var count int var name string err := rows.Scan(&count, &name) if err != nil { return nameMap, err } // TODO: Bulk log this if c.Dev.SuperDebug { log.Print("count: ", count) log.Print("name: ", name) } nameMap[name] += count } return nameMap, rows.Err() } func analyticsRowsToDuoMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[string]map[int64]int64, map[string]int, error) { vMap := make(map[string]map[int64]int64) nameMap := make(map[string]int) defer rows.Close() for rows.Next() { var count int64 var name string var createdAt time.Time err := rows.Scan(&count, &name, &createdAt) if err != nil { return vMap, nameMap, err } // TODO: Bulk log this var unixCreatedAt = createdAt.Unix() if c.Dev.SuperDebug { log.Print("count: ", count) log.Print("name: ", name) log.Print("createdAt: ", createdAt) log.Print("unixCreatedAt: ", unixCreatedAt) } vvMap, ok := vMap[name] if !ok { vvMap = make(map[int64]int64) for key, val := range viewMap { vvMap[key] = val } vMap[name] = vvMap } for _, value := range labelList { if unixCreatedAt > value { vvMap[value] += count break } } nameMap[name] += int(count) } return vMap, nameMap, rows.Err() } type OVItem struct { name string count int viewMap map[int64]int64 } func analyticsVMapToOVList(vMap map[string]map[int64]int64) (ovList []OVItem) { // Order the map for name, viewMap := range vMap { var totcount int for _, count := range viewMap { totcount += int(count) } ovList = append(ovList, OVItem{name, totcount, viewMap}) } // Use bubble sort for now as there shouldn't be too many items for i := 0; i < len(ovList)-1; i++ { for j := 0; j < len(ovList)-1; j++ { if ovList[j].count > ovList[j+1].count { temp := ovList[j] ovList[j] = ovList[j+1] ovList[j+1] = temp } } } // Invert the direction var tOVList []OVItem for i := len(ovList) - 1; i >= 0; i-- { tOVList = append(tOVList, ovList[i]) } return tOVList } func AnalyticsForums(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } basePage.AddScript("chartist/chartist-plugin-legend.min.js") basePage.AddSheet("chartist/chartist-plugin-legend.css") timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) rows, err := qgen.NewAcc().Select("viewchunks_forums").Columns("count, forum, createdAt").Where("forum != ''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } vMap, forumMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } ovList := analyticsVMapToOVList(vMap) var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) fid, err := strconv.Atoi(ovitem.name) if err != nil { return c.InternalError(err, w, r) } var lName string forum, err := c.Forums.Get(fid) if err == sql.ErrNoRows { // TODO: Localise this lName = "Deleted Forum" } else if err != nil { return c.InternalError(err, w, r) } else { lName = forum.Name } legendList = append(legendList, lName) if i >= 6 { break } i++ } graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} c.DebugLogf("graph: %+v\n", graph) // TODO: Sort this slice var forumItems []c.PanelAnalyticsAgentsItem for sfid, count := range forumMap { fid, err := strconv.Atoi(sfid) if err != nil { return c.InternalError(err, w, r) } var lName string forum, err := c.Forums.Get(fid) if err == sql.ErrNoRows { // TODO: Localise this lName = "Deleted Forum" } else if err != nil { return c.InternalError(err, w, r) } else { lName = forum.Name } forumItems = append(forumItems, c.PanelAnalyticsAgentsItem{ Agent: sfid, FriendlyAgent: lName, Count: count, }) } pi := c.PanelAnalyticsDuoPage{basePage, forumItems, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_forums", pi}) } func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } basePage.AddScript("chartist/chartist-plugin-legend.min.js") basePage.AddSheet("chartist/chartist-plugin-legend.css") timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) rows, err := qgen.NewAcc().Select("viewchunks").Columns("count, route, createdAt").Where("route != ''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } vMap, routeMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } ovList := analyticsVMapToOVList(vMap) ex := strings.Split(r.FormValue("ex"), ",") var inEx = func(name string) bool { for _, e := range ex { if e == name { return true } } return false } var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { if inEx(ovitem.name) { continue } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) legendList = append(legendList, ovitem.name) if i >= 6 { break } i++ } graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} c.DebugLogf("graph: %+v\n", graph) // TODO: Sort this slice var routeItems []c.PanelAnalyticsRoutesItem for route, count := range routeMap { if inEx(route) { continue } routeItems = append(routeItems, c.PanelAnalyticsRoutesItem{ Route: route, Count: count, }) } pi := c.PanelAnalyticsRoutesPage{basePage, routeItems, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_routes", pi}) } // Trialling multi-series charts func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } basePage.AddScript("chartist/chartist-plugin-legend.min.js") basePage.AddSheet("chartist/chartist-plugin-legend.css") timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, browser, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } vMap, agentMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } ovList := analyticsVMapToOVList(vMap) ex := strings.Split(r.FormValue("ex"), ",") var inEx = func(name string) bool { for _, e := range ex { if e == name { return true } } return false } var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { if inEx(ovitem.name) { continue } lName, ok := phrases.GetUserAgentPhrase(ovitem.name) if !ok { lName = ovitem.name } if inEx(lName) { continue } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) legendList = append(legendList, lName) if i >= 6 { break } i++ } graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} c.DebugLogf("graph: %+v\n", graph) // TODO: Sort this slice var agentItems []c.PanelAnalyticsAgentsItem for agent, count := range agentMap { if inEx(agent) { continue } aAgent, ok := phrases.GetUserAgentPhrase(agent) if !ok { aAgent = agent } if inEx(aAgent) { continue } agentItems = append(agentItems, c.PanelAnalyticsAgentsItem{ Agent: agent, FriendlyAgent: aAgent, Count: count, }) } pi := c.PanelAnalyticsDuoPage{basePage, agentItems, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_agents", pi}) } func AnalyticsSystems(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } basePage.AddScript("chartist/chartist-plugin-legend.min.js") basePage.AddSheet("chartist/chartist-plugin-legend.css") timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) rows, err := qgen.NewAcc().Select("viewchunks_systems").Columns("count, system, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } vMap, osMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } ovList := analyticsVMapToOVList(vMap) var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) lName, ok := phrases.GetOSPhrase(ovitem.name) if !ok { lName = ovitem.name } legendList = append(legendList, lName) if i >= 6 { break } i++ } graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} c.DebugLogf("graph: %+v\n", graph) // TODO: Sort this slice var systemItems []c.PanelAnalyticsAgentsItem for system, count := range osMap { sSystem, ok := phrases.GetOSPhrase(system) if !ok { sSystem = system } systemItems = append(systemItems, c.PanelAnalyticsAgentsItem{ Agent: system, FriendlyAgent: sSystem, Count: count, }) } pi := c.PanelAnalyticsDuoPage{basePage, systemItems, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_systems", pi}) } func AnalyticsLanguages(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } basePage.AddScript("chartist/chartist-plugin-legend.min.js") basePage.AddSheet("chartist/chartist-plugin-legend.css") timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) rows, err := qgen.NewAcc().Select("viewchunks_langs").Columns("count, lang, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } vMap, langMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return c.InternalError(err, w, r) } ovList := analyticsVMapToOVList(vMap) ex := strings.Split(r.FormValue("ex"), ",") var inEx = func(name string) bool { for _, e := range ex { if e == name { return true } } return false } var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { if inEx(ovitem.name) { continue } lName, ok := phrases.GetHumanLangPhrase(ovitem.name) if !ok { lName = ovitem.name } if inEx(lName) { continue } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) legendList = append(legendList, lName) if i >= 6 { break } i++ } graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} c.DebugLogf("graph: %+v\n", graph) // TODO: Can we de-duplicate these analytics functions further? // TODO: Sort this slice var langItems []c.PanelAnalyticsAgentsItem for lang, count := range langMap { if inEx(lang) { continue } lLang, ok := phrases.GetHumanLangPhrase(lang) if !ok { lLang = lang } if inEx(lLang) { continue } langItems = append(langItems, c.PanelAnalyticsAgentsItem{ Agent: lang, FriendlyAgent: lLang, Count: count, }) } pi := c.PanelAnalyticsDuoPage{basePage, langItems, graph, timeRange.Range} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_langs", pi}) } func AnalyticsReferrers(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics") if ferr != nil { return ferr } timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return c.LocalError(err.Error(), w, r, user) } rows, err := qgen.NewAcc().Select("viewchunks_referrers").Columns("count, domain").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return c.InternalError(err, w, r) } refMap, err := analyticsRowsToRefMap(rows) if err != nil { return c.InternalError(err, w, r) } showSpam := r.FormValue("spam") == "1" var isSpammy = func(domain string) bool { for _, substr := range c.SpammyDomainBits { if strings.Contains(domain, substr) { return true } } return false } // TODO: Sort this slice var refItems []c.PanelAnalyticsAgentsItem for domain, count := range refMap { sdomain := c.SanitiseSingleLine(domain) if !showSpam && isSpammy(sdomain) { continue } refItems = append(refItems, c.PanelAnalyticsAgentsItem{ Agent: sdomain, Count: count, }) } pi := c.PanelAnalyticsReferrersPage{basePage, refItems, timeRange.Range, showSpam} return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_referrers", pi}) }