We now track global user agent stats and have a currently simple Control Panel interface for that.

Fixed a possible out of bounds panic in DefaultRouteViewCounter.Bump()
This commit is contained in:
Azareal 2018-01-09 07:39:29 +00:00
parent c7aec90612
commit a25ee29197
16 changed files with 427 additions and 135 deletions

View File

@ -119,3 +119,14 @@ func SetRouteMapEnum(rme map[string]int) {
func SetReverseRouteMapEnum(rrme map[int]string) { func SetReverseRouteMapEnum(rrme map[int]string) {
reverseRouteMapEnum = rrme reverseRouteMapEnum = rrme
} }
var agentMapEnum map[string]int
var reverseAgentMapEnum map[int]string
func SetAgentMapEnum(ame map[string]int) {
agentMapEnum = ame
}
func SetReverseAgentMapEnum(rame map[int]string) {
reverseAgentMapEnum = rame
}

View File

@ -9,6 +9,7 @@ import (
) )
var GlobalViewCounter *ChunkedViewCounter var GlobalViewCounter *ChunkedViewCounter
var AgentViewCounter *DefaultAgentViewCounter
var RouteViewCounter *DefaultRouteViewCounter var RouteViewCounter *DefaultRouteViewCounter
var TopicViewCounter *DefaultTopicViewCounter var TopicViewCounter *DefaultTopicViewCounter
@ -64,7 +65,64 @@ type RWMutexCounterBucket struct {
sync.RWMutex sync.RWMutex
} }
// The name of the struct clashes with the name of the variable, so we're adding Impl to the end type DefaultAgentViewCounter struct {
agentBuckets []*RWMutexCounterBucket //[AgentID]count
insert *sql.Stmt
}
func NewDefaultAgentViewCounter() (*DefaultAgentViewCounter, error) {
acc := qgen.Builder.Accumulator()
var agentBuckets = make([]*RWMutexCounterBucket, len(agentMapEnum))
for bucketID, _ := range agentBuckets {
agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
counter := &DefaultAgentViewCounter{
agentBuckets: agentBuckets,
insert: acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
AddScheduledFifteenMinuteTask(counter.Tick)
//AddScheduledSecondTask(counter.Tick)
AddShutdownTask(counter.Tick)
return counter, acc.FirstError()
}
func (counter *DefaultAgentViewCounter) Tick() error {
for agentID, agentBucket := range counter.agentBuckets {
var count int
agentBucket.RLock()
count = agentBucket.counter
agentBucket.counter = 0
agentBucket.RUnlock()
err := counter.insertChunk(count, agentID) // TODO: Bulk insert for speed?
if err != nil {
return err
}
}
return nil
}
func (counter *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
if count == 0 {
return nil
}
var agentName = reverseAgentMapEnum[agent]
debugLogf("Inserting a viewchunk with a count of %d for agent %s (%d)", count, agentName, agent)
_, err := counter.insert.Exec(count, agentName)
return err
}
func (counter *DefaultAgentViewCounter) Bump(agent int) {
// TODO: Test this check
debugLog("counter.agentBuckets[", agent, "]: ", counter.agentBuckets[agent])
if len(counter.agentBuckets) <= agent || agent < 0 {
return
}
counter.agentBuckets[agent].Lock()
counter.agentBuckets[agent].counter++
counter.agentBuckets[agent].Unlock()
}
type DefaultRouteViewCounter struct { type DefaultRouteViewCounter struct {
routeBuckets []*RWMutexCounterBucket //[RouteID]count routeBuckets []*RWMutexCounterBucket //[RouteID]count
insert *sql.Stmt insert *sql.Stmt
@ -115,7 +173,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error
func (counter *DefaultRouteViewCounter) Bump(route int) { func (counter *DefaultRouteViewCounter) Bump(route int) {
// TODO: Test this check // TODO: Test this check
debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route])
if len(counter.routeBuckets) <= route { if len(counter.routeBuckets) <= route || route < 0 {
return return
} }
counter.routeBuckets[route].Lock() counter.routeBuckets[route].Lock()

View File

@ -98,6 +98,7 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User
"pre_render_panel_edit_forum": nil, "pre_render_panel_edit_forum": nil,
"pre_render_panel_analytics": nil, "pre_render_panel_analytics": nil,
"pre_render_panel_analytics_routes": nil, "pre_render_panel_analytics_routes": nil,
"pre_render_panel_analytics_agents": nil,
"pre_render_panel_analytics_route_views": nil, "pre_render_panel_analytics_route_views": nil,
"pre_render_panel_settings": nil, "pre_render_panel_settings": nil,
"pre_render_panel_setting": nil, "pre_render_panel_setting": nil,

View File

@ -184,6 +184,20 @@ type PanelAnalyticsRoutesPage struct {
ItemList []PanelAnalyticsRoutesItem ItemList []PanelAnalyticsRoutesItem
} }
type PanelAnalyticsAgentsItem struct {
Agent string
Count int
}
type PanelAnalyticsAgentsPage struct {
Title string
CurrentUser User
Header *HeaderVars
Stats PanelStats
Zone string
ItemList []PanelAnalyticsAgentsItem
}
type PanelAnalyticsRoutePage struct { type PanelAnalyticsRoutePage struct {
Title string Title string
CurrentUser User CurrentUser User

View File

@ -52,6 +52,7 @@ var RouteMap = map[string]interface{}{
"routePanelUsersEditSubmit": routePanelUsersEditSubmit, "routePanelUsersEditSubmit": routePanelUsersEditSubmit,
"routePanelAnalyticsViews": routePanelAnalyticsViews, "routePanelAnalyticsViews": routePanelAnalyticsViews,
"routePanelAnalyticsRoutes": routePanelAnalyticsRoutes, "routePanelAnalyticsRoutes": routePanelAnalyticsRoutes,
"routePanelAnalyticsAgents": routePanelAnalyticsAgents,
"routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews, "routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews,
"routePanelGroups": routePanelGroups, "routePanelGroups": routePanelGroups,
"routePanelGroupsEdit": routePanelGroupsEdit, "routePanelGroupsEdit": routePanelGroupsEdit,
@ -119,32 +120,33 @@ var routeMapEnum = map[string]int{
"routePanelUsersEditSubmit": 34, "routePanelUsersEditSubmit": 34,
"routePanelAnalyticsViews": 35, "routePanelAnalyticsViews": 35,
"routePanelAnalyticsRoutes": 36, "routePanelAnalyticsRoutes": 36,
"routePanelAnalyticsRouteViews": 37, "routePanelAnalyticsAgents": 37,
"routePanelGroups": 38, "routePanelAnalyticsRouteViews": 38,
"routePanelGroupsEdit": 39, "routePanelGroups": 39,
"routePanelGroupsEditPerms": 40, "routePanelGroupsEdit": 40,
"routePanelGroupsEditSubmit": 41, "routePanelGroupsEditPerms": 41,
"routePanelGroupsEditPermsSubmit": 42, "routePanelGroupsEditSubmit": 42,
"routePanelGroupsCreateSubmit": 43, "routePanelGroupsEditPermsSubmit": 43,
"routePanelBackups": 44, "routePanelGroupsCreateSubmit": 44,
"routePanelLogsMod": 45, "routePanelBackups": 45,
"routePanelDebug": 46, "routePanelLogsMod": 46,
"routePanel": 47, "routePanelDebug": 47,
"routeAccountEditCritical": 48, "routePanel": 48,
"routeAccountEditCriticalSubmit": 49, "routeAccountEditCritical": 49,
"routeAccountEditAvatar": 50, "routeAccountEditCriticalSubmit": 50,
"routeAccountEditAvatarSubmit": 51, "routeAccountEditAvatar": 51,
"routeAccountEditUsername": 52, "routeAccountEditAvatarSubmit": 52,
"routeAccountEditUsernameSubmit": 53, "routeAccountEditUsername": 53,
"routeAccountEditEmail": 54, "routeAccountEditUsernameSubmit": 54,
"routeAccountEditEmailTokenSubmit": 55, "routeAccountEditEmail": 55,
"routeProfile": 56, "routeAccountEditEmailTokenSubmit": 56,
"routeBanSubmit": 57, "routeProfile": 57,
"routeUnban": 58, "routeBanSubmit": 58,
"routeActivate": 59, "routeUnban": 59,
"routeIps": 60, "routeActivate": 60,
"routeDynamic": 61, "routeIps": 61,
"routeUploads": 62, "routeDynamic": 62,
"routeUploads": 63,
} }
var reverseRouteMapEnum = map[int]string{ var reverseRouteMapEnum = map[int]string{
0: "routeAPI", 0: "routeAPI",
@ -184,38 +186,69 @@ var reverseRouteMapEnum = map[int]string{
34: "routePanelUsersEditSubmit", 34: "routePanelUsersEditSubmit",
35: "routePanelAnalyticsViews", 35: "routePanelAnalyticsViews",
36: "routePanelAnalyticsRoutes", 36: "routePanelAnalyticsRoutes",
37: "routePanelAnalyticsRouteViews", 37: "routePanelAnalyticsAgents",
38: "routePanelGroups", 38: "routePanelAnalyticsRouteViews",
39: "routePanelGroupsEdit", 39: "routePanelGroups",
40: "routePanelGroupsEditPerms", 40: "routePanelGroupsEdit",
41: "routePanelGroupsEditSubmit", 41: "routePanelGroupsEditPerms",
42: "routePanelGroupsEditPermsSubmit", 42: "routePanelGroupsEditSubmit",
43: "routePanelGroupsCreateSubmit", 43: "routePanelGroupsEditPermsSubmit",
44: "routePanelBackups", 44: "routePanelGroupsCreateSubmit",
45: "routePanelLogsMod", 45: "routePanelBackups",
46: "routePanelDebug", 46: "routePanelLogsMod",
47: "routePanel", 47: "routePanelDebug",
48: "routeAccountEditCritical", 48: "routePanel",
49: "routeAccountEditCriticalSubmit", 49: "routeAccountEditCritical",
50: "routeAccountEditAvatar", 50: "routeAccountEditCriticalSubmit",
51: "routeAccountEditAvatarSubmit", 51: "routeAccountEditAvatar",
52: "routeAccountEditUsername", 52: "routeAccountEditAvatarSubmit",
53: "routeAccountEditUsernameSubmit", 53: "routeAccountEditUsername",
54: "routeAccountEditEmail", 54: "routeAccountEditUsernameSubmit",
55: "routeAccountEditEmailTokenSubmit", 55: "routeAccountEditEmail",
56: "routeProfile", 56: "routeAccountEditEmailTokenSubmit",
57: "routeBanSubmit", 57: "routeProfile",
58: "routeUnban", 58: "routeBanSubmit",
59: "routeActivate", 59: "routeUnban",
60: "routeIps", 60: "routeActivate",
61: "routeDynamic", 61: "routeIps",
62: "routeUploads", 62: "routeDynamic",
63: "routeUploads",
}
var agentMapEnum = map[string]int{
"unknown": 0,
"firefox": 1,
"chrome": 2,
"opera": 3,
"safari": 4,
"edge": 5,
"internet-explorer": 6,
"googlebot": 7,
"yandex": 8,
"bing": 9,
"baidu": 10,
"duckduckgo": 11,
}
var reverseAgentMapEnum = map[int]string{
0: "unknown",
1: "firefox",
2: "chrome",
3: "opera",
4: "safari",
5: "edge",
6: "internet-explorer",
7: "googlebot",
8: "yandex",
9: "bing",
10: "baidu",
11: "duckduckgo",
} }
// TODO: Stop spilling these into the package scope? // TODO: Stop spilling these into the package scope?
func init() { func init() {
common.SetRouteMapEnum(routeMapEnum) common.SetRouteMapEnum(routeMapEnum)
common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
} }
type GenRouter struct { type GenRouter struct {
@ -302,6 +335,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Increment the global view counter // Increment the global view counter
common.GlobalViewCounter.Bump() common.GlobalViewCounter.Bump()
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind
switch {
case strings.Contains(ua,"Google"):
common.AgentViewCounter.Bump(7)
case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that
common.AgentViewCounter.Bump(3)
case strings.Contains(ua,"Chrome"):
common.AgentViewCounter.Bump(2)
case strings.Contains(ua,"Firefox"):
common.AgentViewCounter.Bump(1)
case strings.Contains(ua,"Safari"):
common.AgentViewCounter.Bump(4)
default:
common.AgentViewCounter.Bump(0)
}
// Deal with the session stuff, etc. // Deal with the session stuff, etc.
user, ok := common.PreRoute(w, req) user, ok := common.PreRoute(w, req)
if !ok { if !ok {
@ -592,17 +644,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case "/panel/analytics/routes/": case "/panel/analytics/routes/":
common.RouteViewCounter.Bump(36) common.RouteViewCounter.Bump(36)
err = routePanelAnalyticsRoutes(w,req,user) err = routePanelAnalyticsRoutes(w,req,user)
case "/panel/analytics/route/": case "/panel/analytics/agents/":
common.RouteViewCounter.Bump(37) common.RouteViewCounter.Bump(37)
err = routePanelAnalyticsAgents(w,req,user)
case "/panel/analytics/route/":
common.RouteViewCounter.Bump(38)
err = routePanelAnalyticsRouteViews(w,req,user,extraData) err = routePanelAnalyticsRouteViews(w,req,user,extraData)
case "/panel/groups/": case "/panel/groups/":
common.RouteViewCounter.Bump(38) common.RouteViewCounter.Bump(39)
err = routePanelGroups(w,req,user) err = routePanelGroups(w,req,user)
case "/panel/groups/edit/": case "/panel/groups/edit/":
common.RouteViewCounter.Bump(39) common.RouteViewCounter.Bump(40)
err = routePanelGroupsEdit(w,req,user,extraData) err = routePanelGroupsEdit(w,req,user,extraData)
case "/panel/groups/edit/perms/": case "/panel/groups/edit/perms/":
common.RouteViewCounter.Bump(40) common.RouteViewCounter.Bump(41)
err = routePanelGroupsEditPerms(w,req,user,extraData) err = routePanelGroupsEditPerms(w,req,user,extraData)
case "/panel/groups/edit/submit/": case "/panel/groups/edit/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -611,7 +666,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(41) common.RouteViewCounter.Bump(42)
err = routePanelGroupsEditSubmit(w,req,user,extraData) err = routePanelGroupsEditSubmit(w,req,user,extraData)
case "/panel/groups/edit/perms/submit/": case "/panel/groups/edit/perms/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -620,7 +675,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(42) common.RouteViewCounter.Bump(43)
err = routePanelGroupsEditPermsSubmit(w,req,user,extraData) err = routePanelGroupsEditPermsSubmit(w,req,user,extraData)
case "/panel/groups/create/": case "/panel/groups/create/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -629,7 +684,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(43) common.RouteViewCounter.Bump(44)
err = routePanelGroupsCreateSubmit(w,req,user) err = routePanelGroupsCreateSubmit(w,req,user)
case "/panel/backups/": case "/panel/backups/":
err = common.SuperAdminOnly(w,req,user) err = common.SuperAdminOnly(w,req,user)
@ -638,10 +693,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(44) common.RouteViewCounter.Bump(45)
err = routePanelBackups(w,req,user,extraData) err = routePanelBackups(w,req,user,extraData)
case "/panel/logs/mod/": case "/panel/logs/mod/":
common.RouteViewCounter.Bump(45) common.RouteViewCounter.Bump(46)
err = routePanelLogsMod(w,req,user) err = routePanelLogsMod(w,req,user)
case "/panel/debug/": case "/panel/debug/":
err = common.AdminOnly(w,req,user) err = common.AdminOnly(w,req,user)
@ -650,10 +705,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(46) common.RouteViewCounter.Bump(47)
err = routePanelDebug(w,req,user) err = routePanelDebug(w,req,user)
default: default:
common.RouteViewCounter.Bump(47) common.RouteViewCounter.Bump(48)
err = routePanel(w,req,user) err = routePanel(w,req,user)
} }
if err != nil { if err != nil {
@ -668,7 +723,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(48) common.RouteViewCounter.Bump(49)
err = routeAccountEditCritical(w,req,user) err = routeAccountEditCritical(w,req,user)
case "/user/edit/critical/submit/": case "/user/edit/critical/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -683,7 +738,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(49) common.RouteViewCounter.Bump(50)
err = routeAccountEditCriticalSubmit(w,req,user) err = routeAccountEditCriticalSubmit(w,req,user)
case "/user/edit/avatar/": case "/user/edit/avatar/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -692,7 +747,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(50) common.RouteViewCounter.Bump(51)
err = routeAccountEditAvatar(w,req,user) err = routeAccountEditAvatar(w,req,user)
case "/user/edit/avatar/submit/": case "/user/edit/avatar/submit/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -701,7 +756,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(51) common.RouteViewCounter.Bump(52)
err = routeAccountEditAvatarSubmit(w,req,user) err = routeAccountEditAvatarSubmit(w,req,user)
case "/user/edit/username/": case "/user/edit/username/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -710,7 +765,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(52) common.RouteViewCounter.Bump(53)
err = routeAccountEditUsername(w,req,user) err = routeAccountEditUsername(w,req,user)
case "/user/edit/username/submit/": case "/user/edit/username/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -725,7 +780,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(53) common.RouteViewCounter.Bump(54)
err = routeAccountEditUsernameSubmit(w,req,user) err = routeAccountEditUsernameSubmit(w,req,user)
case "/user/edit/email/": case "/user/edit/email/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -734,7 +789,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(54) common.RouteViewCounter.Bump(55)
err = routeAccountEditEmail(w,req,user) err = routeAccountEditEmail(w,req,user)
case "/user/edit/token/": case "/user/edit/token/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -749,11 +804,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(55) common.RouteViewCounter.Bump(56)
err = routeAccountEditEmailTokenSubmit(w,req,user,extraData) err = routeAccountEditEmailTokenSubmit(w,req,user,extraData)
default: default:
req.URL.Path += extraData req.URL.Path += extraData
common.RouteViewCounter.Bump(56) common.RouteViewCounter.Bump(57)
err = routeProfile(w,req,user) err = routeProfile(w,req,user)
} }
if err != nil { if err != nil {
@ -774,7 +829,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(57) common.RouteViewCounter.Bump(58)
err = routeBanSubmit(w,req,user,extraData) err = routeBanSubmit(w,req,user,extraData)
case "/users/unban/": case "/users/unban/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -789,7 +844,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(58) common.RouteViewCounter.Bump(59)
err = routeUnban(w,req,user,extraData) err = routeUnban(w,req,user,extraData)
case "/users/activate/": case "/users/activate/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -804,7 +859,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(59) common.RouteViewCounter.Bump(60)
err = routeActivate(w,req,user,extraData) err = routeActivate(w,req,user,extraData)
case "/users/ips/": case "/users/ips/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -813,7 +868,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(60) common.RouteViewCounter.Bump(61)
err = routeIps(w,req,user) err = routeIps(w,req,user)
} }
if err != nil { if err != nil {
@ -830,7 +885,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.NotFound(w,req) common.NotFound(w,req)
return return
} }
common.RouteViewCounter.Bump(62) common.RouteViewCounter.Bump(63)
req.URL.Path += extraData req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this? // TODO: Find a way to propagate errors up from this?
router.UploadHandler(w,req) // TODO: Count these views router.UploadHandler(w,req) // TODO: Count these views
@ -874,7 +929,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.RUnlock() router.RUnlock()
if ok { if ok {
common.RouteViewCounter.Bump(61) // TODO: Be more specific about *which* dynamic route it is common.RouteViewCounter.Bump(62) // TODO: Be more specific about *which* dynamic route it is
req.URL.Path += extraData req.URL.Path += extraData
err = handle(w,req,user) err = handle(w,req,user)
if err != nil { if err != nil {

View File

@ -84,6 +84,10 @@ func afterDBInit() (err error) {
if err != nil { if err != nil {
return err return err
} }
common.AgentViewCounter, err = common.NewDefaultAgentViewCounter()
if err != nil {
return err
}
common.RouteViewCounter, err = common.NewDefaultRouteViewCounter() common.RouteViewCounter, err = common.NewDefaultRouteViewCounter()
if err != nil { if err != nil {
return err return err

View File

@ -579,59 +579,6 @@ func routePanelAnalyticsViews(w http.ResponseWriter, r *http.Request, user commo
return nil return nil
} }
func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var routeMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var route string
err := rows.Scan(&count, &route)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("route: ", route)
routeMap[route] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var routeItems []common.PanelAnalyticsRoutesItem
for route, count := range routeMap {
routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{
Route: route,
Count: count,
})
}
pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems}
if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.User, route string) common.RouteError { func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.User, route string) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil { if ferr != nil {
@ -730,6 +677,112 @@ func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user
return nil return nil
} }
func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var routeMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var route string
err := rows.Scan(&count, &route)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("route: ", route)
routeMap[route] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var routeItems []common.PanelAnalyticsRoutesItem
for route, count := range routeMap {
routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{
Route: route,
Count: count,
})
}
pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems}
if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var agentMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks_agents").Columns("count, browser").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var agent string
err := rows.Scan(&count, &agent)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("agent: ", agent)
agentMap[agent] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var agentItems []common.PanelAnalyticsAgentsItem
for agent, count := range agentMap {
agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{
Agent: agent,
Count: count,
})
}
pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", agentItems}
if common.PreRenderHooks["pre_render_panel_analytics_agents"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_agents", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-agents.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil { if ferr != nil {

View File

@ -365,6 +365,16 @@ func createTables(adapter qgen.Adapter) error {
[]qgen.DBTableKey{}, []qgen.DBTableKey{},
) )
qgen.Install.CreateTable("viewchunks_agents", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"count", "int", 0, false, false, "0"},
qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""},
qgen.DBTableColumn{"browser", "varchar", 200, false, false, ""}, // googlebot, firefox, opera, etc.
//qgen.DBTableColumn{"version","varchar",0,false,false,""}, // the version of the browser or bot
},
[]qgen.DBTableKey{},
)
/* /*
qgen.Install.CreateTable("viewchunks_forums", "", "", qgen.Install.CreateTable("viewchunks_forums", "", "",
[]qgen.DBTableColumn{ []qgen.DBTableColumn{

View File

@ -17,6 +17,8 @@ type TmplVars struct {
RouteGroups []*RouteGroup RouteGroups []*RouteGroup
AllRouteNames []string AllRouteNames []string
AllRouteMap map[string]int AllRouteMap map[string]int
AllAgentNames []string
AllAgentMap map[string]int
} }
func main() { func main() {
@ -155,6 +157,26 @@ func main() {
mapIt("routeUploads") mapIt("routeUploads")
tmplVars.AllRouteNames = allRouteNames tmplVars.AllRouteNames = allRouteNames
tmplVars.AllRouteMap = allRouteMap tmplVars.AllRouteMap = allRouteMap
tmplVars.AllAgentNames = []string{
"unknown",
"firefox",
"chrome",
"opera",
"safari",
"edge",
"internet-explorer",
"googlebot",
"yandex",
"bing",
"baidu",
"duckduckgo",
}
tmplVars.AllAgentMap = make(map[string]int)
for id, agent := range tmplVars.AllAgentNames {
tmplVars.AllAgentMap[agent] = id
}
var fileData = `// Code generated by. DO NOT EDIT. var fileData = `// Code generated by. DO NOT EDIT.
/* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */
@ -183,11 +205,19 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
{{$index}}: "{{$element}}",{{end}} {{$index}}: "{{$element}}",{{end}}
} }
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}}
}
var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}}
{{$index}}: "{{$element}}",{{end}}
}
// TODO: Stop spilling these into the package scope? // TODO: Stop spilling these into the package scope?
func init() { func init() {
common.SetRouteMapEnum(routeMapEnum) common.SetRouteMapEnum(routeMapEnum)
common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
} }
type GenRouter struct { type GenRouter struct {
@ -274,6 +304,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Increment the global view counter // Increment the global view counter
common.GlobalViewCounter.Bump() common.GlobalViewCounter.Bump()
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind
switch {
case strings.Contains(ua,"Google"):
common.AgentViewCounter.Bump({{.AllAgentMap.googlebot}})
case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that
common.AgentViewCounter.Bump({{.AllAgentMap.opera}})
case strings.Contains(ua,"Chrome"):
common.AgentViewCounter.Bump({{.AllAgentMap.chrome}})
case strings.Contains(ua,"Firefox"):
common.AgentViewCounter.Bump({{.AllAgentMap.firefox}})
case strings.Contains(ua,"Safari"):
common.AgentViewCounter.Bump({{.AllAgentMap.safari}})
default:
common.AgentViewCounter.Bump({{.AllAgentMap.unknown}})
}
// Deal with the session stuff, etc. // Deal with the session stuff, etc.
user, ok := common.PreRoute(w, req) user, ok := common.PreRoute(w, req)
if !ok { if !ok {

View File

@ -92,6 +92,7 @@ func buildPanelRoutes() {
View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"), View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"),
View("routePanelAnalyticsRoutes", "/panel/analytics/routes/"), View("routePanelAnalyticsRoutes", "/panel/analytics/routes/"),
View("routePanelAnalyticsAgents", "/panel/analytics/agents/"),
View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"), View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"),
View("routePanelGroups", "/panel/groups/"), View("routePanelGroups", "/panel/groups/"),

View File

@ -0,0 +1,5 @@
CREATE TABLE [viewchunks_agents] (
[count] int DEFAULT 0 not null,
[createdAt] datetime not null,
[browser] nvarchar (200) not null
);

View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_agents` (
`count` int DEFAULT 0 not null,
`createdAt` datetime not null,
`browser` varchar(200) not null
);

View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_agents` (
`count` int DEFAULT 0 not null,
`createdAt` timestamp not null,
`browser` varchar (200) not null
);

View File

@ -0,0 +1,18 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<div class="colstack_item colstack_head">
<div class="rowitem"><a>Agents (24 hours)</a></div>
</div>
<div id="panel_analytics_agents" class="colstack_item rowlist">
{{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.Agent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span>
</div>
{{end}}
</div>
</main>
</div>
{{template "footer.html" . }}

View File

@ -35,6 +35,9 @@
<div class="rowitem passive submenu"> <div class="rowitem passive submenu">
<a href="/panel/analytics/routes/">Routes</a> <a href="/panel/analytics/routes/">Routes</a>
</div> </div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/agents/">Agents</a>
</div>
<div class="rowitem passive submenu"> <div class="rowitem passive submenu">
<a href="/panel/analytics/crawlers/">Crawlers</a> <a href="/panel/analytics/crawlers/">Crawlers</a>
</div> </div>

View File

@ -168,10 +168,10 @@
padding-top: 16px; padding-top: 16px;
} }
.ct-series-a .ct-bar, .ct-series-a .ct-line, .ct-series-a .ct-point, .ct-series-a .ct-slice-donut { .ct-series-a .ct-bar, .ct-series-a .ct-line, .ct-series-a .ct-point, .ct-series-a .ct-slice-donut {
stroke: hsl(359,98%,33%) !important; stroke: hsl(359,98%,53%) !important;
} }
.ct-point { .ct-point {
stroke: hsl(359,98%,53%) !important; stroke: hsl(359,98%,33%) !important;
} }
.ct-point:hover { .ct-point:hover {
stroke: hsl(359,98%,50%) !important; stroke: hsl(359,98%,50%) !important;