a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

We now track global user agent stats and have a currently simple Control Panel interface for that. 

Fixed a possible out of bounds panic in DefaultRouteViewCounter.Bump()
This commit is contained in:
Azareal 2018-01-09 07:39:29 +00:00
parent c7aec90612
commit a25ee29197
16 changed files with 427 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
common/common.go
View File

@ -119,3 +119,14 @@ func SetRouteMapEnum(rme map[string]int) {
func SetReverseRouteMapEnum(rrme map[int]string) {
reverseRouteMapEnum = rrme
}
var agentMapEnum map[string]int
var reverseAgentMapEnum map[int]string
func SetAgentMapEnum(ame map[string]int) {
agentMapEnum = ame
}
func SetReverseAgentMapEnum(rame map[int]string) {
reverseAgentMapEnum = rame
}

62
common/counters.go
View File

@ -9,6 +9,7 @@ import (
)
var GlobalViewCounter *ChunkedViewCounter
var AgentViewCounter *DefaultAgentViewCounter
var RouteViewCounter *DefaultRouteViewCounter
var TopicViewCounter *DefaultTopicViewCounter
@ -64,7 +65,64 @@ type RWMutexCounterBucket struct {
sync.RWMutex
}
// The name of the struct clashes with the name of the variable, so we're adding Impl to the end
type DefaultAgentViewCounter struct {
agentBuckets []*RWMutexCounterBucket //[AgentID]count
insert *sql.Stmt
}
func NewDefaultAgentViewCounter() (*DefaultAgentViewCounter, error) {
acc := qgen.Builder.Accumulator()
var agentBuckets = make([]*RWMutexCounterBucket, len(agentMapEnum))
for bucketID, _ := range agentBuckets {
agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
counter := &DefaultAgentViewCounter{
agentBuckets: agentBuckets,
insert: acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
AddScheduledFifteenMinuteTask(counter.Tick)
//AddScheduledSecondTask(counter.Tick)
AddShutdownTask(counter.Tick)
return counter, acc.FirstError()
}
func (counter *DefaultAgentViewCounter) Tick() error {
for agentID, agentBucket := range counter.agentBuckets {
var count int
agentBucket.RLock()
count = agentBucket.counter
agentBucket.counter = 0
agentBucket.RUnlock()
err := counter.insertChunk(count, agentID) // TODO: Bulk insert for speed?
if err != nil {
return err
}
}
return nil
}
func (counter *DefaultAgentViewCounter) insertChunk(count int, agent int) error {
if count == 0 {
return nil
}
var agentName = reverseAgentMapEnum[agent]
debugLogf("Inserting a viewchunk with a count of %d for agent %s (%d)", count, agentName, agent)
_, err := counter.insert.Exec(count, agentName)
return err
}
func (counter *DefaultAgentViewCounter) Bump(agent int) {
// TODO: Test this check
debugLog("counter.agentBuckets[", agent, "]: ", counter.agentBuckets[agent])
if len(counter.agentBuckets) <= agent || agent < 0 {
return
}
counter.agentBuckets[agent].Lock()
counter.agentBuckets[agent].counter++
counter.agentBuckets[agent].Unlock()
}
type DefaultRouteViewCounter struct {
routeBuckets []*RWMutexCounterBucket //[RouteID]count
insert *sql.Stmt
@ -115,7 +173,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error
func (counter *DefaultRouteViewCounter) Bump(route int) {
// TODO: Test this check
debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route])
if len(counter.routeBuckets) <= route {
if len(counter.routeBuckets) <= route || route < 0 {
return
}
counter.routeBuckets[route].Lock()

1
common/extend.go
View File

@ -98,6 +98,7 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User
"pre_render_panel_edit_forum": nil,
"pre_render_panel_analytics": nil,
"pre_render_panel_analytics_routes": nil,
"pre_render_panel_analytics_agents": nil,
"pre_render_panel_analytics_route_views": nil,
"pre_render_panel_settings": nil,
"pre_render_panel_setting": nil,

14
common/pages.go
View File

@ -184,6 +184,20 @@ type PanelAnalyticsRoutesPage struct {
ItemList []PanelAnalyticsRoutesItem
}
type PanelAnalyticsAgentsItem struct {
Agent string
Count int
}
type PanelAnalyticsAgentsPage struct {
Title string
CurrentUser User
Header *HeaderVars
Stats PanelStats
Zone string
ItemList []PanelAnalyticsAgentsItem
}
type PanelAnalyticsRoutePage struct {
Title string
CurrentUser User

211
gen_router.go
View File

@ -52,6 +52,7 @@ var RouteMap = map[string]interface{}{
"routePanelUsersEditSubmit": routePanelUsersEditSubmit,
"routePanelAnalyticsViews": routePanelAnalyticsViews,
"routePanelAnalyticsRoutes": routePanelAnalyticsRoutes,
"routePanelAnalyticsAgents": routePanelAnalyticsAgents,
"routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews,
"routePanelGroups": routePanelGroups,
"routePanelGroupsEdit": routePanelGroupsEdit,
@ -119,32 +120,33 @@ var routeMapEnum = map[string]int{
"routePanelUsersEditSubmit": 34,
"routePanelAnalyticsViews": 35,
"routePanelAnalyticsRoutes": 36,
"routePanelAnalyticsRouteViews": 37,
"routePanelGroups": 38,
"routePanelGroupsEdit": 39,
"routePanelGroupsEditPerms": 40,
"routePanelGroupsEditSubmit": 41,
"routePanelGroupsEditPermsSubmit": 42,
"routePanelGroupsCreateSubmit": 43,
"routePanelBackups": 44,
"routePanelLogsMod": 45,
"routePanelDebug": 46,
"routePanel": 47,
"routeAccountEditCritical": 48,
"routeAccountEditCriticalSubmit": 49,
"routeAccountEditAvatar": 50,
"routeAccountEditAvatarSubmit": 51,
"routeAccountEditUsername": 52,
"routeAccountEditUsernameSubmit": 53,
"routeAccountEditEmail": 54,
"routeAccountEditEmailTokenSubmit": 55,
"routeProfile": 56,
"routeBanSubmit": 57,
"routeUnban": 58,
"routeActivate": 59,
"routeIps": 60,
"routeDynamic": 61,
"routeUploads": 62,
"routePanelAnalyticsAgents": 37,
"routePanelAnalyticsRouteViews": 38,
"routePanelGroups": 39,
"routePanelGroupsEdit": 40,
"routePanelGroupsEditPerms": 41,
"routePanelGroupsEditSubmit": 42,
"routePanelGroupsEditPermsSubmit": 43,
"routePanelGroupsCreateSubmit": 44,
"routePanelBackups": 45,
"routePanelLogsMod": 46,
"routePanelDebug": 47,
"routePanel": 48,
"routeAccountEditCritical": 49,
"routeAccountEditCriticalSubmit": 50,
"routeAccountEditAvatar": 51,
"routeAccountEditAvatarSubmit": 52,
"routeAccountEditUsername": 53,
"routeAccountEditUsernameSubmit": 54,
"routeAccountEditEmail": 55,
"routeAccountEditEmailTokenSubmit": 56,
"routeProfile": 57,
"routeBanSubmit": 58,
"routeUnban": 59,
"routeActivate": 60,
"routeIps": 61,
"routeDynamic": 62,
"routeUploads": 63,
}
var reverseRouteMapEnum = map[int]string{
0: "routeAPI",
@ -184,38 +186,69 @@ var reverseRouteMapEnum = map[int]string{
34: "routePanelUsersEditSubmit",
35: "routePanelAnalyticsViews",
36: "routePanelAnalyticsRoutes",
37: "routePanelAnalyticsRouteViews",
38: "routePanelGroups",
39: "routePanelGroupsEdit",
40: "routePanelGroupsEditPerms",
41: "routePanelGroupsEditSubmit",
42: "routePanelGroupsEditPermsSubmit",
43: "routePanelGroupsCreateSubmit",
44: "routePanelBackups",
45: "routePanelLogsMod",
46: "routePanelDebug",
47: "routePanel",
48: "routeAccountEditCritical",
49: "routeAccountEditCriticalSubmit",
50: "routeAccountEditAvatar",
51: "routeAccountEditAvatarSubmit",
52: "routeAccountEditUsername",
53: "routeAccountEditUsernameSubmit",
54: "routeAccountEditEmail",
55: "routeAccountEditEmailTokenSubmit",
56: "routeProfile",
57: "routeBanSubmit",
58: "routeUnban",
59: "routeActivate",
60: "routeIps",
61: "routeDynamic",
62: "routeUploads",
37: "routePanelAnalyticsAgents",
38: "routePanelAnalyticsRouteViews",
39: "routePanelGroups",
40: "routePanelGroupsEdit",
41: "routePanelGroupsEditPerms",
42: "routePanelGroupsEditSubmit",
43: "routePanelGroupsEditPermsSubmit",
44: "routePanelGroupsCreateSubmit",
45: "routePanelBackups",
46: "routePanelLogsMod",
47: "routePanelDebug",
48: "routePanel",
49: "routeAccountEditCritical",
50: "routeAccountEditCriticalSubmit",
51: "routeAccountEditAvatar",
52: "routeAccountEditAvatarSubmit",
53: "routeAccountEditUsername",
54: "routeAccountEditUsernameSubmit",
55: "routeAccountEditEmail",
56: "routeAccountEditEmailTokenSubmit",
57: "routeProfile",
58: "routeBanSubmit",
59: "routeUnban",
60: "routeActivate",
61: "routeIps",
62: "routeDynamic",
63: "routeUploads",
}
var agentMapEnum = map[string]int{
"unknown": 0,
"firefox": 1,
"chrome": 2,
"opera": 3,
"safari": 4,
"edge": 5,
"internet-explorer": 6,
"googlebot": 7,
"yandex": 8,
"bing": 9,
"baidu": 10,
"duckduckgo": 11,
}
var reverseAgentMapEnum = map[int]string{
0: "unknown",
1: "firefox",
2: "chrome",
3: "opera",
4: "safari",
5: "edge",
6: "internet-explorer",
7: "googlebot",
8: "yandex",
9: "bing",
10: "baidu",
11: "duckduckgo",
}
// TODO: Stop spilling these into the package scope?
func init() {
common.SetRouteMapEnum(routeMapEnum)
common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
}
type GenRouter struct {
@ -301,6 +334,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Increment the global view counter
common.GlobalViewCounter.Bump()
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind
switch {
case strings.Contains(ua,"Google"):
common.AgentViewCounter.Bump(7)
case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that
common.AgentViewCounter.Bump(3)
case strings.Contains(ua,"Chrome"):
common.AgentViewCounter.Bump(2)
case strings.Contains(ua,"Firefox"):
common.AgentViewCounter.Bump(1)
case strings.Contains(ua,"Safari"):
common.AgentViewCounter.Bump(4)
default:
common.AgentViewCounter.Bump(0)
}
// Deal with the session stuff, etc.
user, ok := common.PreRoute(w, req)
@ -592,17 +644,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case "/panel/analytics/routes/":
common.RouteViewCounter.Bump(36)
err = routePanelAnalyticsRoutes(w,req,user)
case "/panel/analytics/route/":
case "/panel/analytics/agents/":
common.RouteViewCounter.Bump(37)
err = routePanelAnalyticsAgents(w,req,user)
case "/panel/analytics/route/":
common.RouteViewCounter.Bump(38)
err = routePanelAnalyticsRouteViews(w,req,user,extraData)
case "/panel/groups/":
common.RouteViewCounter.Bump(38)
common.RouteViewCounter.Bump(39)
err = routePanelGroups(w,req,user)
case "/panel/groups/edit/":
common.RouteViewCounter.Bump(39)
common.RouteViewCounter.Bump(40)
err = routePanelGroupsEdit(w,req,user,extraData)
case "/panel/groups/edit/perms/":
common.RouteViewCounter.Bump(40)
common.RouteViewCounter.Bump(41)
err = routePanelGroupsEditPerms(w,req,user,extraData)
case "/panel/groups/edit/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -611,7 +666,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(41)
common.RouteViewCounter.Bump(42)
err = routePanelGroupsEditSubmit(w,req,user,extraData)
case "/panel/groups/edit/perms/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -620,7 +675,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(42)
common.RouteViewCounter.Bump(43)
err = routePanelGroupsEditPermsSubmit(w,req,user,extraData)
case "/panel/groups/create/":
err = common.NoSessionMismatch(w,req,user)
@ -629,7 +684,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(43)
common.RouteViewCounter.Bump(44)
err = routePanelGroupsCreateSubmit(w,req,user)
case "/panel/backups/":
err = common.SuperAdminOnly(w,req,user)
@ -638,10 +693,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(44)
common.RouteViewCounter.Bump(45)
err = routePanelBackups(w,req,user,extraData)
case "/panel/logs/mod/":
common.RouteViewCounter.Bump(45)
common.RouteViewCounter.Bump(46)
err = routePanelLogsMod(w,req,user)
case "/panel/debug/":
err = common.AdminOnly(w,req,user)
@ -650,10 +705,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(46)
common.RouteViewCounter.Bump(47)
err = routePanelDebug(w,req,user)
default:
common.RouteViewCounter.Bump(47)
common.RouteViewCounter.Bump(48)
err = routePanel(w,req,user)
}
if err != nil {
@ -668,7 +723,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(48)
common.RouteViewCounter.Bump(49)
err = routeAccountEditCritical(w,req,user)
case "/user/edit/critical/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -683,7 +738,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(49)
common.RouteViewCounter.Bump(50)
err = routeAccountEditCriticalSubmit(w,req,user)
case "/user/edit/avatar/":
err = common.MemberOnly(w,req,user)
@ -692,7 +747,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(50)
common.RouteViewCounter.Bump(51)
err = routeAccountEditAvatar(w,req,user)
case "/user/edit/avatar/submit/":
err = common.MemberOnly(w,req,user)
@ -701,7 +756,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(51)
common.RouteViewCounter.Bump(52)
err = routeAccountEditAvatarSubmit(w,req,user)
case "/user/edit/username/":
err = common.MemberOnly(w,req,user)
@ -710,7 +765,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(52)
common.RouteViewCounter.Bump(53)
err = routeAccountEditUsername(w,req,user)
case "/user/edit/username/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -725,7 +780,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(53)
common.RouteViewCounter.Bump(54)
err = routeAccountEditUsernameSubmit(w,req,user)
case "/user/edit/email/":
err = common.MemberOnly(w,req,user)
@ -734,7 +789,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(54)
common.RouteViewCounter.Bump(55)
err = routeAccountEditEmail(w,req,user)
case "/user/edit/token/":
err = common.NoSessionMismatch(w,req,user)
@ -749,11 +804,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(55)
common.RouteViewCounter.Bump(56)
err = routeAccountEditEmailTokenSubmit(w,req,user,extraData)
default:
req.URL.Path += extraData
common.RouteViewCounter.Bump(56)
common.RouteViewCounter.Bump(57)
err = routeProfile(w,req,user)
}
if err != nil {
@ -774,7 +829,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(57)
common.RouteViewCounter.Bump(58)
err = routeBanSubmit(w,req,user,extraData)
case "/users/unban/":
err = common.NoSessionMismatch(w,req,user)
@ -789,7 +844,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(58)
common.RouteViewCounter.Bump(59)
err = routeUnban(w,req,user,extraData)
case "/users/activate/":
err = common.NoSessionMismatch(w,req,user)
@ -804,7 +859,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(59)
common.RouteViewCounter.Bump(60)
err = routeActivate(w,req,user,extraData)
case "/users/ips/":
err = common.MemberOnly(w,req,user)
@ -813,7 +868,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(60)
common.RouteViewCounter.Bump(61)
err = routeIps(w,req,user)
}
if err != nil {
@ -830,7 +885,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.NotFound(w,req)
return
}
common.RouteViewCounter.Bump(62)
common.RouteViewCounter.Bump(63)
req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this?
router.UploadHandler(w,req) // TODO: Count these views
@ -874,7 +929,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.RUnlock()
if ok {
common.RouteViewCounter.Bump(61) // TODO: Be more specific about *which* dynamic route it is
common.RouteViewCounter.Bump(62) // TODO: Be more specific about *which* dynamic route it is
req.URL.Path += extraData
err = handle(w,req,user)
if err != nil {

4
main.go
View File

@ -84,6 +84,10 @@ func afterDBInit() (err error) {
if err != nil {
return err
}
common.AgentViewCounter, err = common.NewDefaultAgentViewCounter()
if err != nil {
return err
}
common.RouteViewCounter, err = common.NewDefaultRouteViewCounter()
if err != nil {
return err

159
panel_routes.go
View File

@ -579,59 +579,6 @@ func routePanelAnalyticsViews(w http.ResponseWriter, r *http.Request, user commo
return nil
}
func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var routeMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var route string
err := rows.Scan(&count, &route)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("route: ", route)
routeMap[route] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var routeItems []common.PanelAnalyticsRoutesItem
for route, count := range routeMap {
routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{
Route: route,
Count: count,
})
}
pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems}
if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.User, route string) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
@ -730,6 +677,112 @@ func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user
return nil
}
func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var routeMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var route string
err := rows.Scan(&count, &route)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("route: ", route)
routeMap[route] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var routeItems []common.PanelAnalyticsRoutesItem
for route, count := range routeMap {
routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{
Route: route,
Count: count,
})
}
pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems}
if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var agentMap = make(map[string]int)
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks_agents").Columns("count, browser").DateCutoff("createdAt", 1, "day").Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var agent string
err := rows.Scan(&count, &agent)
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("agent: ", agent)
agentMap[agent] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var agentItems []common.PanelAnalyticsAgentsItem
for agent, count := range agentMap {
agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{
Agent: agent,
Count: count,
})
}
pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", agentItems}
if common.PreRenderHooks["pre_render_panel_analytics_agents"] != nil {
if common.RunPreRenderHook("pre_render_panel_analytics_agents", w, r, &user, &pi) {
return nil
}
}
err = common.Templates.ExecuteTemplate(w, "panel-analytics-agents.html", pi)
if err != nil {
return common.InternalError(err, w, r)
}
return nil
}
func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {

10
query_gen/tables.go
View File

@ -365,6 +365,16 @@ func createTables(adapter qgen.Adapter) error {
[]qgen.DBTableKey{},
)
qgen.Install.CreateTable("viewchunks_agents", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"count", "int", 0, false, false, "0"},
qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""},
qgen.DBTableColumn{"browser", "varchar", 200, false, false, ""}, // googlebot, firefox, opera, etc.
//qgen.DBTableColumn{"version","varchar",0,false,false,""}, // the version of the browser or bot
},
[]qgen.DBTableKey{},
)
/*
qgen.Install.CreateTable("viewchunks_forums", "", "",
[]qgen.DBTableColumn{

49
router_gen/main.go
View File

@ -17,6 +17,8 @@ type TmplVars struct {
RouteGroups []*RouteGroup
AllRouteNames []string
AllRouteMap map[string]int
AllAgentNames []string
AllAgentMap map[string]int
}
func main() {
@ -155,6 +157,26 @@ func main() {
mapIt("routeUploads")
tmplVars.AllRouteNames = allRouteNames
tmplVars.AllRouteMap = allRouteMap
tmplVars.AllAgentNames = []string{
"unknown",
"firefox",
"chrome",
"opera",
"safari",
"edge",
"internet-explorer",
"googlebot",
"yandex",
"bing",
"baidu",
"duckduckgo",
}
tmplVars.AllAgentMap = make(map[string]int)
for id, agent := range tmplVars.AllAgentNames {
tmplVars.AllAgentMap[agent] = id
}
var fileData = `// Code generated by. DO NOT EDIT.
/* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */
@ -183,11 +205,19 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
{{$index}}: "{{$element}}",{{end}}
}
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}}
}
var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}}
{{$index}}: "{{$element}}",{{end}}
}
// TODO: Stop spilling these into the package scope?
func init() {
common.SetRouteMapEnum(routeMapEnum)
common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
}
type GenRouter struct {
@ -273,6 +303,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Increment the global view counter
common.GlobalViewCounter.Bump()
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind
switch {
case strings.Contains(ua,"Google"):
common.AgentViewCounter.Bump({{.AllAgentMap.googlebot}})
case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that
common.AgentViewCounter.Bump({{.AllAgentMap.opera}})
case strings.Contains(ua,"Chrome"):
common.AgentViewCounter.Bump({{.AllAgentMap.chrome}})
case strings.Contains(ua,"Firefox"):
common.AgentViewCounter.Bump({{.AllAgentMap.firefox}})
case strings.Contains(ua,"Safari"):
common.AgentViewCounter.Bump({{.AllAgentMap.safari}})
default:
common.AgentViewCounter.Bump({{.AllAgentMap.unknown}})
}
// Deal with the session stuff, etc.
user, ok := common.PreRoute(w, req)

1
router_gen/routes.go
View File

@ -92,6 +92,7 @@ func buildPanelRoutes() {
View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"),
View("routePanelAnalyticsRoutes", "/panel/analytics/routes/"),
View("routePanelAnalyticsAgents", "/panel/analytics/agents/"),
View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"),
View("routePanelGroups", "/panel/groups/"),

5
schema/mssql/query_viewchunks_agents.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE [viewchunks_agents] (
[count] int DEFAULT 0 not null,
[createdAt] datetime not null,
[browser] nvarchar (200) not null
);

5
schema/mysql/query_viewchunks_agents.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_agents` (
`count` int DEFAULT 0 not null,
`createdAt` datetime not null,
`browser` varchar(200) not null
);

5
schema/pgsql/query_viewchunks_agents.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_agents` (
`count` int DEFAULT 0 not null,
`createdAt` timestamp not null,
`browser` varchar (200) not null
);

18
templates/panel-analytics-agents.html Normal file
View File

@ -0,0 +1,18 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<div class="colstack_item colstack_head">
<div class="rowitem"><a>Agents (24 hours)</a></div>
</div>
<div id="panel_analytics_agents" class="colstack_item rowlist">
{{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.Agent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span>
</div>
{{end}}
</div>
</main>
</div>
{{template "footer.html" . }}

3
templates/panel-inner-menu.html
View File

@ -35,6 +35,9 @@
<div class="rowitem passive submenu">
<a href="/panel/analytics/routes/">Routes</a>
</div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/agents/">Agents</a>
</div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/crawlers/">Crawlers</a>
</div>

4
themes/cosora/public/panel.css
View File

@ -168,10 +168,10 @@
padding-top: 16px;
}
.ct-series-a .ct-bar, .ct-series-a .ct-line, .ct-series-a .ct-point, .ct-series-a .ct-slice-donut {
stroke: hsl(359,98%,33%) !important;
stroke: hsl(359,98%,53%) !important;
}
.ct-point {
stroke: hsl(359,98%,53%) !important;
stroke: hsl(359,98%,33%) !important;
}
.ct-point:hover {
stroke: hsl(359,98%,50%) !important;