We now have analytics for the operating systems used by users.

Route view bumping logs have been moved from regular debug mode to super debug to reduce the amount of noise.
Added user friendly agent names.
Reduced the amount of logging outside of debug mode.
We now track Mobile Safari and the Samsung Browser.
We now track SeznamBot, TwitterBot, and Discourse's Bot.
We now track Trident.
UAs are now filtered to reduce the amount of bad activity.
Added more bad phrases for bad routes.

Added the viewchunks_systems table.
Began work on referrer tracking.
This commit is contained in:
Azareal 2018-02-04 08:15:20 +00:00
parent 017bce9c09
commit 2455e951aa
21 changed files with 945 additions and 302 deletions

View File

@ -130,3 +130,14 @@ func SetAgentMapEnum(ame map[string]int) {
func SetReverseAgentMapEnum(rame map[int]string) { func SetReverseAgentMapEnum(rame map[int]string) {
reverseAgentMapEnum = rame reverseAgentMapEnum = rame
} }
var osMapEnum map[string]int
var reverseOSMapEnum map[int]string
func SetOSMapEnum(osme map[string]int) {
osMapEnum = osme
}
func SetReverseOSMapEnum(rosme map[int]string) {
reverseOSMapEnum = rosme
}

View File

@ -11,6 +11,7 @@ import (
// Global counters // Global counters
var GlobalViewCounter *DefaultViewCounter var GlobalViewCounter *DefaultViewCounter
var AgentViewCounter *DefaultAgentViewCounter var AgentViewCounter *DefaultAgentViewCounter
var OSViewCounter *DefaultOSViewCounter
var RouteViewCounter *DefaultRouteViewCounter var RouteViewCounter *DefaultRouteViewCounter
var PostCounter *DefaultPostCounter var PostCounter *DefaultPostCounter
var TopicCounter *DefaultTopicCounter var TopicCounter *DefaultTopicCounter
@ -222,6 +223,64 @@ func (counter *DefaultAgentViewCounter) Bump(agent int) {
counter.agentBuckets[agent].Unlock() counter.agentBuckets[agent].Unlock()
} }
type DefaultOSViewCounter struct {
osBuckets []*RWMutexCounterBucket //[OSID]count
insert *sql.Stmt
}
func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) {
acc := qgen.Builder.Accumulator()
var osBuckets = make([]*RWMutexCounterBucket, len(osMapEnum))
for bucketID, _ := range osBuckets {
osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
counter := &DefaultOSViewCounter{
osBuckets: osBuckets,
insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
AddScheduledFifteenMinuteTask(counter.Tick)
//AddScheduledSecondTask(counter.Tick)
AddShutdownTask(counter.Tick)
return counter, acc.FirstError()
}
func (counter *DefaultOSViewCounter) Tick() error {
for osID, osBucket := range counter.osBuckets {
var count int
osBucket.RLock()
count = osBucket.counter
osBucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
osBucket.RUnlock()
err := counter.insertChunk(count, osID) // TODO: Bulk insert for speed?
if err != nil {
return err
}
}
return nil
}
func (counter *DefaultOSViewCounter) insertChunk(count int, os int) error {
if count == 0 {
return nil
}
var osName = reverseOSMapEnum[os]
debugLogf("Inserting a viewchunk with a count of %d for OS %s (%d)", count, osName, os)
_, err := counter.insert.Exec(count, osName)
return err
}
func (counter *DefaultOSViewCounter) Bump(os int) {
// TODO: Test this check
debugDetail("counter.osBuckets[", os, "]: ", counter.osBuckets[os])
if len(counter.osBuckets) <= os || os < 0 {
return
}
counter.osBuckets[os].Lock()
counter.osBuckets[os].counter++
counter.osBuckets[os].Unlock()
}
type DefaultRouteViewCounter struct { type DefaultRouteViewCounter struct {
routeBuckets []*RWMutexCounterBucket //[RouteID]count routeBuckets []*RWMutexCounterBucket //[RouteID]count
insert *sql.Stmt insert *sql.Stmt
@ -271,7 +330,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error
func (counter *DefaultRouteViewCounter) Bump(route int) { func (counter *DefaultRouteViewCounter) Bump(route int) {
// TODO: Test this check // TODO: Test this check
debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) debugDetail("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route])
if len(counter.routeBuckets) <= route || route < 0 { if len(counter.routeBuckets) <= route || route < 0 {
return return
} }

View File

@ -92,27 +92,29 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User
"pre_render_ban": nil, "pre_render_ban": nil,
"pre_render_ip_search": nil, "pre_render_ip_search": nil,
"pre_render_panel_dashboard": nil, "pre_render_panel_dashboard": nil,
"pre_render_panel_forums": nil, "pre_render_panel_forums": nil,
"pre_render_panel_delete_forum": nil, "pre_render_panel_delete_forum": nil,
"pre_render_panel_edit_forum": nil, "pre_render_panel_edit_forum": nil,
"pre_render_panel_analytics_views": nil, "pre_render_panel_analytics_views": nil,
"pre_render_panel_analytics_routes": nil, "pre_render_panel_analytics_routes": nil,
"pre_render_panel_analytics_agents": nil, "pre_render_panel_analytics_agents": nil,
"pre_render_panel_analytics_route_views": nil, "pre_render_panel_analytics_systems": nil,
"pre_render_panel_analytics_agent_views": nil, "pre_render_panel_analytics_route_views": nil,
"pre_render_panel_settings": nil, "pre_render_panel_analytics_agent_views": nil,
"pre_render_panel_setting": nil, "pre_render_panel_analytics_system_views": nil,
"pre_render_panel_word_filters": nil, "pre_render_panel_settings": nil,
"pre_render_panel_word_filters_edit": nil, "pre_render_panel_setting": nil,
"pre_render_panel_plugins": nil, "pre_render_panel_word_filters": nil,
"pre_render_panel_users": nil, "pre_render_panel_word_filters_edit": nil,
"pre_render_panel_edit_user": nil, "pre_render_panel_plugins": nil,
"pre_render_panel_groups": nil, "pre_render_panel_users": nil,
"pre_render_panel_edit_group": nil, "pre_render_panel_edit_user": nil,
"pre_render_panel_edit_group_perms": nil, "pre_render_panel_groups": nil,
"pre_render_panel_themes": nil, "pre_render_panel_edit_group": nil,
"pre_render_panel_modlogs": nil, "pre_render_panel_edit_group_perms": nil,
"pre_render_panel_themes": nil,
"pre_render_panel_modlogs": nil,
"pre_render_error": nil, // Note: This hook isn't run for a few errors whose templates are computed at startup and reused, such as InternalError. This hook is also not available in JS mode. "pre_render_error": nil, // Note: This hook isn't run for a few errors whose templates are computed at startup and reused, such as InternalError. This hook is also not available in JS mode.
"pre_render_security_error": nil, "pre_render_security_error": nil,

View File

@ -190,8 +190,9 @@ type PanelAnalyticsRoutesPage struct {
} }
type PanelAnalyticsAgentsItem struct { type PanelAnalyticsAgentsItem struct {
Agent string Agent string
Count int FriendlyAgent string
Count int
} }
type PanelAnalyticsAgentsPage struct { type PanelAnalyticsAgentsPage struct {
@ -223,6 +224,7 @@ type PanelAnalyticsAgentPage struct {
Stats PanelStats Stats PanelStats
Zone string Zone string
Agent string Agent string
FriendlyAgent string
PrimaryGraph PanelTimeGraph PrimaryGraph PanelTimeGraph
TimeRange string TimeRange string
} }

View File

@ -34,16 +34,18 @@ type LevelPhrases struct {
// ! For the sake of thread safety, you must never modify a *LanguagePack directly, but to create a copy of it and overwrite the entry in the sync.Map // ! For the sake of thread safety, you must never modify a *LanguagePack directly, but to create a copy of it and overwrite the entry in the sync.Map
type LanguagePack struct { type LanguagePack struct {
Name string Name string
Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent. Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent.
Levels LevelPhrases Levels LevelPhrases
GlobalPerms map[string]string GlobalPerms map[string]string
LocalPerms map[string]string LocalPerms map[string]string
SettingLabels map[string]string SettingLabels map[string]string
PermPresets map[string]string PermPresets map[string]string
Accounts map[string]string // TODO: Apply these phrases in the software proper Accounts map[string]string // TODO: Apply these phrases in the software proper
Errors map[string]map[string]string // map[category]map[name]value UserAgents map[string]string
PageTitles map[string]string OperatingSystems map[string]string
Errors map[string]map[string]string // map[category]map[name]value
PageTitles map[string]string
} }
// TODO: Add the ability to edit language JSON files from the Control Panel and automatically scan the files for changes // TODO: Add the ability to edit language JSON files from the Control Panel and automatically scan the files for changes
@ -154,6 +156,22 @@ func GetAccountPhrase(name string) string {
return res return res
} }
func GetUserAgentPhrase(name string) (string, bool) {
res, ok := currentLangPack.Load().(*LanguagePack).UserAgents[name]
if !ok {
return "", false
}
return res, true
}
func GetOSPhrase(name string) (string, bool) {
res, ok := currentLangPack.Load().(*LanguagePack).OperatingSystems[name]
if !ok {
return "", false
}
return res, true
}
// TODO: Does comma ok work with multi-dimensional maps? // TODO: Does comma ok work with multi-dimensional maps?
func GetErrorPhrase(category string, name string) string { func GetErrorPhrase(category string, name string) string {
res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name] res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name]

75
common/requests.go Normal file
View File

@ -0,0 +1,75 @@
package common
import (
"sync"
"sync/atomic"
)
// Add ReferrerItems here after they've had zero views for a while
var referrersToDelete = make(map[string]ReferrerDeletable)
type ReferrerDeletable struct {
item *ReferrerItem
scheduledAt int64 //unixtime
}
type ReferrerItem struct {
Counter int64
}
// ? We'll track referrer domains here rather than the exact URL they arrived from for now, we'll think about expanding later
// ? Referrers are fluid and ever-changing so we have to use string keys rather than 'enum' ints
type DefaultReferrerTracker struct {
odd map[string]*ReferrerItem
even map[string]*ReferrerItem
oddLock sync.RWMutex
evenLock sync.RWMutex
}
func NewDefaultReferrerTracker() *DefaultReferrerTracker {
return &DefaultReferrerTracker{
odd: make(map[string]*ReferrerItem),
even: make(map[string]*ReferrerItem),
}
}
func (ref *DefaultReferrerTracker) Tick() (err error) {
for _, del := range referrersToDelete {
_ = del
// TODO: Calculate the gap between now and the times they were scheduled
}
// TODO: Run the queries and schedule zero view refs for deletion from memory
return nil
}
func (ref *DefaultReferrerTracker) Bump(referrer string) {
if referrer == "" {
return
}
var refItem *ReferrerItem
// Slightly crude and rudimentary, but it should give a basic degree of sharding
if referrer[0]%2 == 0 {
ref.evenLock.RLock()
refItem = ref.even[referrer]
ref.evenLock.RUnlock()
if ref != nil {
atomic.AddInt64(&refItem.Counter, 1)
} else {
ref.evenLock.Lock()
ref.even[referrer] = &ReferrerItem{Counter: 1}
ref.evenLock.Unlock()
}
} else {
ref.oddLock.RLock()
refItem = ref.odd[referrer]
ref.oddLock.RUnlock()
if ref != nil {
atomic.AddInt64(&refItem.Counter, 1)
} else {
ref.oddLock.Lock()
ref.odd[referrer] = &ReferrerItem{Counter: 1}
ref.oddLock.Unlock()
}
}
}

View File

@ -56,8 +56,10 @@ var RouteMap = map[string]interface{}{
"routePanelAnalyticsViews": routePanelAnalyticsViews, "routePanelAnalyticsViews": routePanelAnalyticsViews,
"routePanelAnalyticsRoutes": routePanelAnalyticsRoutes, "routePanelAnalyticsRoutes": routePanelAnalyticsRoutes,
"routePanelAnalyticsAgents": routePanelAnalyticsAgents, "routePanelAnalyticsAgents": routePanelAnalyticsAgents,
"routePanelAnalyticsSystems": routePanelAnalyticsSystems,
"routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews, "routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews,
"routePanelAnalyticsAgentViews": routePanelAnalyticsAgentViews, "routePanelAnalyticsAgentViews": routePanelAnalyticsAgentViews,
"routePanelAnalyticsSystemViews": routePanelAnalyticsSystemViews,
"routePanelAnalyticsPosts": routePanelAnalyticsPosts, "routePanelAnalyticsPosts": routePanelAnalyticsPosts,
"routePanelAnalyticsTopics": routePanelAnalyticsTopics, "routePanelAnalyticsTopics": routePanelAnalyticsTopics,
"routePanelGroups": routePanelGroups, "routePanelGroups": routePanelGroups,
@ -154,60 +156,62 @@ var routeMapEnum = map[string]int{
"routePanelAnalyticsViews": 37, "routePanelAnalyticsViews": 37,
"routePanelAnalyticsRoutes": 38, "routePanelAnalyticsRoutes": 38,
"routePanelAnalyticsAgents": 39, "routePanelAnalyticsAgents": 39,
"routePanelAnalyticsRouteViews": 40, "routePanelAnalyticsSystems": 40,
"routePanelAnalyticsAgentViews": 41, "routePanelAnalyticsRouteViews": 41,
"routePanelAnalyticsPosts": 42, "routePanelAnalyticsAgentViews": 42,
"routePanelAnalyticsTopics": 43, "routePanelAnalyticsSystemViews": 43,
"routePanelGroups": 44, "routePanelAnalyticsPosts": 44,
"routePanelGroupsEdit": 45, "routePanelAnalyticsTopics": 45,
"routePanelGroupsEditPerms": 46, "routePanelGroups": 46,
"routePanelGroupsEditSubmit": 47, "routePanelGroupsEdit": 47,
"routePanelGroupsEditPermsSubmit": 48, "routePanelGroupsEditPerms": 48,
"routePanelGroupsCreateSubmit": 49, "routePanelGroupsEditSubmit": 49,
"routePanelBackups": 50, "routePanelGroupsEditPermsSubmit": 50,
"routePanelLogsMod": 51, "routePanelGroupsCreateSubmit": 51,
"routePanelDebug": 52, "routePanelBackups": 52,
"routePanelDashboard": 53, "routePanelLogsMod": 53,
"routes.AccountEditCritical": 54, "routePanelDebug": 54,
"routeAccountEditCriticalSubmit": 55, "routePanelDashboard": 55,
"routeAccountEditAvatar": 56, "routes.AccountEditCritical": 56,
"routeAccountEditAvatarSubmit": 57, "routeAccountEditCriticalSubmit": 57,
"routeAccountEditUsername": 58, "routeAccountEditAvatar": 58,
"routeAccountEditUsernameSubmit": 59, "routeAccountEditAvatarSubmit": 59,
"routeAccountEditEmail": 60, "routeAccountEditUsername": 60,
"routeAccountEditEmailTokenSubmit": 61, "routeAccountEditUsernameSubmit": 61,
"routeProfile": 62, "routeAccountEditEmail": 62,
"routes.BanUserSubmit": 63, "routeAccountEditEmailTokenSubmit": 63,
"routes.UnbanUser": 64, "routeProfile": 64,
"routes.ActivateUser": 65, "routes.BanUserSubmit": 65,
"routes.IPSearch": 66, "routes.UnbanUser": 66,
"routes.CreateTopicSubmit": 67, "routes.ActivateUser": 67,
"routes.EditTopicSubmit": 68, "routes.IPSearch": 68,
"routes.DeleteTopicSubmit": 69, "routes.CreateTopicSubmit": 69,
"routes.StickTopicSubmit": 70, "routes.EditTopicSubmit": 70,
"routes.UnstickTopicSubmit": 71, "routes.DeleteTopicSubmit": 71,
"routes.LockTopicSubmit": 72, "routes.StickTopicSubmit": 72,
"routes.UnlockTopicSubmit": 73, "routes.UnstickTopicSubmit": 73,
"routes.MoveTopicSubmit": 74, "routes.LockTopicSubmit": 74,
"routeLikeTopicSubmit": 75, "routes.UnlockTopicSubmit": 75,
"routes.ViewTopic": 76, "routes.MoveTopicSubmit": 76,
"routeCreateReplySubmit": 77, "routeLikeTopicSubmit": 77,
"routes.ReplyEditSubmit": 78, "routes.ViewTopic": 78,
"routes.ReplyDeleteSubmit": 79, "routeCreateReplySubmit": 79,
"routeReplyLikeSubmit": 80, "routes.ReplyEditSubmit": 80,
"routeProfileReplyCreateSubmit": 81, "routes.ReplyDeleteSubmit": 81,
"routes.ProfileReplyEditSubmit": 82, "routeReplyLikeSubmit": 82,
"routes.ProfileReplyDeleteSubmit": 83, "routeProfileReplyCreateSubmit": 83,
"routes.PollVote": 84, "routes.ProfileReplyEditSubmit": 84,
"routes.PollResults": 85, "routes.ProfileReplyDeleteSubmit": 85,
"routes.AccountLogin": 86, "routes.PollVote": 86,
"routes.AccountRegister": 87, "routes.PollResults": 87,
"routeLogout": 88, "routes.AccountLogin": 88,
"routes.AccountLoginSubmit": 89, "routes.AccountRegister": 89,
"routes.AccountRegisterSubmit": 90, "routeLogout": 90,
"routeDynamic": 91, "routes.AccountLoginSubmit": 91,
"routeUploads": 92, "routes.AccountRegisterSubmit": 92,
"BadRoute": 93, "routeDynamic": 93,
"routeUploads": 94,
"BadRoute": 95,
} }
var reverseRouteMapEnum = map[int]string{ var reverseRouteMapEnum = map[int]string{
0: "routeAPI", 0: "routeAPI",
@ -250,60 +254,78 @@ var reverseRouteMapEnum = map[int]string{
37: "routePanelAnalyticsViews", 37: "routePanelAnalyticsViews",
38: "routePanelAnalyticsRoutes", 38: "routePanelAnalyticsRoutes",
39: "routePanelAnalyticsAgents", 39: "routePanelAnalyticsAgents",
40: "routePanelAnalyticsRouteViews", 40: "routePanelAnalyticsSystems",
41: "routePanelAnalyticsAgentViews", 41: "routePanelAnalyticsRouteViews",
42: "routePanelAnalyticsPosts", 42: "routePanelAnalyticsAgentViews",
43: "routePanelAnalyticsTopics", 43: "routePanelAnalyticsSystemViews",
44: "routePanelGroups", 44: "routePanelAnalyticsPosts",
45: "routePanelGroupsEdit", 45: "routePanelAnalyticsTopics",
46: "routePanelGroupsEditPerms", 46: "routePanelGroups",
47: "routePanelGroupsEditSubmit", 47: "routePanelGroupsEdit",
48: "routePanelGroupsEditPermsSubmit", 48: "routePanelGroupsEditPerms",
49: "routePanelGroupsCreateSubmit", 49: "routePanelGroupsEditSubmit",
50: "routePanelBackups", 50: "routePanelGroupsEditPermsSubmit",
51: "routePanelLogsMod", 51: "routePanelGroupsCreateSubmit",
52: "routePanelDebug", 52: "routePanelBackups",
53: "routePanelDashboard", 53: "routePanelLogsMod",
54: "routes.AccountEditCritical", 54: "routePanelDebug",
55: "routeAccountEditCriticalSubmit", 55: "routePanelDashboard",
56: "routeAccountEditAvatar", 56: "routes.AccountEditCritical",
57: "routeAccountEditAvatarSubmit", 57: "routeAccountEditCriticalSubmit",
58: "routeAccountEditUsername", 58: "routeAccountEditAvatar",
59: "routeAccountEditUsernameSubmit", 59: "routeAccountEditAvatarSubmit",
60: "routeAccountEditEmail", 60: "routeAccountEditUsername",
61: "routeAccountEditEmailTokenSubmit", 61: "routeAccountEditUsernameSubmit",
62: "routeProfile", 62: "routeAccountEditEmail",
63: "routes.BanUserSubmit", 63: "routeAccountEditEmailTokenSubmit",
64: "routes.UnbanUser", 64: "routeProfile",
65: "routes.ActivateUser", 65: "routes.BanUserSubmit",
66: "routes.IPSearch", 66: "routes.UnbanUser",
67: "routes.CreateTopicSubmit", 67: "routes.ActivateUser",
68: "routes.EditTopicSubmit", 68: "routes.IPSearch",
69: "routes.DeleteTopicSubmit", 69: "routes.CreateTopicSubmit",
70: "routes.StickTopicSubmit", 70: "routes.EditTopicSubmit",
71: "routes.UnstickTopicSubmit", 71: "routes.DeleteTopicSubmit",
72: "routes.LockTopicSubmit", 72: "routes.StickTopicSubmit",
73: "routes.UnlockTopicSubmit", 73: "routes.UnstickTopicSubmit",
74: "routes.MoveTopicSubmit", 74: "routes.LockTopicSubmit",
75: "routeLikeTopicSubmit", 75: "routes.UnlockTopicSubmit",
76: "routes.ViewTopic", 76: "routes.MoveTopicSubmit",
77: "routeCreateReplySubmit", 77: "routeLikeTopicSubmit",
78: "routes.ReplyEditSubmit", 78: "routes.ViewTopic",
79: "routes.ReplyDeleteSubmit", 79: "routeCreateReplySubmit",
80: "routeReplyLikeSubmit", 80: "routes.ReplyEditSubmit",
81: "routeProfileReplyCreateSubmit", 81: "routes.ReplyDeleteSubmit",
82: "routes.ProfileReplyEditSubmit", 82: "routeReplyLikeSubmit",
83: "routes.ProfileReplyDeleteSubmit", 83: "routeProfileReplyCreateSubmit",
84: "routes.PollVote", 84: "routes.ProfileReplyEditSubmit",
85: "routes.PollResults", 85: "routes.ProfileReplyDeleteSubmit",
86: "routes.AccountLogin", 86: "routes.PollVote",
87: "routes.AccountRegister", 87: "routes.PollResults",
88: "routeLogout", 88: "routes.AccountLogin",
89: "routes.AccountLoginSubmit", 89: "routes.AccountRegister",
90: "routes.AccountRegisterSubmit", 90: "routeLogout",
91: "routeDynamic", 91: "routes.AccountLoginSubmit",
92: "routeUploads", 92: "routes.AccountRegisterSubmit",
93: "BadRoute", 93: "routeDynamic",
94: "routeUploads",
95: "BadRoute",
}
var osMapEnum = map[string]int{
"unknown": 0,
"windows": 1,
"linux": 2,
"mac": 3,
"android": 4,
"iphone": 5,
}
var reverseOSMapEnum = map[int]string{
0: "unknown",
1: "windows",
2: "linux",
3: "mac",
4: "android",
5: "iphone",
} }
var agentMapEnum = map[string]int{ var agentMapEnum = map[string]int{
"unknown": 0, "unknown": 0,
@ -313,22 +335,27 @@ var agentMapEnum = map[string]int{
"safari": 4, "safari": 4,
"edge": 5, "edge": 5,
"internetexplorer": 6, "internetexplorer": 6,
"androidchrome": 7, "trident": 7,
"mobilesafari": 8, "androidchrome": 8,
"ucbrowser": 9, "mobilesafari": 9,
"googlebot": 10, "samsung": 10,
"yandex": 11, "ucbrowser": 11,
"bing": 12, "googlebot": 12,
"baidu": 13, "yandex": 13,
"duckduckgo": 14, "bing": 14,
"discord": 15, "baidu": 15,
"cloudflare": 16, "duckduckgo": 16,
"uptimebot": 17, "seznambot": 17,
"lynx": 18, "discord": 18,
"blank": 19, "twitter": 19,
"malformed": 20, "cloudflare": 20,
"suspicious": 21, "uptimebot": 21,
"zgrab": 22, "discourse": 22,
"lynx": 23,
"blank": 24,
"malformed": 25,
"suspicious": 26,
"zgrab": 27,
} }
var reverseAgentMapEnum = map[int]string{ var reverseAgentMapEnum = map[int]string{
0: "unknown", 0: "unknown",
@ -338,31 +365,37 @@ var reverseAgentMapEnum = map[int]string{
4: "safari", 4: "safari",
5: "edge", 5: "edge",
6: "internetexplorer", 6: "internetexplorer",
7: "androidchrome", 7: "trident",
8: "mobilesafari", 8: "androidchrome",
9: "ucbrowser", 9: "mobilesafari",
10: "googlebot", 10: "samsung",
11: "yandex", 11: "ucbrowser",
12: "bing", 12: "googlebot",
13: "baidu", 13: "yandex",
14: "duckduckgo", 14: "bing",
15: "discord", 15: "baidu",
16: "cloudflare", 16: "duckduckgo",
17: "uptimebot", 17: "seznambot",
18: "lynx", 18: "discord",
19: "blank", 19: "twitter",
20: "malformed", 20: "cloudflare",
21: "suspicious", 21: "uptimebot",
22: "zgrab", 22: "discourse",
23: "lynx",
24: "blank",
25: "malformed",
26: "suspicious",
27: "zgrab",
} }
var markToAgent = map[string]string{ var markToAgent = map[string]string{
"OPR":"opera", "OPR":"opera",
"Chrome":"chrome", "Chrome":"chrome",
"Firefox":"firefox", "Firefox":"firefox",
"MSIE":"internetexplorer", "MSIE":"internetexplorer",
//"Trident":"internetexplorer", "Trident":"trident", // Hack to support IE11
"Edge":"edge", "Edge":"edge",
"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
"SamsungBrowser":"samsung",
"UCBrowser":"ucbrowser", "UCBrowser":"ucbrowser",
"Google":"googlebot", "Google":"googlebot",
@ -372,9 +405,12 @@ var markToAgent = map[string]string{
"Baiduspider":"baidu", "Baiduspider":"baidu",
"bingbot":"bing", "bingbot":"bing",
"BingPreview":"bing", "BingPreview":"bing",
"SeznamBot":"seznambot",
"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
"Uptimebot":"uptimebot", "Uptimebot":"uptimebot",
"Discordbot":"discord", "Discordbot":"discord",
"Twitterbot":"twitter",
"Discourse":"discourse",
"zgrab":"zgrab", "zgrab":"zgrab",
} }
@ -390,6 +426,8 @@ func init() {
common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum) common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum)
common.SetOSMapEnum(osMapEnum)
common.SetReverseOSMapEnum(reverseOSMapEnum)
} }
type GenRouter struct { type GenRouter struct {
@ -456,7 +494,7 @@ func (router *GenRouter) DumpRequest(req *http.Request) {
func (router *GenRouter) SuspiciousRequest(req *http.Request) { func (router *GenRouter) SuspiciousRequest(req *http.Request) {
log.Print("Suspicious Request") log.Print("Suspicious Request")
router.DumpRequest(req) router.DumpRequest(req)
common.AgentViewCounter.Bump(21) common.AgentViewCounter.Bump(26)
} }
// TODO: Pass the default route or config struct to the router rather than accessing it via a package global // TODO: Pass the default route or config struct to the router rather than accessing it via a package global
@ -484,7 +522,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("")) w.Write([]byte(""))
log.Print("Malformed Request") log.Print("Malformed Request")
router.DumpRequest(req) router.DumpRequest(req)
common.AgentViewCounter.Bump(20) common.AgentViewCounter.Bump(25)
return return
} }
@ -512,19 +550,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if common.Dev.SuperDebug { if common.Dev.SuperDebug {
log.Print("before routes.StaticFile") log.Print("before routes.StaticFile")
log.Print("Method: ", req.Method) router.DumpRequest(req)
for key, value := range req.Header {
for _, vvalue := range value {
log.Print("Header '" + key + "': " + vvalue + "!!")
}
}
log.Print("prefix: ", prefix)
log.Print("req.Host: ", req.Host)
log.Print("req.URL.Path: ", req.URL.Path)
log.Print("req.URL.RawQuery: ", req.URL.RawQuery)
log.Print("extraData: ", extraData)
log.Print("req.Referer(): ", req.Referer())
log.Print("req.RemoteAddr: ", req.RemoteAddr)
} }
if prefix == "/static" { if prefix == "/static" {
@ -544,12 +570,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Use a more efficient detector instead of smashing every possible combination in // TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
if ua == "" { if ua == "" {
common.AgentViewCounter.Bump(19) common.AgentViewCounter.Bump(24)
if common.Dev.DebugMode { if common.Dev.DebugMode {
log.Print("Blank UA: ", req.UserAgent()) log.Print("Blank UA: ", req.UserAgent())
router.DumpRequest(req) router.DumpRequest(req)
} }
} else { } else {
var runeEquals = func(a []rune, b []rune) bool {
if len(a) != len(b) {
return false
}
for i, item := range a {
if item != b[i] {
return false
}
}
return true
}
// WIP UA Parser // WIP UA Parser
var indices []int var indices []int
var items []string var items []string
@ -557,10 +595,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
for index, item := range ua { for index, item := range ua {
if (item > 64 && item < 91) || (item > 96 && item < 123) { if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item) buffer = append(buffer, item)
} else if len(buffer) != 0 { } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' {
items = append(items, string(buffer)) if len(buffer) != 0 {
indices = append(indices, index - 1) items = append(items, string(buffer))
buffer = buffer[:0] indices = append(indices, index - 1)
buffer = buffer[:0]
}
} else {
// TODO: Test this
items = items[:0]
indices = indices[:0]
router.SuspiciousRequest(req)
log.Print("UA Buffer: ", buffer)
log.Print("UA Buffer String: ", string(buffer))
break
} }
} }
@ -575,19 +623,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} }
} }
if common.Dev.SuperDebug {
log.Print("parsed agent: ", agent)
}
if common.Dev.DebugMode { var os string
log.Print("parsed agent: ",agent) for _, mark := range items {
switch(mark) {
case "Windows":
os = "windows"
case "Linux":
os = "linux"
case "Mac":
os = "mac"
case "iPhone":
os = "iphone"
case "Android":
os = "android"
}
}
if os == "" {
os = "unknown"
}
if common.Dev.SuperDebug {
log.Print("os: ", os)
log.Printf("items: %+v\n",items)
} }
// Special handling // Special handling
switch(agent) { switch(agent) {
case "chrome": case "chrome":
for _, mark := range items { if os == "android" {
if mark == "Android" { agent = "androidchrome"
agent = "androidchrome" }
break case "safari":
} if os == "iphone" {
agent = "mobilesafari"
}
case "trident":
// Hack to support IE11, change this after we start logging versions
if strings.Contains(ua,"rv:11") {
agent = "internetexplorer"
} }
case "zgrab": case "zgrab":
router.SuspiciousRequest(req) router.SuspiciousRequest(req)
@ -602,6 +678,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
common.AgentViewCounter.Bump(agentMapEnum[agent]) common.AgentViewCounter.Bump(agentMapEnum[agent])
} }
common.OSViewCounter.Bump(osMapEnum[os])
} }
// Deal with the session stuff, etc. // Deal with the session stuff, etc.
@ -925,12 +1002,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.RouteViewCounter.Bump(39) common.RouteViewCounter.Bump(39)
err = routePanelAnalyticsAgents(w,req,user) err = routePanelAnalyticsAgents(w,req,user)
case "/panel/analytics/route/": case "/panel/analytics/systems/":
err = common.ParseForm(w,req,user)
if err != nil {
router.handleError(err,w,req,user)
return
}
common.RouteViewCounter.Bump(40) common.RouteViewCounter.Bump(40)
err = routePanelAnalyticsSystems(w,req,user)
case "/panel/analytics/route/":
common.RouteViewCounter.Bump(41)
err = routePanelAnalyticsRouteViews(w,req,user,extraData) err = routePanelAnalyticsRouteViews(w,req,user,extraData)
case "/panel/analytics/agent/": case "/panel/analytics/agent/":
common.RouteViewCounter.Bump(41) common.RouteViewCounter.Bump(42)
err = routePanelAnalyticsAgentViews(w,req,user,extraData) err = routePanelAnalyticsAgentViews(w,req,user,extraData)
case "/panel/analytics/system/":
common.RouteViewCounter.Bump(43)
err = routePanelAnalyticsSystemViews(w,req,user,extraData)
case "/panel/analytics/posts/": case "/panel/analytics/posts/":
err = common.ParseForm(w,req,user) err = common.ParseForm(w,req,user)
if err != nil { if err != nil {
@ -938,7 +1027,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(42) common.RouteViewCounter.Bump(44)
err = routePanelAnalyticsPosts(w,req,user) err = routePanelAnalyticsPosts(w,req,user)
case "/panel/analytics/topics/": case "/panel/analytics/topics/":
err = common.ParseForm(w,req,user) err = common.ParseForm(w,req,user)
@ -947,16 +1036,16 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(43) common.RouteViewCounter.Bump(45)
err = routePanelAnalyticsTopics(w,req,user) err = routePanelAnalyticsTopics(w,req,user)
case "/panel/groups/": case "/panel/groups/":
common.RouteViewCounter.Bump(44) common.RouteViewCounter.Bump(46)
err = routePanelGroups(w,req,user) err = routePanelGroups(w,req,user)
case "/panel/groups/edit/": case "/panel/groups/edit/":
common.RouteViewCounter.Bump(45) common.RouteViewCounter.Bump(47)
err = routePanelGroupsEdit(w,req,user,extraData) err = routePanelGroupsEdit(w,req,user,extraData)
case "/panel/groups/edit/perms/": case "/panel/groups/edit/perms/":
common.RouteViewCounter.Bump(46) common.RouteViewCounter.Bump(48)
err = routePanelGroupsEditPerms(w,req,user,extraData) err = routePanelGroupsEditPerms(w,req,user,extraData)
case "/panel/groups/edit/submit/": case "/panel/groups/edit/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -965,7 +1054,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(47) common.RouteViewCounter.Bump(49)
err = routePanelGroupsEditSubmit(w,req,user,extraData) err = routePanelGroupsEditSubmit(w,req,user,extraData)
case "/panel/groups/edit/perms/submit/": case "/panel/groups/edit/perms/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -974,7 +1063,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(48) common.RouteViewCounter.Bump(50)
err = routePanelGroupsEditPermsSubmit(w,req,user,extraData) err = routePanelGroupsEditPermsSubmit(w,req,user,extraData)
case "/panel/groups/create/": case "/panel/groups/create/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -983,7 +1072,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(49) common.RouteViewCounter.Bump(51)
err = routePanelGroupsCreateSubmit(w,req,user) err = routePanelGroupsCreateSubmit(w,req,user)
case "/panel/backups/": case "/panel/backups/":
err = common.SuperAdminOnly(w,req,user) err = common.SuperAdminOnly(w,req,user)
@ -992,10 +1081,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(50) common.RouteViewCounter.Bump(52)
err = routePanelBackups(w,req,user,extraData) err = routePanelBackups(w,req,user,extraData)
case "/panel/logs/mod/": case "/panel/logs/mod/":
common.RouteViewCounter.Bump(51) common.RouteViewCounter.Bump(53)
err = routePanelLogsMod(w,req,user) err = routePanelLogsMod(w,req,user)
case "/panel/debug/": case "/panel/debug/":
err = common.AdminOnly(w,req,user) err = common.AdminOnly(w,req,user)
@ -1004,10 +1093,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(52) common.RouteViewCounter.Bump(54)
err = routePanelDebug(w,req,user) err = routePanelDebug(w,req,user)
default: default:
common.RouteViewCounter.Bump(53) common.RouteViewCounter.Bump(55)
err = routePanelDashboard(w,req,user) err = routePanelDashboard(w,req,user)
} }
if err != nil { if err != nil {
@ -1022,7 +1111,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(54) common.RouteViewCounter.Bump(56)
err = routes.AccountEditCritical(w,req,user) err = routes.AccountEditCritical(w,req,user)
case "/user/edit/critical/submit/": case "/user/edit/critical/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1037,7 +1126,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(55) common.RouteViewCounter.Bump(57)
err = routeAccountEditCriticalSubmit(w,req,user) err = routeAccountEditCriticalSubmit(w,req,user)
case "/user/edit/avatar/": case "/user/edit/avatar/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -1046,7 +1135,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(56) common.RouteViewCounter.Bump(58)
err = routeAccountEditAvatar(w,req,user) err = routeAccountEditAvatar(w,req,user)
case "/user/edit/avatar/submit/": case "/user/edit/avatar/submit/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -1066,7 +1155,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(57) common.RouteViewCounter.Bump(59)
err = routeAccountEditAvatarSubmit(w,req,user) err = routeAccountEditAvatarSubmit(w,req,user)
case "/user/edit/username/": case "/user/edit/username/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -1075,7 +1164,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(58) common.RouteViewCounter.Bump(60)
err = routeAccountEditUsername(w,req,user) err = routeAccountEditUsername(w,req,user)
case "/user/edit/username/submit/": case "/user/edit/username/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1090,7 +1179,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(59) common.RouteViewCounter.Bump(61)
err = routeAccountEditUsernameSubmit(w,req,user) err = routeAccountEditUsernameSubmit(w,req,user)
case "/user/edit/email/": case "/user/edit/email/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -1099,7 +1188,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(60) common.RouteViewCounter.Bump(62)
err = routeAccountEditEmail(w,req,user) err = routeAccountEditEmail(w,req,user)
case "/user/edit/token/": case "/user/edit/token/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1114,11 +1203,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(61) common.RouteViewCounter.Bump(63)
err = routeAccountEditEmailTokenSubmit(w,req,user,extraData) err = routeAccountEditEmailTokenSubmit(w,req,user,extraData)
default: default:
req.URL.Path += extraData req.URL.Path += extraData
common.RouteViewCounter.Bump(62) common.RouteViewCounter.Bump(64)
err = routeProfile(w,req,user) err = routeProfile(w,req,user)
} }
if err != nil { if err != nil {
@ -1139,7 +1228,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(63) common.RouteViewCounter.Bump(65)
err = routes.BanUserSubmit(w,req,user,extraData) err = routes.BanUserSubmit(w,req,user,extraData)
case "/users/unban/": case "/users/unban/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1154,7 +1243,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(64) common.RouteViewCounter.Bump(66)
err = routes.UnbanUser(w,req,user,extraData) err = routes.UnbanUser(w,req,user,extraData)
case "/users/activate/": case "/users/activate/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1169,7 +1258,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(65) common.RouteViewCounter.Bump(67)
err = routes.ActivateUser(w,req,user,extraData) err = routes.ActivateUser(w,req,user,extraData)
case "/users/ips/": case "/users/ips/":
err = common.MemberOnly(w,req,user) err = common.MemberOnly(w,req,user)
@ -1178,7 +1267,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(66) common.RouteViewCounter.Bump(68)
err = routes.IPSearch(w,req,user) err = routes.IPSearch(w,req,user)
} }
if err != nil { if err != nil {
@ -1204,7 +1293,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(67) common.RouteViewCounter.Bump(69)
err = routes.CreateTopicSubmit(w,req,user) err = routes.CreateTopicSubmit(w,req,user)
case "/topic/edit/submit/": case "/topic/edit/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1219,7 +1308,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(68) common.RouteViewCounter.Bump(70)
err = routes.EditTopicSubmit(w,req,user,extraData) err = routes.EditTopicSubmit(w,req,user,extraData)
case "/topic/delete/submit/": case "/topic/delete/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1235,7 +1324,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
req.URL.Path += extraData req.URL.Path += extraData
common.RouteViewCounter.Bump(69) common.RouteViewCounter.Bump(71)
err = routes.DeleteTopicSubmit(w,req,user) err = routes.DeleteTopicSubmit(w,req,user)
case "/topic/stick/submit/": case "/topic/stick/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1250,7 +1339,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(70) common.RouteViewCounter.Bump(72)
err = routes.StickTopicSubmit(w,req,user,extraData) err = routes.StickTopicSubmit(w,req,user,extraData)
case "/topic/unstick/submit/": case "/topic/unstick/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1265,7 +1354,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(71) common.RouteViewCounter.Bump(73)
err = routes.UnstickTopicSubmit(w,req,user,extraData) err = routes.UnstickTopicSubmit(w,req,user,extraData)
case "/topic/lock/submit/": case "/topic/lock/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1281,7 +1370,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
req.URL.Path += extraData req.URL.Path += extraData
common.RouteViewCounter.Bump(72) common.RouteViewCounter.Bump(74)
err = routes.LockTopicSubmit(w,req,user) err = routes.LockTopicSubmit(w,req,user)
case "/topic/unlock/submit/": case "/topic/unlock/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1296,7 +1385,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(73) common.RouteViewCounter.Bump(75)
err = routes.UnlockTopicSubmit(w,req,user,extraData) err = routes.UnlockTopicSubmit(w,req,user,extraData)
case "/topic/move/submit/": case "/topic/move/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1311,7 +1400,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(74) common.RouteViewCounter.Bump(76)
err = routes.MoveTopicSubmit(w,req,user,extraData) err = routes.MoveTopicSubmit(w,req,user,extraData)
case "/topic/like/submit/": case "/topic/like/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1326,10 +1415,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(75) common.RouteViewCounter.Bump(77)
err = routeLikeTopicSubmit(w,req,user,extraData) err = routeLikeTopicSubmit(w,req,user,extraData)
default: default:
common.RouteViewCounter.Bump(76) common.RouteViewCounter.Bump(78)
err = routes.ViewTopic(w,req,user, extraData) err = routes.ViewTopic(w,req,user, extraData)
} }
if err != nil { if err != nil {
@ -1355,7 +1444,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(77) common.RouteViewCounter.Bump(79)
err = routeCreateReplySubmit(w,req,user) err = routeCreateReplySubmit(w,req,user)
case "/reply/edit/submit/": case "/reply/edit/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1370,7 +1459,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(78) common.RouteViewCounter.Bump(80)
err = routes.ReplyEditSubmit(w,req,user,extraData) err = routes.ReplyEditSubmit(w,req,user,extraData)
case "/reply/delete/submit/": case "/reply/delete/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1385,7 +1474,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(79) common.RouteViewCounter.Bump(81)
err = routes.ReplyDeleteSubmit(w,req,user,extraData) err = routes.ReplyDeleteSubmit(w,req,user,extraData)
case "/reply/like/submit/": case "/reply/like/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1400,7 +1489,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(80) common.RouteViewCounter.Bump(82)
err = routeReplyLikeSubmit(w,req,user,extraData) err = routeReplyLikeSubmit(w,req,user,extraData)
} }
if err != nil { if err != nil {
@ -1421,7 +1510,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(81) common.RouteViewCounter.Bump(83)
err = routeProfileReplyCreateSubmit(w,req,user) err = routeProfileReplyCreateSubmit(w,req,user)
case "/profile/reply/edit/submit/": case "/profile/reply/edit/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1436,7 +1525,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(82) common.RouteViewCounter.Bump(84)
err = routes.ProfileReplyEditSubmit(w,req,user,extraData) err = routes.ProfileReplyEditSubmit(w,req,user,extraData)
case "/profile/reply/delete/submit/": case "/profile/reply/delete/submit/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1451,7 +1540,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(83) common.RouteViewCounter.Bump(85)
err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData) err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData)
} }
if err != nil { if err != nil {
@ -1472,10 +1561,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(84) common.RouteViewCounter.Bump(86)
err = routes.PollVote(w,req,user,extraData) err = routes.PollVote(w,req,user,extraData)
case "/poll/results/": case "/poll/results/":
common.RouteViewCounter.Bump(85) common.RouteViewCounter.Bump(87)
err = routes.PollResults(w,req,user,extraData) err = routes.PollResults(w,req,user,extraData)
} }
if err != nil { if err != nil {
@ -1484,10 +1573,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case "/accounts": case "/accounts":
switch(req.URL.Path) { switch(req.URL.Path) {
case "/accounts/login/": case "/accounts/login/":
common.RouteViewCounter.Bump(86) common.RouteViewCounter.Bump(88)
err = routes.AccountLogin(w,req,user) err = routes.AccountLogin(w,req,user)
case "/accounts/create/": case "/accounts/create/":
common.RouteViewCounter.Bump(87) common.RouteViewCounter.Bump(89)
err = routes.AccountRegister(w,req,user) err = routes.AccountRegister(w,req,user)
case "/accounts/logout/": case "/accounts/logout/":
err = common.NoSessionMismatch(w,req,user) err = common.NoSessionMismatch(w,req,user)
@ -1502,7 +1591,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(88) common.RouteViewCounter.Bump(90)
err = routeLogout(w,req,user) err = routeLogout(w,req,user)
case "/accounts/login/submit/": case "/accounts/login/submit/":
err = common.ParseForm(w,req,user) err = common.ParseForm(w,req,user)
@ -1511,7 +1600,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(89) common.RouteViewCounter.Bump(91)
err = routes.AccountLoginSubmit(w,req,user) err = routes.AccountLoginSubmit(w,req,user)
case "/accounts/create/submit/": case "/accounts/create/submit/":
err = common.ParseForm(w,req,user) err = common.ParseForm(w,req,user)
@ -1520,7 +1609,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
common.RouteViewCounter.Bump(90) common.RouteViewCounter.Bump(92)
err = routes.AccountRegisterSubmit(w,req,user) err = routes.AccountRegisterSubmit(w,req,user)
} }
if err != nil { if err != nil {
@ -1537,7 +1626,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.NotFound(w,req) common.NotFound(w,req)
return return
} }
common.RouteViewCounter.Bump(92) common.RouteViewCounter.Bump(94)
req.URL.Path += extraData req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this? // TODO: Find a way to propagate errors up from this?
router.UploadHandler(w,req) // TODO: Count these views router.UploadHandler(w,req) // TODO: Count these views
@ -1580,7 +1669,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.RUnlock() router.RUnlock()
if ok { if ok {
common.RouteViewCounter.Bump(91) // TODO: Be more specific about *which* dynamic route it is common.RouteViewCounter.Bump(93) // TODO: Be more specific about *which* dynamic route it is
req.URL.Path += extraData req.URL.Path += extraData
err = handle(w,req,user) err = handle(w,req,user)
if err != nil { if err != nil {
@ -1591,10 +1680,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Log all bad routes for the admin to figure out where users are going wrong? // TODO: Log all bad routes for the admin to figure out where users are going wrong?
lowerPath := strings.ToLower(req.URL.Path) lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
router.SuspiciousRequest(req) router.SuspiciousRequest(req)
} }
common.RouteViewCounter.Bump(93) common.RouteViewCounter.Bump(95)
common.NotFound(w,req) common.NotFound(w,req)
} }
} }

View File

@ -89,5 +89,45 @@
"panel_mod_logs":"Moderation Logs", "panel_mod_logs":"Moderation Logs",
"panel_admin_logs":"Administration Logs", "panel_admin_logs":"Administration Logs",
"panel_debug":"Debug" "panel_debug":"Debug"
},
"UserAgents": {
"chrome": "Google Chrome",
"firefox":"Mozilla Firefox",
"opera":"Opera",
"safari":"Safari",
"edge": "Edge",
"internetexplorer":"MS Internet Explorer",
"trident":"Trident Engine",
"androidchrome":"Chrome for Android",
"mobilesafari":"Mobile Safari",
"samsung":"Samsung Browser",
"ucbrowser":"UCBrowser",
"googlebot":"Googlebot",
"yandex":"Yandex",
"bing":"Bing",
"baidu":"Baidu",
"duckduckgo":"DuckDuckBot",
"seznambot":"SeznamBot",
"discord":"Discord",
"twitter":"Twitterbot",
"cloudflare":"Cloudflare Alwayson",
"uptimebot":"Uptimebot",
"discourse":"Discourse Forum Onebox",
"lynx":"Lynx",
"zgrab":"Zgrab Application Scanner",
"suspicious":"Suspicious",
"unknown":"Unknown",
"blank":"Blank",
"malformed":"Malformed"
},
"OperatingSystems": {
"windows": "Microsoft Windows",
"linux":"Linux",
"android": "Android",
"iphone":"iPhone",
"unknown":"Unknown"
} }
} }

View File

@ -105,6 +105,10 @@ func afterDBInit() (err error) {
if err != nil { if err != nil {
return err return err
} }
common.OSViewCounter, err = common.NewDefaultOSViewCounter()
if err != nil {
return err
}
common.RouteViewCounter, err = common.NewDefaultRouteViewCounter() common.RouteViewCounter, err = common.NewDefaultRouteViewCounter()
if err != nil { if err != nil {
return err return err

View File

@ -799,11 +799,14 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user
if err != nil { if err != nil {
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix() var unixCreatedAt = createdAt.Unix()
log.Print("unixCreatedAt: ", unixCreatedAt) if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList { for _, value := range labelList {
if unixCreatedAt > value { if unixCreatedAt > value {
viewMap[value] += count viewMap[value] += count
@ -822,10 +825,98 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user
graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
log.Printf("graph: %+v\n", graph) log.Printf("graph: %+v\n", graph)
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", html.EscapeString(agent), graph, timeRange.Range} // ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff
agent = html.EscapeString(agent)
friendlyAgent, ok := common.GetUserAgentPhrase(agent)
if !ok {
friendlyAgent = agent
}
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", agent, friendlyAgent, graph, timeRange.Range}
return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi) return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi)
} }
func routePanelAnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.User, system string) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
headerVars.Stylesheets = append(headerVars.Stylesheets, "chartist/chartist.min.css")
headerVars.Scripts = append(headerVars.Scripts, "chartist/chartist.min.js")
timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange"))
if err != nil {
return common.LocalError(err.Error(), w, r, user)
}
var revLabelList []int64
var labelList []int64
var viewMap = make(map[int64]int64)
var currentTime = time.Now().Unix()
for i := 1; i <= timeRange.Slices; i++ {
var label = currentTime - int64(i*timeRange.SliceWidth)
revLabelList = append(revLabelList, label)
viewMap[label] = 0
}
for _, value := range revLabelList {
labelList = append(labelList, value)
}
var viewList []int64
log.Print("in routePanelAnalyticsSystemViews")
acc := qgen.Builder.Accumulator()
// TODO: Verify the agent is valid
rows, err := acc.Select("viewchunks_systems").Columns("count, createdAt").Where("system = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system)
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int64
var createdAt time.Time
err := rows.Scan(&count, &createdAt)
if err != nil {
return common.InternalError(err, w, r)
}
var unixCreatedAt = createdAt.Unix()
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList {
if unixCreatedAt > value {
viewMap[value] += count
break
}
}
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
for _, value := range revLabelList {
viewList = append(viewList, viewMap[value])
}
graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
log.Printf("graph: %+v\n", graph)
system = html.EscapeString(system)
friendlySystem, ok := common.GetOSPhrase(system)
if !ok {
friendlySystem = system
}
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", system, friendlySystem, graph, timeRange.Range}
return panelRenderTemplate("panel_analytics_system_views", w, r, user, &pi)
}
func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil { if ferr != nil {
@ -870,11 +961,14 @@ func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user comm
if err != nil { if err != nil {
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix() var unixCreatedAt = createdAt.Unix()
log.Print("unixCreatedAt: ", unixCreatedAt) if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList { for _, value := range labelList {
if unixCreatedAt > value { if unixCreatedAt > value {
viewMap[value] += count viewMap[value] += count
@ -943,11 +1037,14 @@ func routePanelAnalyticsPosts(w http.ResponseWriter, r *http.Request, user commo
if err != nil { if err != nil {
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix() var unixCreatedAt = createdAt.Unix()
log.Print("unixCreatedAt: ", unixCreatedAt) if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList { for _, value := range labelList {
if unixCreatedAt > value { if unixCreatedAt > value {
viewMap[value] += count viewMap[value] += count
@ -999,8 +1096,10 @@ func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user comm
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
log.Print("count: ", count) if common.Dev.DebugMode {
log.Print("route: ", route) log.Print("count: ", count)
log.Print("route: ", route)
}
routeMap[route] += count routeMap[route] += count
} }
err = rows.Err() err = rows.Err()
@ -1048,8 +1147,10 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
log.Print("count: ", count) if common.Dev.DebugMode {
log.Print("agent: ", agent) log.Print("count: ", count)
log.Print("agent: ", agent)
}
agentMap[agent] += count agentMap[agent] += count
} }
err = rows.Err() err = rows.Err()
@ -1060,9 +1161,14 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
// TODO: Sort this slice // TODO: Sort this slice
var agentItems []common.PanelAnalyticsAgentsItem var agentItems []common.PanelAnalyticsAgentsItem
for agent, count := range agentMap { for agent, count := range agentMap {
aAgent, ok := common.GetUserAgentPhrase(agent)
if !ok {
aAgent = agent
}
agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{ agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{
Agent: agent, Agent: agent,
Count: count, FriendlyAgent: aAgent,
Count: count,
}) })
} }
@ -1070,6 +1176,62 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
return panelRenderTemplate("panel_analytics_agents", w, r, user, &pi) return panelRenderTemplate("panel_analytics_agents", w, r, user, &pi)
} }
func routePanelAnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var osMap = make(map[string]int)
timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange"))
if err != nil {
return common.LocalError(err.Error(), w, r, user)
}
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks_systems").Columns("count, system").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var system string
err := rows.Scan(&count, &system)
if err != nil {
return common.InternalError(err, w, r)
}
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("system: ", system)
}
osMap[system] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var systemItems []common.PanelAnalyticsAgentsItem
for system, count := range osMap {
sSystem, ok := common.GetOSPhrase(system)
if !ok {
sSystem = system
}
systemItems = append(systemItems, common.PanelAnalyticsAgentsItem{
Agent: system,
FriendlyAgent: sSystem,
Count: count,
})
}
pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", systemItems, timeRange.Range}
return panelRenderTemplate("panel_analytics_systems", w, r, user, &pi)
}
func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil { if ferr != nil {

View File

@ -412,6 +412,15 @@ func createTables(adapter qgen.Adapter) error {
[]qgen.DBTableKey{}, []qgen.DBTableKey{},
) )
qgen.Install.CreateTable("viewchunks_systems", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"count", "int", 0, false, false, "0"},
qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""},
qgen.DBTableColumn{"system", "varchar", 200, false, false, ""}, // windows, android, bot, etc.
},
[]qgen.DBTableKey{},
)
/* /*
qgen.Install.CreateTable("viewchunks_forums", "", "", qgen.Install.CreateTable("viewchunks_forums", "", "",
[]qgen.DBTableColumn{ []qgen.DBTableColumn{

View File

@ -19,6 +19,8 @@ type TmplVars struct {
AllRouteMap map[string]int AllRouteMap map[string]int
AllAgentNames []string AllAgentNames []string
AllAgentMap map[string]int AllAgentMap map[string]int
AllOSNames []string
AllOSMap map[string]int
} }
func main() { func main() {
@ -158,6 +160,20 @@ func main() {
mapIt("BadRoute") mapIt("BadRoute")
tmplVars.AllRouteNames = allRouteNames tmplVars.AllRouteNames = allRouteNames
tmplVars.AllRouteMap = allRouteMap tmplVars.AllRouteMap = allRouteMap
tmplVars.AllOSNames = []string{
"unknown",
"windows",
"linux",
"mac",
"android",
"iphone",
}
tmplVars.AllOSMap = make(map[string]int)
for id, os := range tmplVars.AllOSNames {
tmplVars.AllOSMap[os] = id
}
tmplVars.AllAgentNames = []string{ tmplVars.AllAgentNames = []string{
"unknown", "unknown",
"firefox", "firefox",
@ -166,9 +182,11 @@ func main() {
"safari", "safari",
"edge", "edge",
"internetexplorer", "internetexplorer",
"trident", // Hack to support IE11
"androidchrome", "androidchrome",
"mobilesafari", // Coming soon "mobilesafari",
"samsung",
"ucbrowser", "ucbrowser",
"googlebot", "googlebot",
@ -176,9 +194,12 @@ func main() {
"bing", "bing",
"baidu", "baidu",
"duckduckgo", "duckduckgo",
"seznambot",
"discord", "discord",
"twitter",
"cloudflare", "cloudflare",
"uptimebot", "uptimebot",
"discourse",
"lynx", "lynx",
"blank", "blank",
"malformed", "malformed",
@ -219,6 +240,12 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
{{$index}}: "{{$element}}",{{end}} {{$index}}: "{{$element}}",{{end}}
} }
var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}}
"{{$element}}": {{$index}},{{end}}
}
var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}}
{{$index}}: "{{$element}}",{{end}}
}
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}} "{{$element}}": {{$index}},{{end}}
} }
@ -230,9 +257,10 @@ var markToAgent = map[string]string{
"Chrome":"chrome", "Chrome":"chrome",
"Firefox":"firefox", "Firefox":"firefox",
"MSIE":"internetexplorer", "MSIE":"internetexplorer",
//"Trident":"internetexplorer", "Trident":"trident", // Hack to support IE11
"Edge":"edge", "Edge":"edge",
"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
"SamsungBrowser":"samsung",
"UCBrowser":"ucbrowser", "UCBrowser":"ucbrowser",
"Google":"googlebot", "Google":"googlebot",
@ -242,9 +270,12 @@ var markToAgent = map[string]string{
"Baiduspider":"baidu", "Baiduspider":"baidu",
"bingbot":"bing", "bingbot":"bing",
"BingPreview":"bing", "BingPreview":"bing",
"SeznamBot":"seznambot",
"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
"Uptimebot":"uptimebot", "Uptimebot":"uptimebot",
"Discordbot":"discord", "Discordbot":"discord",
"Twitterbot":"twitter",
"Discourse":"discourse",
"zgrab":"zgrab", "zgrab":"zgrab",
} }
@ -260,6 +291,8 @@ func init() {
common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum) common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum)
common.SetOSMapEnum(osMapEnum)
common.SetReverseOSMapEnum(reverseOSMapEnum)
} }
type GenRouter struct { type GenRouter struct {
@ -382,19 +415,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if common.Dev.SuperDebug { if common.Dev.SuperDebug {
log.Print("before routes.StaticFile") log.Print("before routes.StaticFile")
log.Print("Method: ", req.Method) router.DumpRequest(req)
for key, value := range req.Header {
for _, vvalue := range value {
log.Print("Header '" + key + "': " + vvalue + "!!")
}
}
log.Print("prefix: ", prefix)
log.Print("req.Host: ", req.Host)
log.Print("req.URL.Path: ", req.URL.Path)
log.Print("req.URL.RawQuery: ", req.URL.RawQuery)
log.Print("extraData: ", extraData)
log.Print("req.Referer(): ", req.Referer())
log.Print("req.RemoteAddr: ", req.RemoteAddr)
} }
if prefix == "/static" { if prefix == "/static" {
@ -420,6 +441,18 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.DumpRequest(req) router.DumpRequest(req)
} }
} else { } else {
var runeEquals = func(a []rune, b []rune) bool {
if len(a) != len(b) {
return false
}
for i, item := range a {
if item != b[i] {
return false
}
}
return true
}
// WIP UA Parser // WIP UA Parser
var indices []int var indices []int
var items []string var items []string
@ -427,10 +460,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
for index, item := range ua { for index, item := range ua {
if (item > 64 && item < 91) || (item > 96 && item < 123) { if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item) buffer = append(buffer, item)
} else if len(buffer) != 0 { } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' {
items = append(items, string(buffer)) if len(buffer) != 0 {
indices = append(indices, index - 1) items = append(items, string(buffer))
buffer = buffer[:0] indices = append(indices, index - 1)
buffer = buffer[:0]
}
} else {
// TODO: Test this
items = items[:0]
indices = indices[:0]
router.SuspiciousRequest(req)
log.Print("UA Buffer: ", buffer)
log.Print("UA Buffer String: ", string(buffer))
break
} }
} }
@ -445,19 +488,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
} }
} }
if common.Dev.SuperDebug {
log.Print("parsed agent: ", agent)
}
if common.Dev.DebugMode { var os string
log.Print("parsed agent: ",agent) for _, mark := range items {
switch(mark) {
case "Windows":
os = "windows"
case "Linux":
os = "linux"
case "Mac":
os = "mac"
case "iPhone":
os = "iphone"
case "Android":
os = "android"
}
}
if os == "" {
os = "unknown"
}
if common.Dev.SuperDebug {
log.Print("os: ", os)
log.Printf("items: %+v\n",items)
} }
// Special handling // Special handling
switch(agent) { switch(agent) {
case "chrome": case "chrome":
for _, mark := range items { if os == "android" {
if mark == "Android" { agent = "androidchrome"
agent = "androidchrome" }
break case "safari":
} if os == "iphone" {
agent = "mobilesafari"
}
case "trident":
// Hack to support IE11, change this after we start logging versions
if strings.Contains(ua,"rv:11") {
agent = "internetexplorer"
} }
case "zgrab": case "zgrab":
router.SuspiciousRequest(req) router.SuspiciousRequest(req)
@ -472,6 +543,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
common.AgentViewCounter.Bump(agentMapEnum[agent]) common.AgentViewCounter.Bump(agentMapEnum[agent])
} }
common.OSViewCounter.Bump(osMapEnum[os])
} }
// Deal with the session stuff, etc. // Deal with the session stuff, etc.
@ -551,7 +623,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Log all bad routes for the admin to figure out where users are going wrong? // TODO: Log all bad routes for the admin to figure out where users are going wrong?
lowerPath := strings.ToLower(req.URL.Path) lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
router.SuspiciousRequest(req) router.SuspiciousRequest(req)
} }
common.RouteViewCounter.Bump({{.AllRouteMap.BadRoute}}) common.RouteViewCounter.Bump({{.AllRouteMap.BadRoute}})

View File

@ -166,8 +166,10 @@ func buildPanelRoutes() {
View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"), View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"),
View("routePanelAnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"), View("routePanelAnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"),
View("routePanelAnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"), View("routePanelAnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"),
View("routePanelAnalyticsSystems", "/panel/analytics/systems/").Before("ParseForm"),
View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"), View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"),
View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"), View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"),
View("routePanelAnalyticsSystemViews", "/panel/analytics/system/", "extraData"),
View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"), View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"),
View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"), View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"),

View File

@ -0,0 +1,5 @@
CREATE TABLE [viewchunks_systems] (
[count] int DEFAULT 0 not null,
[createdAt] datetime not null,
[system] nvarchar (200) not null
);

View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_systems` (
`count` int DEFAULT 0 not null,
`createdAt` datetime not null,
`system` varchar(200) not null
);

View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_systems` (
`count` int DEFAULT 0 not null,
`createdAt` timestamp not null,
`system` varchar (200) not null
);

View File

@ -41,6 +41,9 @@
<div class="rowitem passive submenu"> <div class="rowitem passive submenu">
<a href="/panel/analytics/agents/">Agents</a> <a href="/panel/analytics/agents/">Agents</a>
</div> </div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/systems/">Systems</a>
</div>
<div class="rowitem passive submenu"> <div class="rowitem passive submenu">
<a href="/panel/analytics/crawlers/">Crawlers</a> <a href="/panel/analytics/crawlers/">Crawlers</a>
</div> </div>

View File

@ -5,7 +5,7 @@
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/agent/{{.Agent}}" method="get"> <form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/agent/{{.Agent}}" method="get">
<div class="colstack_item colstack_head"> <div class="colstack_item colstack_head">
<div class="rowitem"> <div class="rowitem">
<a>{{.Agent}} Views</a> <a>{{.FriendlyAgent}} Views</a>
<select class="timeRangeSelector to_right" name="timeRange"> <select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option> <option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option> <option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>

View File

@ -19,7 +19,7 @@
<div id="panel_analytics_agents" class="colstack_item rowlist"> <div id="panel_analytics_agents" class="colstack_item rowlist">
{{range .ItemList}} {{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent"> <div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.Agent}}</a> <a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.FriendlyAgent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span> <span class="panel_compacttext to_right">{{.Count}} views</span>
</div> </div>
{{end}} {{end}}

View File

@ -0,0 +1,51 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/system/{{.Agent}}" method="get">
<div class="colstack_item colstack_head">
<div class="rowitem">
<a>{{.FriendlyAgent}} Views</a>
<select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>
<option val="one-day"{{if eq .TimeRange "one-day"}} selected{{end}}>1 day</option>
<option val="twelve-hours"{{if eq .TimeRange "twelve-hours"}} selected{{end}}>12 hours</option>
<option val="six-hours"{{if eq .TimeRange "six-hours"}} selected{{end}}>6 hours</option>
</select>
</div>
</div>
</form>
<div id="panel_analytics_systems" class="colstack_graph_holder">
<div class="ct-chart"></div>
</div>
</main>
</div>
<script>
let labels = [];
let rawLabels = [{{range .PrimaryGraph.Labels}}
{{.}},{{end}}
];
for(const i in rawLabels) {
let date = new Date(rawLabels[i]*1000);
console.log("date: ", date);
let minutes = "0" + date.getMinutes();
let label = date.getHours() + ":" + minutes.substr(-2);
console.log("label:", label);
labels.push(label);
}
labels = labels.reverse()
let seriesData = [{{range .PrimaryGraph.Series}}
{{.}},{{end}}
];
seriesData = seriesData.reverse();
Chartist.Line('.ct-chart', {
labels: labels,
series: [seriesData],
}, {
height: '250px',
});
</script>
{{template "footer.html" . }}

View File

@ -0,0 +1,29 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/systems/" method="get">
<div class="colstack_item colstack_head">
<div class="rowitem">
<a>Operating Systems</a>
<select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>
<option val="one-day"{{if eq .TimeRange "one-day"}} selected{{end}}>1 day</option>
<option val="twelve-hours"{{if eq .TimeRange "twelve-hours"}} selected{{end}}>12 hours</option>
<option val="six-hours"{{if eq .TimeRange "six-hours"}} selected{{end}}>6 hours</option>
</select>
</div>
</div>
</form>
<div id="panel_analytics_systems" class="colstack_item rowlist">
{{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/system/{{.Agent}}" class="panel_upshift">{{.FriendlyAgent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span>
</div>
{{end}}
</div>
</main>
</div>
{{template "footer.html" . }}