diff --git a/common/common.go b/common/common.go index 32db2a31..c02a8cdb 100644 --- a/common/common.go +++ b/common/common.go @@ -130,3 +130,14 @@ func SetAgentMapEnum(ame map[string]int) { func SetReverseAgentMapEnum(rame map[int]string) { reverseAgentMapEnum = rame } + +var osMapEnum map[string]int +var reverseOSMapEnum map[int]string + +func SetOSMapEnum(osme map[string]int) { + osMapEnum = osme +} + +func SetReverseOSMapEnum(rosme map[int]string) { + reverseOSMapEnum = rosme +} diff --git a/common/counters.go b/common/counters.go index 865150c8..c342ea1a 100644 --- a/common/counters.go +++ b/common/counters.go @@ -11,6 +11,7 @@ import ( // Global counters var GlobalViewCounter *DefaultViewCounter var AgentViewCounter *DefaultAgentViewCounter +var OSViewCounter *DefaultOSViewCounter var RouteViewCounter *DefaultRouteViewCounter var PostCounter *DefaultPostCounter var TopicCounter *DefaultTopicCounter @@ -222,6 +223,64 @@ func (counter *DefaultAgentViewCounter) Bump(agent int) { counter.agentBuckets[agent].Unlock() } +type DefaultOSViewCounter struct { + osBuckets []*RWMutexCounterBucket //[OSID]count + insert *sql.Stmt +} + +func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) { + acc := qgen.Builder.Accumulator() + var osBuckets = make([]*RWMutexCounterBucket, len(osMapEnum)) + for bucketID, _ := range osBuckets { + osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} + } + counter := &DefaultOSViewCounter{ + osBuckets: osBuckets, + insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + } + AddScheduledFifteenMinuteTask(counter.Tick) + //AddScheduledSecondTask(counter.Tick) + AddShutdownTask(counter.Tick) + return counter, acc.FirstError() +} + +func (counter *DefaultOSViewCounter) Tick() error { + for osID, osBucket := range counter.osBuckets { + var count int + osBucket.RLock() + count = osBucket.counter + osBucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? + osBucket.RUnlock() + + err := counter.insertChunk(count, osID) // TODO: Bulk insert for speed? + if err != nil { + return err + } + } + return nil +} + +func (counter *DefaultOSViewCounter) insertChunk(count int, os int) error { + if count == 0 { + return nil + } + var osName = reverseOSMapEnum[os] + debugLogf("Inserting a viewchunk with a count of %d for OS %s (%d)", count, osName, os) + _, err := counter.insert.Exec(count, osName) + return err +} + +func (counter *DefaultOSViewCounter) Bump(os int) { + // TODO: Test this check + debugDetail("counter.osBuckets[", os, "]: ", counter.osBuckets[os]) + if len(counter.osBuckets) <= os || os < 0 { + return + } + counter.osBuckets[os].Lock() + counter.osBuckets[os].counter++ + counter.osBuckets[os].Unlock() +} + type DefaultRouteViewCounter struct { routeBuckets []*RWMutexCounterBucket //[RouteID]count insert *sql.Stmt @@ -271,7 +330,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error func (counter *DefaultRouteViewCounter) Bump(route int) { // TODO: Test this check - debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) + debugDetail("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) if len(counter.routeBuckets) <= route || route < 0 { return } diff --git a/common/extend.go b/common/extend.go index 562bb8fb..678d0301 100644 --- a/common/extend.go +++ b/common/extend.go @@ -92,27 +92,29 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User "pre_render_ban": nil, "pre_render_ip_search": nil, - "pre_render_panel_dashboard": nil, - "pre_render_panel_forums": nil, - "pre_render_panel_delete_forum": nil, - "pre_render_panel_edit_forum": nil, - "pre_render_panel_analytics_views": nil, - "pre_render_panel_analytics_routes": nil, - "pre_render_panel_analytics_agents": nil, - "pre_render_panel_analytics_route_views": nil, - "pre_render_panel_analytics_agent_views": nil, - "pre_render_panel_settings": nil, - "pre_render_panel_setting": nil, - "pre_render_panel_word_filters": nil, - "pre_render_panel_word_filters_edit": nil, - "pre_render_panel_plugins": nil, - "pre_render_panel_users": nil, - "pre_render_panel_edit_user": nil, - "pre_render_panel_groups": nil, - "pre_render_panel_edit_group": nil, - "pre_render_panel_edit_group_perms": nil, - "pre_render_panel_themes": nil, - "pre_render_panel_modlogs": nil, + "pre_render_panel_dashboard": nil, + "pre_render_panel_forums": nil, + "pre_render_panel_delete_forum": nil, + "pre_render_panel_edit_forum": nil, + "pre_render_panel_analytics_views": nil, + "pre_render_panel_analytics_routes": nil, + "pre_render_panel_analytics_agents": nil, + "pre_render_panel_analytics_systems": nil, + "pre_render_panel_analytics_route_views": nil, + "pre_render_panel_analytics_agent_views": nil, + "pre_render_panel_analytics_system_views": nil, + "pre_render_panel_settings": nil, + "pre_render_panel_setting": nil, + "pre_render_panel_word_filters": nil, + "pre_render_panel_word_filters_edit": nil, + "pre_render_panel_plugins": nil, + "pre_render_panel_users": nil, + "pre_render_panel_edit_user": nil, + "pre_render_panel_groups": nil, + "pre_render_panel_edit_group": nil, + "pre_render_panel_edit_group_perms": nil, + "pre_render_panel_themes": nil, + "pre_render_panel_modlogs": nil, "pre_render_error": nil, // Note: This hook isn't run for a few errors whose templates are computed at startup and reused, such as InternalError. This hook is also not available in JS mode. "pre_render_security_error": nil, diff --git a/common/pages.go b/common/pages.go index 11d96dd1..4576b17c 100644 --- a/common/pages.go +++ b/common/pages.go @@ -190,8 +190,9 @@ type PanelAnalyticsRoutesPage struct { } type PanelAnalyticsAgentsItem struct { - Agent string - Count int + Agent string + FriendlyAgent string + Count int } type PanelAnalyticsAgentsPage struct { @@ -223,6 +224,7 @@ type PanelAnalyticsAgentPage struct { Stats PanelStats Zone string Agent string + FriendlyAgent string PrimaryGraph PanelTimeGraph TimeRange string } diff --git a/common/phrases.go b/common/phrases.go index a132fbc3..fc947a3c 100644 --- a/common/phrases.go +++ b/common/phrases.go @@ -34,16 +34,18 @@ type LevelPhrases struct { // ! For the sake of thread safety, you must never modify a *LanguagePack directly, but to create a copy of it and overwrite the entry in the sync.Map type LanguagePack struct { - Name string - Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent. - Levels LevelPhrases - GlobalPerms map[string]string - LocalPerms map[string]string - SettingLabels map[string]string - PermPresets map[string]string - Accounts map[string]string // TODO: Apply these phrases in the software proper - Errors map[string]map[string]string // map[category]map[name]value - PageTitles map[string]string + Name string + Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent. + Levels LevelPhrases + GlobalPerms map[string]string + LocalPerms map[string]string + SettingLabels map[string]string + PermPresets map[string]string + Accounts map[string]string // TODO: Apply these phrases in the software proper + UserAgents map[string]string + OperatingSystems map[string]string + Errors map[string]map[string]string // map[category]map[name]value + PageTitles map[string]string } // TODO: Add the ability to edit language JSON files from the Control Panel and automatically scan the files for changes @@ -154,6 +156,22 @@ func GetAccountPhrase(name string) string { return res } +func GetUserAgentPhrase(name string) (string, bool) { + res, ok := currentLangPack.Load().(*LanguagePack).UserAgents[name] + if !ok { + return "", false + } + return res, true +} + +func GetOSPhrase(name string) (string, bool) { + res, ok := currentLangPack.Load().(*LanguagePack).OperatingSystems[name] + if !ok { + return "", false + } + return res, true +} + // TODO: Does comma ok work with multi-dimensional maps? func GetErrorPhrase(category string, name string) string { res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name] diff --git a/common/requests.go b/common/requests.go new file mode 100644 index 00000000..9aeae62b --- /dev/null +++ b/common/requests.go @@ -0,0 +1,75 @@ +package common + +import ( + "sync" + "sync/atomic" +) + +// Add ReferrerItems here after they've had zero views for a while +var referrersToDelete = make(map[string]ReferrerDeletable) + +type ReferrerDeletable struct { + item *ReferrerItem + scheduledAt int64 //unixtime +} + +type ReferrerItem struct { + Counter int64 +} + +// ? We'll track referrer domains here rather than the exact URL they arrived from for now, we'll think about expanding later +// ? Referrers are fluid and ever-changing so we have to use string keys rather than 'enum' ints +type DefaultReferrerTracker struct { + odd map[string]*ReferrerItem + even map[string]*ReferrerItem + oddLock sync.RWMutex + evenLock sync.RWMutex +} + +func NewDefaultReferrerTracker() *DefaultReferrerTracker { + return &DefaultReferrerTracker{ + odd: make(map[string]*ReferrerItem), + even: make(map[string]*ReferrerItem), + } +} + +func (ref *DefaultReferrerTracker) Tick() (err error) { + for _, del := range referrersToDelete { + _ = del + // TODO: Calculate the gap between now and the times they were scheduled + } + // TODO: Run the queries and schedule zero view refs for deletion from memory + return nil +} + +func (ref *DefaultReferrerTracker) Bump(referrer string) { + if referrer == "" { + return + } + var refItem *ReferrerItem + + // Slightly crude and rudimentary, but it should give a basic degree of sharding + if referrer[0]%2 == 0 { + ref.evenLock.RLock() + refItem = ref.even[referrer] + ref.evenLock.RUnlock() + if ref != nil { + atomic.AddInt64(&refItem.Counter, 1) + } else { + ref.evenLock.Lock() + ref.even[referrer] = &ReferrerItem{Counter: 1} + ref.evenLock.Unlock() + } + } else { + ref.oddLock.RLock() + refItem = ref.odd[referrer] + ref.oddLock.RUnlock() + if ref != nil { + atomic.AddInt64(&refItem.Counter, 1) + } else { + ref.oddLock.Lock() + ref.odd[referrer] = &ReferrerItem{Counter: 1} + ref.oddLock.Unlock() + } + } +} diff --git a/gen_router.go b/gen_router.go index 88cc173f..34fc2649 100644 --- a/gen_router.go +++ b/gen_router.go @@ -56,8 +56,10 @@ var RouteMap = map[string]interface{}{ "routePanelAnalyticsViews": routePanelAnalyticsViews, "routePanelAnalyticsRoutes": routePanelAnalyticsRoutes, "routePanelAnalyticsAgents": routePanelAnalyticsAgents, + "routePanelAnalyticsSystems": routePanelAnalyticsSystems, "routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews, "routePanelAnalyticsAgentViews": routePanelAnalyticsAgentViews, + "routePanelAnalyticsSystemViews": routePanelAnalyticsSystemViews, "routePanelAnalyticsPosts": routePanelAnalyticsPosts, "routePanelAnalyticsTopics": routePanelAnalyticsTopics, "routePanelGroups": routePanelGroups, @@ -154,60 +156,62 @@ var routeMapEnum = map[string]int{ "routePanelAnalyticsViews": 37, "routePanelAnalyticsRoutes": 38, "routePanelAnalyticsAgents": 39, - "routePanelAnalyticsRouteViews": 40, - "routePanelAnalyticsAgentViews": 41, - "routePanelAnalyticsPosts": 42, - "routePanelAnalyticsTopics": 43, - "routePanelGroups": 44, - "routePanelGroupsEdit": 45, - "routePanelGroupsEditPerms": 46, - "routePanelGroupsEditSubmit": 47, - "routePanelGroupsEditPermsSubmit": 48, - "routePanelGroupsCreateSubmit": 49, - "routePanelBackups": 50, - "routePanelLogsMod": 51, - "routePanelDebug": 52, - "routePanelDashboard": 53, - "routes.AccountEditCritical": 54, - "routeAccountEditCriticalSubmit": 55, - "routeAccountEditAvatar": 56, - "routeAccountEditAvatarSubmit": 57, - "routeAccountEditUsername": 58, - "routeAccountEditUsernameSubmit": 59, - "routeAccountEditEmail": 60, - "routeAccountEditEmailTokenSubmit": 61, - "routeProfile": 62, - "routes.BanUserSubmit": 63, - "routes.UnbanUser": 64, - "routes.ActivateUser": 65, - "routes.IPSearch": 66, - "routes.CreateTopicSubmit": 67, - "routes.EditTopicSubmit": 68, - "routes.DeleteTopicSubmit": 69, - "routes.StickTopicSubmit": 70, - "routes.UnstickTopicSubmit": 71, - "routes.LockTopicSubmit": 72, - "routes.UnlockTopicSubmit": 73, - "routes.MoveTopicSubmit": 74, - "routeLikeTopicSubmit": 75, - "routes.ViewTopic": 76, - "routeCreateReplySubmit": 77, - "routes.ReplyEditSubmit": 78, - "routes.ReplyDeleteSubmit": 79, - "routeReplyLikeSubmit": 80, - "routeProfileReplyCreateSubmit": 81, - "routes.ProfileReplyEditSubmit": 82, - "routes.ProfileReplyDeleteSubmit": 83, - "routes.PollVote": 84, - "routes.PollResults": 85, - "routes.AccountLogin": 86, - "routes.AccountRegister": 87, - "routeLogout": 88, - "routes.AccountLoginSubmit": 89, - "routes.AccountRegisterSubmit": 90, - "routeDynamic": 91, - "routeUploads": 92, - "BadRoute": 93, + "routePanelAnalyticsSystems": 40, + "routePanelAnalyticsRouteViews": 41, + "routePanelAnalyticsAgentViews": 42, + "routePanelAnalyticsSystemViews": 43, + "routePanelAnalyticsPosts": 44, + "routePanelAnalyticsTopics": 45, + "routePanelGroups": 46, + "routePanelGroupsEdit": 47, + "routePanelGroupsEditPerms": 48, + "routePanelGroupsEditSubmit": 49, + "routePanelGroupsEditPermsSubmit": 50, + "routePanelGroupsCreateSubmit": 51, + "routePanelBackups": 52, + "routePanelLogsMod": 53, + "routePanelDebug": 54, + "routePanelDashboard": 55, + "routes.AccountEditCritical": 56, + "routeAccountEditCriticalSubmit": 57, + "routeAccountEditAvatar": 58, + "routeAccountEditAvatarSubmit": 59, + "routeAccountEditUsername": 60, + "routeAccountEditUsernameSubmit": 61, + "routeAccountEditEmail": 62, + "routeAccountEditEmailTokenSubmit": 63, + "routeProfile": 64, + "routes.BanUserSubmit": 65, + "routes.UnbanUser": 66, + "routes.ActivateUser": 67, + "routes.IPSearch": 68, + "routes.CreateTopicSubmit": 69, + "routes.EditTopicSubmit": 70, + "routes.DeleteTopicSubmit": 71, + "routes.StickTopicSubmit": 72, + "routes.UnstickTopicSubmit": 73, + "routes.LockTopicSubmit": 74, + "routes.UnlockTopicSubmit": 75, + "routes.MoveTopicSubmit": 76, + "routeLikeTopicSubmit": 77, + "routes.ViewTopic": 78, + "routeCreateReplySubmit": 79, + "routes.ReplyEditSubmit": 80, + "routes.ReplyDeleteSubmit": 81, + "routeReplyLikeSubmit": 82, + "routeProfileReplyCreateSubmit": 83, + "routes.ProfileReplyEditSubmit": 84, + "routes.ProfileReplyDeleteSubmit": 85, + "routes.PollVote": 86, + "routes.PollResults": 87, + "routes.AccountLogin": 88, + "routes.AccountRegister": 89, + "routeLogout": 90, + "routes.AccountLoginSubmit": 91, + "routes.AccountRegisterSubmit": 92, + "routeDynamic": 93, + "routeUploads": 94, + "BadRoute": 95, } var reverseRouteMapEnum = map[int]string{ 0: "routeAPI", @@ -250,60 +254,78 @@ var reverseRouteMapEnum = map[int]string{ 37: "routePanelAnalyticsViews", 38: "routePanelAnalyticsRoutes", 39: "routePanelAnalyticsAgents", - 40: "routePanelAnalyticsRouteViews", - 41: "routePanelAnalyticsAgentViews", - 42: "routePanelAnalyticsPosts", - 43: "routePanelAnalyticsTopics", - 44: "routePanelGroups", - 45: "routePanelGroupsEdit", - 46: "routePanelGroupsEditPerms", - 47: "routePanelGroupsEditSubmit", - 48: "routePanelGroupsEditPermsSubmit", - 49: "routePanelGroupsCreateSubmit", - 50: "routePanelBackups", - 51: "routePanelLogsMod", - 52: "routePanelDebug", - 53: "routePanelDashboard", - 54: "routes.AccountEditCritical", - 55: "routeAccountEditCriticalSubmit", - 56: "routeAccountEditAvatar", - 57: "routeAccountEditAvatarSubmit", - 58: "routeAccountEditUsername", - 59: "routeAccountEditUsernameSubmit", - 60: "routeAccountEditEmail", - 61: "routeAccountEditEmailTokenSubmit", - 62: "routeProfile", - 63: "routes.BanUserSubmit", - 64: "routes.UnbanUser", - 65: "routes.ActivateUser", - 66: "routes.IPSearch", - 67: "routes.CreateTopicSubmit", - 68: "routes.EditTopicSubmit", - 69: "routes.DeleteTopicSubmit", - 70: "routes.StickTopicSubmit", - 71: "routes.UnstickTopicSubmit", - 72: "routes.LockTopicSubmit", - 73: "routes.UnlockTopicSubmit", - 74: "routes.MoveTopicSubmit", - 75: "routeLikeTopicSubmit", - 76: "routes.ViewTopic", - 77: "routeCreateReplySubmit", - 78: "routes.ReplyEditSubmit", - 79: "routes.ReplyDeleteSubmit", - 80: "routeReplyLikeSubmit", - 81: "routeProfileReplyCreateSubmit", - 82: "routes.ProfileReplyEditSubmit", - 83: "routes.ProfileReplyDeleteSubmit", - 84: "routes.PollVote", - 85: "routes.PollResults", - 86: "routes.AccountLogin", - 87: "routes.AccountRegister", - 88: "routeLogout", - 89: "routes.AccountLoginSubmit", - 90: "routes.AccountRegisterSubmit", - 91: "routeDynamic", - 92: "routeUploads", - 93: "BadRoute", + 40: "routePanelAnalyticsSystems", + 41: "routePanelAnalyticsRouteViews", + 42: "routePanelAnalyticsAgentViews", + 43: "routePanelAnalyticsSystemViews", + 44: "routePanelAnalyticsPosts", + 45: "routePanelAnalyticsTopics", + 46: "routePanelGroups", + 47: "routePanelGroupsEdit", + 48: "routePanelGroupsEditPerms", + 49: "routePanelGroupsEditSubmit", + 50: "routePanelGroupsEditPermsSubmit", + 51: "routePanelGroupsCreateSubmit", + 52: "routePanelBackups", + 53: "routePanelLogsMod", + 54: "routePanelDebug", + 55: "routePanelDashboard", + 56: "routes.AccountEditCritical", + 57: "routeAccountEditCriticalSubmit", + 58: "routeAccountEditAvatar", + 59: "routeAccountEditAvatarSubmit", + 60: "routeAccountEditUsername", + 61: "routeAccountEditUsernameSubmit", + 62: "routeAccountEditEmail", + 63: "routeAccountEditEmailTokenSubmit", + 64: "routeProfile", + 65: "routes.BanUserSubmit", + 66: "routes.UnbanUser", + 67: "routes.ActivateUser", + 68: "routes.IPSearch", + 69: "routes.CreateTopicSubmit", + 70: "routes.EditTopicSubmit", + 71: "routes.DeleteTopicSubmit", + 72: "routes.StickTopicSubmit", + 73: "routes.UnstickTopicSubmit", + 74: "routes.LockTopicSubmit", + 75: "routes.UnlockTopicSubmit", + 76: "routes.MoveTopicSubmit", + 77: "routeLikeTopicSubmit", + 78: "routes.ViewTopic", + 79: "routeCreateReplySubmit", + 80: "routes.ReplyEditSubmit", + 81: "routes.ReplyDeleteSubmit", + 82: "routeReplyLikeSubmit", + 83: "routeProfileReplyCreateSubmit", + 84: "routes.ProfileReplyEditSubmit", + 85: "routes.ProfileReplyDeleteSubmit", + 86: "routes.PollVote", + 87: "routes.PollResults", + 88: "routes.AccountLogin", + 89: "routes.AccountRegister", + 90: "routeLogout", + 91: "routes.AccountLoginSubmit", + 92: "routes.AccountRegisterSubmit", + 93: "routeDynamic", + 94: "routeUploads", + 95: "BadRoute", +} +var osMapEnum = map[string]int{ + "unknown": 0, + "windows": 1, + "linux": 2, + "mac": 3, + "android": 4, + "iphone": 5, +} +var reverseOSMapEnum = map[int]string{ + 0: "unknown", + 1: "windows", + 2: "linux", + 3: "mac", + 4: "android", + 5: "iphone", } var agentMapEnum = map[string]int{ "unknown": 0, @@ -313,22 +335,27 @@ var agentMapEnum = map[string]int{ "safari": 4, "edge": 5, "internetexplorer": 6, - "androidchrome": 7, - "mobilesafari": 8, - "ucbrowser": 9, - "googlebot": 10, - "yandex": 11, - "bing": 12, - "baidu": 13, - "duckduckgo": 14, - "discord": 15, - "cloudflare": 16, - "uptimebot": 17, - "lynx": 18, - "blank": 19, - "malformed": 20, - "suspicious": 21, - "zgrab": 22, + "trident": 7, + "androidchrome": 8, + "mobilesafari": 9, + "samsung": 10, + "ucbrowser": 11, + "googlebot": 12, + "yandex": 13, + "bing": 14, + "baidu": 15, + "duckduckgo": 16, + "seznambot": 17, + "discord": 18, + "twitter": 19, + "cloudflare": 20, + "uptimebot": 21, + "discourse": 22, + "lynx": 23, + "blank": 24, + "malformed": 25, + "suspicious": 26, + "zgrab": 27, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -338,31 +365,37 @@ var reverseAgentMapEnum = map[int]string{ 4: "safari", 5: "edge", 6: "internetexplorer", - 7: "androidchrome", - 8: "mobilesafari", - 9: "ucbrowser", - 10: "googlebot", - 11: "yandex", - 12: "bing", - 13: "baidu", - 14: "duckduckgo", - 15: "discord", - 16: "cloudflare", - 17: "uptimebot", - 18: "lynx", - 19: "blank", - 20: "malformed", - 21: "suspicious", - 22: "zgrab", + 7: "trident", + 8: "androidchrome", + 9: "mobilesafari", + 10: "samsung", + 11: "ucbrowser", + 12: "googlebot", + 13: "yandex", + 14: "bing", + 15: "baidu", + 16: "duckduckgo", + 17: "seznambot", + 18: "discord", + 19: "twitter", + 20: "cloudflare", + 21: "uptimebot", + 22: "discourse", + 23: "lynx", + 24: "blank", + 25: "malformed", + 26: "suspicious", + 27: "zgrab", } var markToAgent = map[string]string{ "OPR":"opera", "Chrome":"chrome", "Firefox":"firefox", "MSIE":"internetexplorer", - //"Trident":"internetexplorer", + "Trident":"trident", // Hack to support IE11 "Edge":"edge", "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this + "SamsungBrowser":"samsung", "UCBrowser":"ucbrowser", "Google":"googlebot", @@ -372,9 +405,12 @@ var markToAgent = map[string]string{ "Baiduspider":"baidu", "bingbot":"bing", "BingPreview":"bing", + "SeznamBot":"seznambot", "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "Uptimebot":"uptimebot", "Discordbot":"discord", + "Twitterbot":"twitter", + "Discourse":"discourse", "zgrab":"zgrab", } @@ -390,6 +426,8 @@ func init() { common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetAgentMapEnum(agentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum) + common.SetOSMapEnum(osMapEnum) + common.SetReverseOSMapEnum(reverseOSMapEnum) } type GenRouter struct { @@ -456,7 +494,7 @@ func (router *GenRouter) DumpRequest(req *http.Request) { func (router *GenRouter) SuspiciousRequest(req *http.Request) { log.Print("Suspicious Request") router.DumpRequest(req) - common.AgentViewCounter.Bump(21) + common.AgentViewCounter.Bump(26) } // TODO: Pass the default route or config struct to the router rather than accessing it via a package global @@ -484,7 +522,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.Write([]byte("")) log.Print("Malformed Request") router.DumpRequest(req) - common.AgentViewCounter.Bump(20) + common.AgentViewCounter.Bump(25) return } @@ -512,19 +550,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before routes.StaticFile") - log.Print("Method: ", req.Method) - for key, value := range req.Header { - for _, vvalue := range value { - log.Print("Header '" + key + "': " + vvalue + "!!") - } - } - log.Print("prefix: ", prefix) - log.Print("req.Host: ", req.Host) - log.Print("req.URL.Path: ", req.URL.Path) - log.Print("req.URL.RawQuery: ", req.URL.RawQuery) - log.Print("extraData: ", extraData) - log.Print("req.Referer(): ", req.Referer()) - log.Print("req.RemoteAddr: ", req.RemoteAddr) + router.DumpRequest(req) } if prefix == "/static" { @@ -544,12 +570,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Use a more efficient detector instead of smashing every possible combination in ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another if ua == "" { - common.AgentViewCounter.Bump(19) + common.AgentViewCounter.Bump(24) if common.Dev.DebugMode { log.Print("Blank UA: ", req.UserAgent()) router.DumpRequest(req) } } else { + var runeEquals = func(a []rune, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, item := range a { + if item != b[i] { + return false + } + } + return true + } + // WIP UA Parser var indices []int var items []string @@ -557,10 +595,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for index, item := range ua { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) - buffer = buffer[:0] + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' { + if len(buffer) != 0 { + items = append(items, string(buffer)) + indices = append(indices, index - 1) + buffer = buffer[:0] + } + } else { + // TODO: Test this + items = items[:0] + indices = indices[:0] + router.SuspiciousRequest(req) + log.Print("UA Buffer: ", buffer) + log.Print("UA Buffer String: ", string(buffer)) + break } } @@ -575,19 +623,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + if common.Dev.SuperDebug { + log.Print("parsed agent: ", agent) + } - if common.Dev.DebugMode { - log.Print("parsed agent: ",agent) + var os string + for _, mark := range items { + switch(mark) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + } + } + if os == "" { + os = "unknown" + } + if common.Dev.SuperDebug { + log.Print("os: ", os) + log.Printf("items: %+v\n",items) } // Special handling switch(agent) { case "chrome": - for _, mark := range items { - if mark == "Android" { - agent = "androidchrome" - break - } + if os == "android" { + agent = "androidchrome" + } + case "safari": + if os == "iphone" { + agent = "mobilesafari" + } + case "trident": + // Hack to support IE11, change this after we start logging versions + if strings.Contains(ua,"rv:11") { + agent = "internetexplorer" } case "zgrab": router.SuspiciousRequest(req) @@ -602,6 +678,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { common.AgentViewCounter.Bump(agentMapEnum[agent]) } + common.OSViewCounter.Bump(osMapEnum[os]) } // Deal with the session stuff, etc. @@ -925,12 +1002,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { common.RouteViewCounter.Bump(39) err = routePanelAnalyticsAgents(w,req,user) - case "/panel/analytics/route/": + case "/panel/analytics/systems/": + err = common.ParseForm(w,req,user) + if err != nil { + router.handleError(err,w,req,user) + return + } + common.RouteViewCounter.Bump(40) + err = routePanelAnalyticsSystems(w,req,user) + case "/panel/analytics/route/": + common.RouteViewCounter.Bump(41) err = routePanelAnalyticsRouteViews(w,req,user,extraData) case "/panel/analytics/agent/": - common.RouteViewCounter.Bump(41) + common.RouteViewCounter.Bump(42) err = routePanelAnalyticsAgentViews(w,req,user,extraData) + case "/panel/analytics/system/": + common.RouteViewCounter.Bump(43) + err = routePanelAnalyticsSystemViews(w,req,user,extraData) case "/panel/analytics/posts/": err = common.ParseForm(w,req,user) if err != nil { @@ -938,7 +1027,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(42) + common.RouteViewCounter.Bump(44) err = routePanelAnalyticsPosts(w,req,user) case "/panel/analytics/topics/": err = common.ParseForm(w,req,user) @@ -947,16 +1036,16 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(43) + common.RouteViewCounter.Bump(45) err = routePanelAnalyticsTopics(w,req,user) case "/panel/groups/": - common.RouteViewCounter.Bump(44) + common.RouteViewCounter.Bump(46) err = routePanelGroups(w,req,user) case "/panel/groups/edit/": - common.RouteViewCounter.Bump(45) + common.RouteViewCounter.Bump(47) err = routePanelGroupsEdit(w,req,user,extraData) case "/panel/groups/edit/perms/": - common.RouteViewCounter.Bump(46) + common.RouteViewCounter.Bump(48) err = routePanelGroupsEditPerms(w,req,user,extraData) case "/panel/groups/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -965,7 +1054,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(47) + common.RouteViewCounter.Bump(49) err = routePanelGroupsEditSubmit(w,req,user,extraData) case "/panel/groups/edit/perms/submit/": err = common.NoSessionMismatch(w,req,user) @@ -974,7 +1063,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(48) + common.RouteViewCounter.Bump(50) err = routePanelGroupsEditPermsSubmit(w,req,user,extraData) case "/panel/groups/create/": err = common.NoSessionMismatch(w,req,user) @@ -983,7 +1072,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(49) + common.RouteViewCounter.Bump(51) err = routePanelGroupsCreateSubmit(w,req,user) case "/panel/backups/": err = common.SuperAdminOnly(w,req,user) @@ -992,10 +1081,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(50) + common.RouteViewCounter.Bump(52) err = routePanelBackups(w,req,user,extraData) case "/panel/logs/mod/": - common.RouteViewCounter.Bump(51) + common.RouteViewCounter.Bump(53) err = routePanelLogsMod(w,req,user) case "/panel/debug/": err = common.AdminOnly(w,req,user) @@ -1004,10 +1093,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(52) + common.RouteViewCounter.Bump(54) err = routePanelDebug(w,req,user) default: - common.RouteViewCounter.Bump(53) + common.RouteViewCounter.Bump(55) err = routePanelDashboard(w,req,user) } if err != nil { @@ -1022,7 +1111,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(54) + common.RouteViewCounter.Bump(56) err = routes.AccountEditCritical(w,req,user) case "/user/edit/critical/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1037,7 +1126,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(55) + common.RouteViewCounter.Bump(57) err = routeAccountEditCriticalSubmit(w,req,user) case "/user/edit/avatar/": err = common.MemberOnly(w,req,user) @@ -1046,7 +1135,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(56) + common.RouteViewCounter.Bump(58) err = routeAccountEditAvatar(w,req,user) case "/user/edit/avatar/submit/": err = common.MemberOnly(w,req,user) @@ -1066,7 +1155,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(57) + common.RouteViewCounter.Bump(59) err = routeAccountEditAvatarSubmit(w,req,user) case "/user/edit/username/": err = common.MemberOnly(w,req,user) @@ -1075,7 +1164,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(58) + common.RouteViewCounter.Bump(60) err = routeAccountEditUsername(w,req,user) case "/user/edit/username/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1090,7 +1179,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(59) + common.RouteViewCounter.Bump(61) err = routeAccountEditUsernameSubmit(w,req,user) case "/user/edit/email/": err = common.MemberOnly(w,req,user) @@ -1099,7 +1188,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(60) + common.RouteViewCounter.Bump(62) err = routeAccountEditEmail(w,req,user) case "/user/edit/token/": err = common.NoSessionMismatch(w,req,user) @@ -1114,11 +1203,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(61) + common.RouteViewCounter.Bump(63) err = routeAccountEditEmailTokenSubmit(w,req,user,extraData) default: req.URL.Path += extraData - common.RouteViewCounter.Bump(62) + common.RouteViewCounter.Bump(64) err = routeProfile(w,req,user) } if err != nil { @@ -1139,7 +1228,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(63) + common.RouteViewCounter.Bump(65) err = routes.BanUserSubmit(w,req,user,extraData) case "/users/unban/": err = common.NoSessionMismatch(w,req,user) @@ -1154,7 +1243,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(64) + common.RouteViewCounter.Bump(66) err = routes.UnbanUser(w,req,user,extraData) case "/users/activate/": err = common.NoSessionMismatch(w,req,user) @@ -1169,7 +1258,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(65) + common.RouteViewCounter.Bump(67) err = routes.ActivateUser(w,req,user,extraData) case "/users/ips/": err = common.MemberOnly(w,req,user) @@ -1178,7 +1267,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(66) + common.RouteViewCounter.Bump(68) err = routes.IPSearch(w,req,user) } if err != nil { @@ -1204,7 +1293,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(67) + common.RouteViewCounter.Bump(69) err = routes.CreateTopicSubmit(w,req,user) case "/topic/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1219,7 +1308,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(68) + common.RouteViewCounter.Bump(70) err = routes.EditTopicSubmit(w,req,user,extraData) case "/topic/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1235,7 +1324,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } req.URL.Path += extraData - common.RouteViewCounter.Bump(69) + common.RouteViewCounter.Bump(71) err = routes.DeleteTopicSubmit(w,req,user) case "/topic/stick/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1250,7 +1339,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(70) + common.RouteViewCounter.Bump(72) err = routes.StickTopicSubmit(w,req,user,extraData) case "/topic/unstick/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1265,7 +1354,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(71) + common.RouteViewCounter.Bump(73) err = routes.UnstickTopicSubmit(w,req,user,extraData) case "/topic/lock/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1281,7 +1370,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } req.URL.Path += extraData - common.RouteViewCounter.Bump(72) + common.RouteViewCounter.Bump(74) err = routes.LockTopicSubmit(w,req,user) case "/topic/unlock/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1296,7 +1385,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(73) + common.RouteViewCounter.Bump(75) err = routes.UnlockTopicSubmit(w,req,user,extraData) case "/topic/move/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1311,7 +1400,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(74) + common.RouteViewCounter.Bump(76) err = routes.MoveTopicSubmit(w,req,user,extraData) case "/topic/like/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1326,10 +1415,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(75) + common.RouteViewCounter.Bump(77) err = routeLikeTopicSubmit(w,req,user,extraData) default: - common.RouteViewCounter.Bump(76) + common.RouteViewCounter.Bump(78) err = routes.ViewTopic(w,req,user, extraData) } if err != nil { @@ -1355,7 +1444,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(77) + common.RouteViewCounter.Bump(79) err = routeCreateReplySubmit(w,req,user) case "/reply/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1370,7 +1459,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(78) + common.RouteViewCounter.Bump(80) err = routes.ReplyEditSubmit(w,req,user,extraData) case "/reply/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1385,7 +1474,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(79) + common.RouteViewCounter.Bump(81) err = routes.ReplyDeleteSubmit(w,req,user,extraData) case "/reply/like/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1400,7 +1489,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(80) + common.RouteViewCounter.Bump(82) err = routeReplyLikeSubmit(w,req,user,extraData) } if err != nil { @@ -1421,7 +1510,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(81) + common.RouteViewCounter.Bump(83) err = routeProfileReplyCreateSubmit(w,req,user) case "/profile/reply/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1436,7 +1525,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(82) + common.RouteViewCounter.Bump(84) err = routes.ProfileReplyEditSubmit(w,req,user,extraData) case "/profile/reply/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1451,7 +1540,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(83) + common.RouteViewCounter.Bump(85) err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData) } if err != nil { @@ -1472,10 +1561,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(84) + common.RouteViewCounter.Bump(86) err = routes.PollVote(w,req,user,extraData) case "/poll/results/": - common.RouteViewCounter.Bump(85) + common.RouteViewCounter.Bump(87) err = routes.PollResults(w,req,user,extraData) } if err != nil { @@ -1484,10 +1573,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { case "/accounts": switch(req.URL.Path) { case "/accounts/login/": - common.RouteViewCounter.Bump(86) + common.RouteViewCounter.Bump(88) err = routes.AccountLogin(w,req,user) case "/accounts/create/": - common.RouteViewCounter.Bump(87) + common.RouteViewCounter.Bump(89) err = routes.AccountRegister(w,req,user) case "/accounts/logout/": err = common.NoSessionMismatch(w,req,user) @@ -1502,7 +1591,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(88) + common.RouteViewCounter.Bump(90) err = routeLogout(w,req,user) case "/accounts/login/submit/": err = common.ParseForm(w,req,user) @@ -1511,7 +1600,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(89) + common.RouteViewCounter.Bump(91) err = routes.AccountLoginSubmit(w,req,user) case "/accounts/create/submit/": err = common.ParseForm(w,req,user) @@ -1520,7 +1609,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(90) + common.RouteViewCounter.Bump(92) err = routes.AccountRegisterSubmit(w,req,user) } if err != nil { @@ -1537,7 +1626,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { common.NotFound(w,req) return } - common.RouteViewCounter.Bump(92) + common.RouteViewCounter.Bump(94) req.URL.Path += extraData // TODO: Find a way to propagate errors up from this? router.UploadHandler(w,req) // TODO: Count these views @@ -1580,7 +1669,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { router.RUnlock() if ok { - common.RouteViewCounter.Bump(91) // TODO: Be more specific about *which* dynamic route it is + common.RouteViewCounter.Bump(93) // TODO: Be more specific about *which* dynamic route it is req.URL.Path += extraData err = handle(w,req,user) if err != nil { @@ -1591,10 +1680,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Log all bad routes for the admin to figure out where users are going wrong? lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { + if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { router.SuspiciousRequest(req) } - common.RouteViewCounter.Bump(93) + common.RouteViewCounter.Bump(95) common.NotFound(w,req) } } diff --git a/langs/english.json b/langs/english.json index cc3ae649..007aafed 100644 --- a/langs/english.json +++ b/langs/english.json @@ -89,5 +89,45 @@ "panel_mod_logs":"Moderation Logs", "panel_admin_logs":"Administration Logs", "panel_debug":"Debug" + }, + + "UserAgents": { + "chrome": "Google Chrome", + "firefox":"Mozilla Firefox", + "opera":"Opera", + "safari":"Safari", + "edge": "Edge", + "internetexplorer":"MS Internet Explorer", + "trident":"Trident Engine", + "androidchrome":"Chrome for Android", + "mobilesafari":"Mobile Safari", + "samsung":"Samsung Browser", + "ucbrowser":"UCBrowser", + + "googlebot":"Googlebot", + "yandex":"Yandex", + "bing":"Bing", + "baidu":"Baidu", + "duckduckgo":"DuckDuckBot", + "seznambot":"SeznamBot", + "discord":"Discord", + "twitter":"Twitterbot", + "cloudflare":"Cloudflare Alwayson", + "uptimebot":"Uptimebot", + "discourse":"Discourse Forum Onebox", + "lynx":"Lynx", + + "zgrab":"Zgrab Application Scanner", + "suspicious":"Suspicious", + "unknown":"Unknown", + "blank":"Blank", + "malformed":"Malformed" + }, + "OperatingSystems": { + "windows": "Microsoft Windows", + "linux":"Linux", + "android": "Android", + "iphone":"iPhone", + "unknown":"Unknown" } } \ No newline at end of file diff --git a/main.go b/main.go index f171f810..96ebbfcd 100644 --- a/main.go +++ b/main.go @@ -105,6 +105,10 @@ func afterDBInit() (err error) { if err != nil { return err } + common.OSViewCounter, err = common.NewDefaultOSViewCounter() + if err != nil { + return err + } common.RouteViewCounter, err = common.NewDefaultRouteViewCounter() if err != nil { return err diff --git a/panel_routes.go b/panel_routes.go index 412e0098..474cef33 100644 --- a/panel_routes.go +++ b/panel_routes.go @@ -799,11 +799,14 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -822,10 +825,98 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} log.Printf("graph: %+v\n", graph) - pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", html.EscapeString(agent), graph, timeRange.Range} + // ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff + agent = html.EscapeString(agent) + friendlyAgent, ok := common.GetUserAgentPhrase(agent) + if !ok { + friendlyAgent = agent + } + + pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", agent, friendlyAgent, graph, timeRange.Range} return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi) } +func routePanelAnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.User, system string) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + headerVars.Stylesheets = append(headerVars.Stylesheets, "chartist/chartist.min.css") + headerVars.Scripts = append(headerVars.Scripts, "chartist/chartist.min.js") + + timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange")) + if err != nil { + return common.LocalError(err.Error(), w, r, user) + } + + var revLabelList []int64 + var labelList []int64 + var viewMap = make(map[int64]int64) + var currentTime = time.Now().Unix() + + for i := 1; i <= timeRange.Slices; i++ { + var label = currentTime - int64(i*timeRange.SliceWidth) + revLabelList = append(revLabelList, label) + viewMap[label] = 0 + } + for _, value := range revLabelList { + labelList = append(labelList, value) + } + + var viewList []int64 + log.Print("in routePanelAnalyticsSystemViews") + + acc := qgen.Builder.Accumulator() + // TODO: Verify the agent is valid + rows, err := acc.Select("viewchunks_systems").Columns("count, createdAt").Where("system = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system) + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int64 + var createdAt time.Time + err := rows.Scan(&count, &createdAt) + if err != nil { + return common.InternalError(err, w, r) + } + + var unixCreatedAt = createdAt.Unix() + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + + for _, value := range labelList { + if unixCreatedAt > value { + viewMap[value] += count + break + } + } + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + for _, value := range revLabelList { + viewList = append(viewList, viewMap[value]) + } + graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + log.Printf("graph: %+v\n", graph) + + system = html.EscapeString(system) + friendlySystem, ok := common.GetOSPhrase(system) + if !ok { + friendlySystem = system + } + + pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", system, friendlySystem, graph, timeRange.Range} + return panelRenderTemplate("panel_analytics_system_views", w, r, user, &pi) +} + func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { @@ -870,11 +961,14 @@ func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user comm if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -943,11 +1037,14 @@ func routePanelAnalyticsPosts(w http.ResponseWriter, r *http.Request, user commo if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -999,8 +1096,10 @@ func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user comm return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("route: ", route) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("route: ", route) + } routeMap[route] += count } err = rows.Err() @@ -1048,8 +1147,10 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("agent: ", agent) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("agent: ", agent) + } agentMap[agent] += count } err = rows.Err() @@ -1060,9 +1161,14 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm // TODO: Sort this slice var agentItems []common.PanelAnalyticsAgentsItem for agent, count := range agentMap { + aAgent, ok := common.GetUserAgentPhrase(agent) + if !ok { + aAgent = agent + } agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{ - Agent: agent, - Count: count, + Agent: agent, + FriendlyAgent: aAgent, + Count: count, }) } @@ -1070,6 +1176,62 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm return panelRenderTemplate("panel_analytics_agents", w, r, user, &pi) } +func routePanelAnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + var osMap = make(map[string]int) + + timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange")) + if err != nil { + return common.LocalError(err.Error(), w, r, user) + } + + acc := qgen.Builder.Accumulator() + rows, err := acc.Select("viewchunks_systems").Columns("count, system").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int + var system string + err := rows.Scan(&count, &system) + if err != nil { + return common.InternalError(err, w, r) + } + + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("system: ", system) + } + osMap[system] += count + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + // TODO: Sort this slice + var systemItems []common.PanelAnalyticsAgentsItem + for system, count := range osMap { + sSystem, ok := common.GetOSPhrase(system) + if !ok { + sSystem = system + } + systemItems = append(systemItems, common.PanelAnalyticsAgentsItem{ + Agent: system, + FriendlyAgent: sSystem, + Count: count, + }) + } + + pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", systemItems, timeRange.Range} + return panelRenderTemplate("panel_analytics_systems", w, r, user, &pi) +} + func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { diff --git a/query_gen/tables.go b/query_gen/tables.go index 4b191fb3..6a9b433a 100644 --- a/query_gen/tables.go +++ b/query_gen/tables.go @@ -412,6 +412,15 @@ func createTables(adapter qgen.Adapter) error { []qgen.DBTableKey{}, ) + qgen.Install.CreateTable("viewchunks_systems", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"count", "int", 0, false, false, "0"}, + qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""}, + qgen.DBTableColumn{"system", "varchar", 200, false, false, ""}, // windows, android, bot, etc. + }, + []qgen.DBTableKey{}, + ) + /* qgen.Install.CreateTable("viewchunks_forums", "", "", []qgen.DBTableColumn{ diff --git a/router_gen/main.go b/router_gen/main.go index 6697a5c1..3f4f271c 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -19,6 +19,8 @@ type TmplVars struct { AllRouteMap map[string]int AllAgentNames []string AllAgentMap map[string]int + AllOSNames []string + AllOSMap map[string]int } func main() { @@ -158,6 +160,20 @@ func main() { mapIt("BadRoute") tmplVars.AllRouteNames = allRouteNames tmplVars.AllRouteMap = allRouteMap + + tmplVars.AllOSNames = []string{ + "unknown", + "windows", + "linux", + "mac", + "android", + "iphone", + } + tmplVars.AllOSMap = make(map[string]int) + for id, os := range tmplVars.AllOSNames { + tmplVars.AllOSMap[os] = id + } + tmplVars.AllAgentNames = []string{ "unknown", "firefox", @@ -166,9 +182,11 @@ func main() { "safari", "edge", "internetexplorer", + "trident", // Hack to support IE11 "androidchrome", - "mobilesafari", // Coming soon + "mobilesafari", + "samsung", "ucbrowser", "googlebot", @@ -176,9 +194,12 @@ func main() { "bing", "baidu", "duckduckgo", + "seznambot", "discord", + "twitter", "cloudflare", "uptimebot", + "discourse", "lynx", "blank", "malformed", @@ -219,6 +240,12 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} {{$index}}: "{{$element}}",{{end}} } +var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}} + "{{$element}}": {{$index}},{{end}} +} +var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}} + {{$index}}: "{{$element}}",{{end}} +} var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} "{{$element}}": {{$index}},{{end}} } @@ -230,9 +257,10 @@ var markToAgent = map[string]string{ "Chrome":"chrome", "Firefox":"firefox", "MSIE":"internetexplorer", - //"Trident":"internetexplorer", + "Trident":"trident", // Hack to support IE11 "Edge":"edge", "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this + "SamsungBrowser":"samsung", "UCBrowser":"ucbrowser", "Google":"googlebot", @@ -242,9 +270,12 @@ var markToAgent = map[string]string{ "Baiduspider":"baidu", "bingbot":"bing", "BingPreview":"bing", + "SeznamBot":"seznambot", "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "Uptimebot":"uptimebot", "Discordbot":"discord", + "Twitterbot":"twitter", + "Discourse":"discourse", "zgrab":"zgrab", } @@ -260,6 +291,8 @@ func init() { common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetAgentMapEnum(agentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum) + common.SetOSMapEnum(osMapEnum) + common.SetReverseOSMapEnum(reverseOSMapEnum) } type GenRouter struct { @@ -382,19 +415,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before routes.StaticFile") - log.Print("Method: ", req.Method) - for key, value := range req.Header { - for _, vvalue := range value { - log.Print("Header '" + key + "': " + vvalue + "!!") - } - } - log.Print("prefix: ", prefix) - log.Print("req.Host: ", req.Host) - log.Print("req.URL.Path: ", req.URL.Path) - log.Print("req.URL.RawQuery: ", req.URL.RawQuery) - log.Print("extraData: ", extraData) - log.Print("req.Referer(): ", req.Referer()) - log.Print("req.RemoteAddr: ", req.RemoteAddr) + router.DumpRequest(req) } if prefix == "/static" { @@ -420,6 +441,18 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { router.DumpRequest(req) } } else { + var runeEquals = func(a []rune, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, item := range a { + if item != b[i] { + return false + } + } + return true + } + // WIP UA Parser var indices []int var items []string @@ -427,10 +460,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for index, item := range ua { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) - buffer = buffer[:0] + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' { + if len(buffer) != 0 { + items = append(items, string(buffer)) + indices = append(indices, index - 1) + buffer = buffer[:0] + } + } else { + // TODO: Test this + items = items[:0] + indices = indices[:0] + router.SuspiciousRequest(req) + log.Print("UA Buffer: ", buffer) + log.Print("UA Buffer String: ", string(buffer)) + break } } @@ -445,19 +488,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + if common.Dev.SuperDebug { + log.Print("parsed agent: ", agent) + } - if common.Dev.DebugMode { - log.Print("parsed agent: ",agent) + var os string + for _, mark := range items { + switch(mark) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + } + } + if os == "" { + os = "unknown" + } + if common.Dev.SuperDebug { + log.Print("os: ", os) + log.Printf("items: %+v\n",items) } // Special handling switch(agent) { case "chrome": - for _, mark := range items { - if mark == "Android" { - agent = "androidchrome" - break - } + if os == "android" { + agent = "androidchrome" + } + case "safari": + if os == "iphone" { + agent = "mobilesafari" + } + case "trident": + // Hack to support IE11, change this after we start logging versions + if strings.Contains(ua,"rv:11") { + agent = "internetexplorer" } case "zgrab": router.SuspiciousRequest(req) @@ -472,6 +543,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { common.AgentViewCounter.Bump(agentMapEnum[agent]) } + common.OSViewCounter.Bump(osMapEnum[os]) } // Deal with the session stuff, etc. @@ -551,7 +623,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Log all bad routes for the admin to figure out where users are going wrong? lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { + if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { router.SuspiciousRequest(req) } common.RouteViewCounter.Bump({{.AllRouteMap.BadRoute}}) diff --git a/router_gen/routes.go b/router_gen/routes.go index 812db555..ada460d0 100644 --- a/router_gen/routes.go +++ b/router_gen/routes.go @@ -166,8 +166,10 @@ func buildPanelRoutes() { View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"), View("routePanelAnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"), View("routePanelAnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"), + View("routePanelAnalyticsSystems", "/panel/analytics/systems/").Before("ParseForm"), View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"), View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"), + View("routePanelAnalyticsSystemViews", "/panel/analytics/system/", "extraData"), View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"), View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"), diff --git a/schema/mssql/query_viewchunks_systems.sql b/schema/mssql/query_viewchunks_systems.sql new file mode 100644 index 00000000..09db462d --- /dev/null +++ b/schema/mssql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE [viewchunks_systems] ( + [count] int DEFAULT 0 not null, + [createdAt] datetime not null, + [system] nvarchar (200) not null +); \ No newline at end of file diff --git a/schema/mysql/query_viewchunks_systems.sql b/schema/mysql/query_viewchunks_systems.sql new file mode 100644 index 00000000..f65c2fb5 --- /dev/null +++ b/schema/mysql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_systems` ( + `count` int DEFAULT 0 not null, + `createdAt` datetime not null, + `system` varchar(200) not null +); \ No newline at end of file diff --git a/schema/pgsql/query_viewchunks_systems.sql b/schema/pgsql/query_viewchunks_systems.sql new file mode 100644 index 00000000..7ae425a7 --- /dev/null +++ b/schema/pgsql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_systems` ( + `count` int DEFAULT 0 not null, + `createdAt` timestamp not null, + `system` varchar (200) not null +); \ No newline at end of file diff --git a/templates/panel-inner-menu.html b/templates/panel-inner-menu.html index cf144959..ff831c7c 100644 --- a/templates/panel-inner-menu.html +++ b/templates/panel-inner-menu.html @@ -41,6 +41,9 @@
Agents
+
+ Systems +
Crawlers
diff --git a/templates/panel_analytics_agent_views.html b/templates/panel_analytics_agent_views.html index 8186a913..9bcdef67 100644 --- a/templates/panel_analytics_agent_views.html +++ b/templates/panel_analytics_agent_views.html @@ -5,7 +5,7 @@
- {{.Agent}} Views + {{.FriendlyAgent}} Views + + + + + + +
+
+
+
+
+
+ + + +{{template "footer.html" . }} diff --git a/templates/panel_analytics_systems.html b/templates/panel_analytics_systems.html new file mode 100644 index 00000000..36997967 --- /dev/null +++ b/templates/panel_analytics_systems.html @@ -0,0 +1,29 @@ +{{template "header.html" . }} +
+{{template "panel-menu.html" . }} +
+
+
+
+ Operating Systems + +
+
+
+
+ {{range .ItemList}} +
+ {{.FriendlyAgent}} + {{.Count}} views +
+ {{end}} +
+
+
+{{template "footer.html" . }}