Added the global topic counter and associated graphs.

Wildcards should work properly in robots.txt now
Fixed the padding on the registration and login pages for Cosora.
Moved the Websockets route into the new router.
We now log more suspicious requests.

Started moving routes into /routes/
Added the topicchunks table.

bot_routes.go

@@ -15,11 +15,11 @@ import (
 func routeRobotsTxt(w http.ResponseWriter, r *http.Request) common.RouteError {
 	// TODO: Do we have to put * or something at the end of the paths?
 	_, _ = w.Write([]byte(`User-agent: *
-Disallow: /panel/
+Disallow: /panel/*
 Disallow: /topics/create/
-Disallow: /user/edit/
-Disallow: /accounts/
-Disallow: /report/
+Disallow: /user/edit/*
+Disallow: /accounts/*
+Disallow: /report/*
 `))
 	return nil
 }

common/counters.go

@@ -13,6 +13,7 @@ var GlobalViewCounter *DefaultViewCounter
 var AgentViewCounter *DefaultAgentViewCounter
 var RouteViewCounter *DefaultRouteViewCounter
 var PostCounter *DefaultPostCounter
+var TopicCounter *DefaultTopicCounter
 
 // Local counters
 var TopicViewCounter *DefaultTopicViewCounter
@@ -111,6 +112,53 @@ func (counter *DefaultPostCounter) insertChunk(count int64) error {
 	return err
 }
 
+type DefaultTopicCounter struct {
+	buckets       [2]int64
+	currentBucket int64
+
+	insert *sql.Stmt
+}
+
+func NewTopicCounter() (*DefaultTopicCounter, error) {
+	acc := qgen.Builder.Accumulator()
+	counter := &DefaultTopicCounter{
+		currentBucket: 0,
+		insert:        acc.Insert("topicchunks").Columns("count, createdAt").Fields("?,UTC_TIMESTAMP()").Prepare(),
+	}
+	AddScheduledFifteenMinuteTask(counter.Tick)
+	//AddScheduledSecondTask(counter.Tick)
+	AddShutdownTask(counter.Tick)
+	return counter, acc.FirstError()
+}
+
+func (counter *DefaultTopicCounter) Tick() (err error) {
+	var oldBucket = counter.currentBucket
+	var nextBucket int64 // 0
+	if counter.currentBucket == 0 {
+		nextBucket = 1
+	}
+	atomic.AddInt64(&counter.buckets[oldBucket], counter.buckets[nextBucket])
+	atomic.StoreInt64(&counter.buckets[nextBucket], 0)
+	atomic.StoreInt64(&counter.currentBucket, nextBucket)
+
+	var previousViewChunk = counter.buckets[oldBucket]
+	atomic.AddInt64(&counter.buckets[oldBucket], -previousViewChunk)
+	return counter.insertChunk(previousViewChunk)
+}
+
+func (counter *DefaultTopicCounter) Bump() {
+	atomic.AddInt64(&counter.buckets[counter.currentBucket], 1)
+}
+
+func (counter *DefaultTopicCounter) insertChunk(count int64) error {
+	if count == 0 {
+		return nil
+	}
+	debugLogf("Inserting a topicchunk with a count of %d", count)
+	_, err := counter.insert.Exec(count)
+	return err
+}
+
 type RWMutexCounterBucket struct {
 	counter int
 	sync.RWMutex

main.go

@@ -96,6 +96,10 @@ func afterDBInit() (err error) {
 	if err != nil {
 		return err
 	}
+	common.TopicCounter, err = common.NewTopicCounter()
+	if err != nil {
+		return err
+	}
 	common.TopicViewCounter, err = common.NewDefaultTopicViewCounter()
 	if err != nil {
 		return err
@@ -300,10 +304,8 @@ func main() {
 		}
 	}()
 
-	// TODO: Move these routes into the new routes list
 	log.Print("Initialising the router")
 	router = NewGenRouter(http.FileServer(http.Dir("./uploads")))
-	router.HandleFunc("/ws/", routeWebsockets)
 
 	log.Print("Initialising the plugins")
 	common.InitPlugins()

member_routes.go

@@ -243,6 +243,7 @@ func routeTopicCreateSubmit(w http.ResponseWriter, r *http.Request, user common.
 	}
 
 	common.PostCounter.Bump()
+	common.TopicCounter.Bump()
 	http.Redirect(w, r, "/topic/"+strconv.Itoa(tid), http.StatusSeeOther)
 	return nil
 }

mod_routes.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	//"log"
-	//"fmt"
 	"encoding/json"
 	"html"
 	"log"
@@ -13,50 +11,6 @@ import (
 	"./common"
 )
 
-// TODO: Update the stats after edits so that we don't under or over decrement stats during deletes
-// TODO: Disable stat updates in posts handled by plugin_guilds
-func routeEditTopicSubmit(w http.ResponseWriter, r *http.Request, user common.User, stid string) common.RouteError {
-	isJs := (r.PostFormValue("js") == "1")
-
-	tid, err := strconv.Atoi(stid)
-	if err != nil {
-		return common.PreErrorJSQ("The provided TopicID is not a valid number.", w, r, isJs)
-	}
-
-	topic, err := common.Topics.Get(tid)
-	if err == ErrNoRows {
-		return common.PreErrorJSQ("The topic you tried to edit doesn't exist.", w, r, isJs)
-	} else if err != nil {
-		return common.InternalErrorJSQ(err, w, r, isJs)
-	}
-
-	// TODO: Add hooks to make use of headerLite
-	_, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID)
-	if ferr != nil {
-		return ferr
-	}
-	if !user.Perms.ViewTopic || !user.Perms.EditTopic {
-		return common.NoPermissionsJSQ(w, r, user, isJs)
-	}
-
-	err = topic.Update(r.PostFormValue("topic_name"), r.PostFormValue("topic_content"))
-	if err != nil {
-		return common.InternalErrorJSQ(err, w, r, isJs)
-	}
-
-	err = common.Forums.UpdateLastTopic(topic.ID, user.ID, topic.ParentID)
-	if err != nil && err != ErrNoRows {
-		return common.InternalErrorJSQ(err, w, r, isJs)
-	}
-
-	if !isJs {
-		http.Redirect(w, r, "/topic/"+strconv.Itoa(tid), http.StatusSeeOther)
-	} else {
-		_, _ = w.Write(successJSONBytes)
-	}
-	return nil
-}
-
 // TODO: Add support for soft-deletion and add a permission for hard delete in addition to the usual
 // TODO: Disable stat updates in posts handled by plugin_guilds
 func routeDeleteTopicSubmit(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
@@ -462,7 +416,7 @@ func routeReplyDeleteSubmit(w http.ResponseWriter, r *http.Request, user common.
 
 	//log.Printf("Reply #%d was deleted by common.User #%d", rid, user.ID)
 	if !isJs {
-		//http.Redirect(w,r, "/topic/" + strconv.Itoa(tid), http.StatusSeeOther)
+		http.Redirect(w, r, "/topic/"+strconv.Itoa(reply.ParentID), http.StatusSeeOther)
 	} else {
 		w.Write(successJSONBytes)
 	}

panel_routes.go

@@ -826,6 +826,79 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user
 	return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi)
 }
 
+func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
+	headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
+	if ferr != nil {
+		return ferr
+	}
+	headerVars.Stylesheets = append(headerVars.Stylesheets, "chartist/chartist.min.css")
+	headerVars.Scripts = append(headerVars.Scripts, "chartist/chartist.min.js")
+
+	timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange"))
+	if err != nil {
+		return common.LocalError(err.Error(), w, r, user)
+	}
+
+	var revLabelList []int64
+	var labelList []int64
+	var viewMap = make(map[int64]int64)
+	var currentTime = time.Now().Unix()
+
+	for i := 1; i <= timeRange.Slices; i++ {
+		var label = currentTime - int64(i*timeRange.SliceWidth)
+		revLabelList = append(revLabelList, label)
+		viewMap[label] = 0
+	}
+	for _, value := range revLabelList {
+		labelList = append(labelList, value)
+	}
+
+	var viewList []int64
+	log.Print("in routePanelAnalyticsTopics")
+
+	acc := qgen.Builder.Accumulator()
+	rows, err := acc.Select("topicchunks").Columns("count, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
+	if err != nil && err != ErrNoRows {
+		return common.InternalError(err, w, r)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var count int64
+		var createdAt time.Time
+		err := rows.Scan(&count, &createdAt)
+		if err != nil {
+			return common.InternalError(err, w, r)
+		}
+		log.Print("count: ", count)
+		log.Print("createdAt: ", createdAt)
+
+		var unixCreatedAt = createdAt.Unix()
+		log.Print("unixCreatedAt: ", unixCreatedAt)
+		for _, value := range labelList {
+			if unixCreatedAt > value {
+				viewMap[value] += count
+				break
+			}
+		}
+	}
+	err = rows.Err()
+	if err != nil {
+		return common.InternalError(err, w, r)
+	}
+
+	var viewItems []common.PanelAnalyticsItem
+	for _, value := range revLabelList {
+		viewList = append(viewList, viewMap[value])
+		viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
+	}
+	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	log.Printf("graph: %+v\n", graph)
+
+	pi := common.PanelAnalyticsPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", graph, viewItems, timeRange.Range}
+	return panelRenderTemplate("panel_analytics_topics", w, r, user, &pi)
+}
+
 func routePanelAnalyticsPosts(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
 	headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
 	if ferr != nil {
@@ -1110,7 +1183,7 @@ func routePanelWordFilters(w http.ResponseWriter, r *http.Request, user common.U
 	return panelRenderTemplate("panel_word_filters", w, r, user, &pi)
 }
 
-func routePanelWordFiltersCreate(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
+func routePanelWordFiltersCreateSubmit(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
 	_, ferr := common.SimplePanelUserCheck(w, r, &user)
 	if ferr != nil {
 		return ferr

query_gen/tables.go

@@ -386,6 +386,15 @@ func createTables(adapter qgen.Adapter) error {
 		)
 	*/
 
+	qgen.Install.CreateTable("topicchunks", "", "",
+		[]qgen.DBTableColumn{
+			qgen.DBTableColumn{"count", "int", 0, false, false, "0"},
+			qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""},
+			// TODO: Add a column for the parent forum?
+		},
+		[]qgen.DBTableKey{},
+	)
+
 	qgen.Install.CreateTable("postchunks", "", "",
 		[]qgen.DBTableColumn{
 			qgen.DBTableColumn{"count", "int", 0, false, false, "0"},

router_gen/main.go

@@ -181,7 +181,6 @@ func main() {
 	for id, agent := range tmplVars.AllAgentNames {
 		tmplVars.AllAgentMap[agent] = id
 	}
-	var graveSym = "`"
 
 	var fileData = `// Code generated by. DO NOT EDIT.
 /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */
@@ -195,6 +194,7 @@ import (
 	"net/http"
 
 	"./common"
+	"./routes"
 )
 
 var ErrNoRoute = errors.New("That route doesn't exist.")
@@ -285,8 +285,22 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	var prefix, extraData string
 	prefix = req.URL.Path[0:strings.IndexByte(req.URL.Path[1:],'/') + 1]
 	if req.URL.Path[len(req.URL.Path) - 1] != '/' {
-		// TODO: Cover more suspicious strings and at a lower layer than this and more efficiently
-		if strings.Contains(req.URL.Path,"'") || strings.Contains(req.URL.Path,";") || strings.Contains(req.URL.Path,"\"") || strings.Contains(req.URL.Path,"` + graveSym + `") || strings.Contains(req.URL.Path,"%") {
+		// TODO: Cover more suspicious strings and at a lower layer than this
+		for _, char := range req.URL.Path {
+			if char != '&' && !(char > 44 && char < 58) && char != '=' && char != '?' && !(char > 64 && char < 91) && char != '\\' && char != '_' && !(char > 96 && char < 123) {
+				log.Print("Suspicious UA: ", req.UserAgent())
+				log.Print("Method: ", req.Method)
+				for key, value := range req.Header {
+					for _, vvalue := range value {
+						log.Print("Header '" + key + "': " + vvalue + "!!")
+					}
+				}
+				log.Print("req.URL.Path: ", req.URL.Path)
+				log.Print("req.Referer(): ", req.Referer())
+				log.Print("req.RemoteAddr: ", req.RemoteAddr)
+			}
+		}
+		if strings.Contains(req.URL.Path,"..") || strings.Contains(req.URL.Path,"--") {
 			log.Print("Suspicious UA: ", req.UserAgent())
 			log.Print("Method: ", req.Method)
 			for key, value := range req.Header {

router_gen/route_impl.go

@@ -95,6 +95,10 @@ func AnonAction(fname string, path string, args ...string) *RouteImpl {
 	return route(fname, path, args...).Before("ParseForm")
 }
 
+func Special(fname string, path string, args ...string) *RouteImpl {
+	return route(fname, path, args...).LitBefore("req.URL.Path += extraData")
+}
+
 // Make this it's own type to force the user to manipulate methods on it to set parameters
 type uploadAction struct {
 	Route *RouteImpl

router_gen/routes.go

@@ -30,6 +30,8 @@ func routes() {
 	buildReplyRoutes()
 	buildProfileReplyRoutes()
 	buildAccountRoutes()
+
+	addRoute(Special("routeWebsockets", "/ws/"))
 }
 
 // TODO: Test the email token route
@@ -64,7 +66,7 @@ func buildTopicRoutes() {
 	topicGroup.Routes(
 		View("routeTopicID", "/topic/", "extraData"),
 		Action("routeTopicCreateSubmit", "/topic/create/submit/"),
-		Action("routeEditTopicSubmit", "/topic/edit/submit/", "extraData"),
+		Action("routes.EditTopicSubmit", "/topic/edit/submit/", "extraData"),
 		Action("routeDeleteTopicSubmit", "/topic/delete/submit/").LitBefore("req.URL.Path += extraData"),
 		Action("routeStickTopicSubmit", "/topic/stick/submit/", "extraData"),
 		Action("routeUnstickTopicSubmit", "/topic/unstick/submit/", "extraData"),
@@ -134,7 +136,7 @@ func buildPanelRoutes() {
 		Action("routePanelSettingEditSubmit", "/panel/settings/edit/submit/", "extraData"),
 
 		View("routePanelWordFilters", "/panel/settings/word-filters/"),
-		Action("routePanelWordFiltersCreate", "/panel/settings/word-filters/create/"),
+		Action("routePanelWordFiltersCreateSubmit", "/panel/settings/word-filters/create/"),
 		View("routePanelWordFiltersEdit", "/panel/settings/word-filters/edit/", "extraData"),
 		Action("routePanelWordFiltersEditSubmit", "/panel/settings/word-filters/edit/submit/", "extraData"),
 		Action("routePanelWordFiltersDeleteSubmit", "/panel/settings/word-filters/delete/submit/", "extraData"),
@@ -157,6 +159,7 @@ func buildPanelRoutes() {
 		View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"),
 		View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"),
 		View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"),
+		View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"),
 
 		View("routePanelGroups", "/panel/groups/"),
 		View("routePanelGroupsEdit", "/panel/groups/edit/", "extraData"),

routes/filler.txt

@@ -1 +0,0 @@
-This file is here so that Git will include this folder in the repository.

routes/topic.go

@@ -0,0 +1,55 @@
+package routes
+
+import (
+	"database/sql"
+	"net/http"
+	"strconv"
+
+	"../common"
+)
+
+var successJSONBytes = []byte(`{"success":"1"}`)
+
+// TODO: Update the stats after edits so that we don't under or over decrement stats during deletes
+// TODO: Disable stat updates in posts handled by plugin_guilds
+func EditTopicSubmit(w http.ResponseWriter, r *http.Request, user common.User, stid string) common.RouteError {
+	isJs := (r.PostFormValue("js") == "1")
+
+	tid, err := strconv.Atoi(stid)
+	if err != nil {
+		return common.PreErrorJSQ("The provided TopicID is not a valid number.", w, r, isJs)
+	}
+
+	topic, err := common.Topics.Get(tid)
+	if err == sql.ErrNoRows {
+		return common.PreErrorJSQ("The topic you tried to edit doesn't exist.", w, r, isJs)
+	} else if err != nil {
+		return common.InternalErrorJSQ(err, w, r, isJs)
+	}
+
+	// TODO: Add hooks to make use of headerLite
+	_, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID)
+	if ferr != nil {
+		return ferr
+	}
+	if !user.Perms.ViewTopic || !user.Perms.EditTopic {
+		return common.NoPermissionsJSQ(w, r, user, isJs)
+	}
+
+	err = topic.Update(r.PostFormValue("topic_name"), r.PostFormValue("topic_content"))
+	if err != nil {
+		return common.InternalErrorJSQ(err, w, r, isJs)
+	}
+
+	err = common.Forums.UpdateLastTopic(topic.ID, user.ID, topic.ParentID)
+	if err != nil && err != sql.ErrNoRows {
+		return common.InternalErrorJSQ(err, w, r, isJs)
+	}
+
+	if !isJs {
+		http.Redirect(w, r, "/topic/"+strconv.Itoa(tid), http.StatusSeeOther)
+	} else {
+		_, _ = w.Write(successJSONBytes)
+	}
+	return nil
+}

schema/mssql/query_topicchunks.sql

@@ -0,0 +1,4 @@
+CREATE TABLE [topicchunks] (
+	[count] int DEFAULT 0 not null,
+	[createdAt] datetime not null
+);

schema/mysql/query_topicchunks.sql

@@ -0,0 +1,4 @@
+CREATE TABLE `topicchunks` (
+	`count` int DEFAULT 0 not null,
+	`createdAt` datetime not null
+);

schema/pgsql/query_topicchunks.sql

@@ -0,0 +1,4 @@
+CREATE TABLE `topicchunks` (
+	`count` int DEFAULT 0 not null,
+	`createdAt` timestamp not null
+);

templates/create_topic.html

@@ -1,5 +1,5 @@
 {{template "header.html" . }}
-<main>
+<main id="create_topic_page">
 	<div class="rowblock rowhead">
 		<div class="rowitem"><h1>Create Topic</h1></div>
 	</div>

templates/login.html

@@ -1,5 +1,5 @@
 {{template "header.html" . }}
-<main>
+<main id="login_page">
 	<div class="rowblock rowhead">
 		<div class="rowitem"><h1>Login</h1></div>
 	</div>

templates/panel-inner-menu.html

@@ -32,6 +32,9 @@
 		<div class="rowitem passive submenu">
 			<a href="/panel/analytics/posts/">Posts</a>
 		</div>
+		<div class="rowitem passive submenu">
+			<a href="/panel/analytics/topics/">Topics</a>
+		</div>
 		<div class="rowitem passive submenu">
 			<a href="/panel/analytics/routes/">Routes</a>
 		</div>

templates/panel_analytics_posts.html

@@ -5,7 +5,7 @@
 		<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/posts/" method="get">
 			<div class="colstack_item colstack_head">
 				<div class="rowitem">
-					<a>Posts</a>
+					<a>Post Counts</a>
 					<select class="timeRangeSelector to_right" name="timeRange">
 						<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
 						<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>

templates/panel_analytics_topics.html

@@ -0,0 +1,62 @@
+{{template "header.html" . }}
+<div class="colstack panel_stack">
+{{template "panel-menu.html" . }}
+	<main id="panel_dashboard_right" class="colstack_right">
+		<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/topics/" method="get">
+			<div class="colstack_item colstack_head">
+				<div class="rowitem">
+					<a>Topic Counts</a>
+					<select class="timeRangeSelector to_right" name="timeRange">
+						<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
+						<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>
+						<option val="one-day"{{if eq .TimeRange "one-day"}} selected{{end}}>1 day</option>
+						<option val="twelve-hours"{{if eq .TimeRange "twelve-hours"}} selected{{end}}>12 hours</option>
+						<option val="six-hours"{{if eq .TimeRange "six-hours"}} selected{{end}}>6 hours</option>
+					</select>
+				</div>
+			</div>
+		</form>
+		<div id="panel_analytics_topics" class="colstack_graph_holder">
+			<div class="ct_chart" aria-label="Topic Chart"></div>
+		</div>
+		<div class="colstack_item colstack_head">
+			<div class="rowitem"><a>Details</a></div>
+		</div>
+		<div id="panel_analytics_topics_table" class="colstack_item rowlist" aria-label="Topic Table, this has the same information as the topic chart">
+			{{range .ViewItems}}
+			<div class="rowitem panel_compactrow editable_parent">
+				<a class="panel_upshift unix_to_24_hour_time">{{.Time}}</a>
+				<span class="panel_compacttext to_right">{{.Count}} views</span>
+			</div>
+			{{end}}
+		</div>
+	</main>
+</div>
+<script>
+let labels = [];
+let rawLabels = [{{range .PrimaryGraph.Labels}}
+{{.}},{{end}}
+];
+for(const i in rawLabels) {
+	let date = new Date(rawLabels[i]*1000);
+	console.log("date: ", date);
+	let minutes = "0" + date.getMinutes();
+	let label = date.getHours() + ":" + minutes.substr(-2);
+	console.log("label:", label);
+	labels.push(label);
+}
+labels = labels.reverse()
+
+let seriesData = [{{range .PrimaryGraph.Series}}
+{{.}},{{end}}
+];
+seriesData = seriesData.reverse();
+
+Chartist.Line('.ct_chart', {
+  labels: labels,
+  series: [seriesData],
+}, {
+  height: '250px',
+});
+</script>
+{{template "footer.html" . }}

templates/register.html

@@ -1,5 +1,5 @@
 {{template "header.html" . }}
-<main>
+<main id="register_page">
 	<div class="rowblock rowhead">
 		<div class="rowitem"><h1>Create Account</h1></div>
 	</div>

themes/cosora/public/main.css

@@ -1138,6 +1138,22 @@ textarea {
 	margin-left: auto;
 }
 
+#create_topic_page .close_form, #create_topic_page .formlabel, #login_page .formlabel, #register_page .formlabel {
+	display: none;
+}
+#login_page .formrow:not(:first-child):not(:last-child), #register_page .formrow:not(:first-child):not(:last-child) {
+	margin-top: 4px;
+}
+#login_page .formrow:not(:first-child), #register_page .formrow:not(:first-child) {
+	padding-top: 3px;
+}
+#login_page .formrow:not(:last-child), #register_page .formrow:not(:last-child) {
+	padding-bottom: 0px;
+}
+#login_page .formrow, #register_page .formrow {
+	padding: 16px;
+}
+
 /* TODO: Highlight the one we're currently on? */
 .pageset {
 	display: flex;