track favicon stats

experiment with tracking average route performance
temporary error route stub
optimise dumprequest
add DisableAnalytics config setting
fix double hyphens in slugs being mistaken for sql injection
more querygen tests

You wil need to run the updater / patcher for this commit.

cmd/query_gen/tables.go

@@ -662,6 +662,7 @@ func createTables(adapter qgen.Adapter) (err error) {
 	createTable("viewchunks", "", "",
 		[]tC{
 			tC{"count", "int", 0, false, false, "0"},
+			tC{"avg", "int", 0, false, false, "0"},
 			tC{"createdAt", "datetime", 0, false, false, ""},
 			tC{"route", "varchar", 200, false, false, ""}, // todo: set a default empty here
 		}, nil,

common/counters/agents.go

@@ -50,7 +50,7 @@ func (co *DefaultAgentViewCounter) insertChunk(count int64, agent int) error {
 
 func (co *DefaultAgentViewCounter) Bump(agent int) {
 	// TODO: Test this check
-	c.DebugDetail("co.buckets[", agent, "]: ", co.buckets[agent])
+	c.DebugDetail("buckets[", agent, "]: ", co.buckets[agent])
 	if len(co.buckets) <= agent || agent < 0 {
 		return
 	}

common/counters/langs.go

@@ -157,7 +157,7 @@ func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
 	}
 
 	// TODO: Test this check
-	c.DebugDetail("co.buckets[", id, "]: ", co.buckets[id])
+	c.DebugDetail("buckets[", id, "]: ", co.buckets[id])
 	if len(co.buckets) <= id || id < 0 {
 		return validCode
 	}

common/counters/referrers.go

@@ -85,7 +85,7 @@ func (ref *DefaultReferrerTracker) insertChunk(referrer string, count int64) err
 	if count == 0 {
 		return nil
 	}
-	c.DebugDetailf("Inserting a vchunk with a count of %d for referrer %s", count, referrer)
+	c.DebugDetailf("Inserting a vchunk with a count of %d for ref %s", count, referrer)
 	_, err := ref.insert.Exec(count, referrer)
 	return err
 }

common/counters/routes.go

@@ -2,6 +2,8 @@ package counters
 
 import (
 	"database/sql"
+	"sync"
+	"time"
 
 	c "github.com/Azareal/Gosora/common"
 	qgen "github.com/Azareal/Gosora/query_gen"
@@ -10,56 +12,68 @@ import (
 
 var RouteViewCounter *DefaultRouteViewCounter
 
+type RVBucket struct {
+	counter int
+	avg     int
+
+	sync.Mutex
+}
+
 // TODO: Make this lockless?
 type DefaultRouteViewCounter struct {
-	buckets []*RWMutexCounterBucket //[RouteID]count
+	buckets []*RVBucket //[RouteID]count
 	insert  *sql.Stmt
 	insert5 *sql.Stmt
 }
 
 func NewDefaultRouteViewCounter(acc *qgen.Accumulator) (*DefaultRouteViewCounter, error) {
-	routeBuckets := make([]*RWMutexCounterBucket, len(routeMapEnum))
+	routeBuckets := make([]*RVBucket, len(routeMapEnum))
 	for bucketID, _ := range routeBuckets {
-		routeBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
+		routeBuckets[bucketID] = &RVBucket{counter: 0, avg: 0}
 	}
 
-	fields := "?,UTC_TIMESTAMP(),?"
+	fields := "?,?,UTC_TIMESTAMP(),?"
 	co := &DefaultRouteViewCounter{
 		buckets: routeBuckets,
-		insert:  acc.Insert("viewchunks").Columns("count,createdAt,route").Fields(fields).Prepare(),
+		insert:  acc.Insert("viewchunks").Columns("count,avg,createdAt,route").Fields(fields).Prepare(),
-		insert5:  acc.BulkInsert("viewchunks").Columns("count,createdAt,route").Fields(fields,fields,fields,fields,fields).Prepare(),
+		insert5: acc.BulkInsert("viewchunks").Columns("count,avg,createdAt,route").Fields(fields, fields, fields, fields, fields).Prepare(),
+	}
+	if !c.Config.DisableAnalytics {
+		c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
+		//c.AddScheduledSecondTask(co.Tick)
+		c.AddShutdownTask(co.Tick)
 	}
-	c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
-	//c.AddScheduledSecondTask(co.Tick)
-	c.AddShutdownTask(co.Tick)
 	return co, acc.FirstError()
 }
 
 type RVCount struct {
 	RouteID int
-	Count int
+	Count   int
+	Avg     int
 }
 
-func (co *DefaultRouteViewCounter) Tick() error {
+func (co *DefaultRouteViewCounter) Tick() (err error) {
 	var tb []RVCount
 	for routeID, b := range co.buckets {
-		var count int
+		var count, avg int
-		b.RLock()
+		b.Lock()
 		count = b.counter
 		b.counter = 0
-		b.RUnlock()
+		avg = b.avg
+		b.avg = 0
+		b.Unlock()
 
 		if count == 0 {
 			continue
 		}
-		tb = append(tb, RVCount{routeID,count})
+		tb = append(tb, RVCount{routeID, count, avg})
 	}
 
 	// TODO: Expand on this?
 	var i int
 	if len(tb) >= 5 {
-		for ; len(tb) > (i+5); i += 5 {
+		for ; len(tb) > (i + 5); i += 5 {
-			err := co.insert5Chunk(tb[i:i+5])
+			err := co.insert5Chunk(tb[i : i+5])
 			if err != nil {
 				c.DebugLogf("tb: %+v\n", tb)
 				c.DebugLog("i: ", i)
@@ -69,7 +83,8 @@ func (co *DefaultRouteViewCounter) Tick() error {
 	}
 
 	for ; len(tb) > i; i++ {
-		err := co.insertChunk(tb[i].Count, tb[i].RouteID)
+		it := tb[i]
+		err = co.insertChunk(it.Count, it.Avg, it.RouteID)
 		if err != nil {
 			c.DebugLogf("tb: %+v\n", tb)
 			c.DebugLog("i: ", i)
@@ -80,32 +95,40 @@ func (co *DefaultRouteViewCounter) Tick() error {
 	return nil
 }
 
-func (co *DefaultRouteViewCounter) insertChunk(count, route int) error {
+func (co *DefaultRouteViewCounter) insertChunk(count, avg, route int) error {
 	routeName := reverseRouteMapEnum[route]
-	c.DebugLogf("Inserting a vchunk with a count of %d for route %s (%d)", count, routeName, route)
+	c.DebugLogf("Inserting a vchunk with a count of %d, avg of %d for route %s (%d)", count, avg, routeName, route)
-	_, err := co.insert.Exec(count, routeName)
+	_, err := co.insert.Exec(count, avg, routeName)
 	return err
 }
 
 func (co *DefaultRouteViewCounter) insert5Chunk(rvs []RVCount) error {
-	args := make([]interface{}, len(rvs) * 2)
+	args := make([]interface{}, len(rvs)*3)
 	i := 0
 	for _, rv := range rvs {
 		routeName := reverseRouteMapEnum[rv.RouteID]
-		c.DebugLogf("Queueing a vchunk with a count of %d for routes %s (%d)", rv.Count, routeName, rv.RouteID)
+		if rv.Avg == 0 {
+			c.DebugLogf("Queueing a vchunk with a count of %d for routes %s (%d)", rv.Count, routeName, rv.RouteID)
+		} else {
+			c.DebugLogf("Queueing a vchunk with count %d, avg %d for routes %s (%d)", rv.Count, rv.Avg, routeName, rv.RouteID)
+		}
 		args[i] = rv.Count
-		args[i+1] = routeName
+		args[i+1] = rv.Avg
-		i += 2
+		args[i+2] = routeName
+		i += 3
 	}
-	c.DebugLogf("args: %+v\n", args)
+	c.DebugDetailf("args: %+v\n", args)
 	_, err := co.insert5.Exec(args...)
 	return err
 }
 
 func (co *DefaultRouteViewCounter) Bump(route int) {
+	if c.Config.DisableAnalytics {
+		return
+	}
 	// TODO: Test this check
 	b := co.buckets[route]
-	c.DebugDetail("co.buckets[", route, "]: ", b)
+	c.DebugDetail("buckets[", route, "]: ", b)
 	if len(co.buckets) <= route || route < 0 {
 		return
 	}
@@ -114,3 +137,27 @@ func (co *DefaultRouteViewCounter) Bump(route int) {
 	b.counter++
 	b.Unlock()
 }
+
+// TODO: Eliminate the lock?
+func (co *DefaultRouteViewCounter) Bump2(route int, t time.Time) {
+	if c.Config.DisableAnalytics {
+		return
+	}
+	// TODO: Test this check
+	b := co.buckets[route]
+	c.DebugDetail("buckets[", route, "]: ", b)
+	if len(co.buckets) <= route || route < 0 {
+		return
+	}
+	micro := int(time.Since(t).Microseconds())
+	b.Lock()
+	b.counter++
+	if micro != b.avg {
+		if b.avg == 0 {
+			b.avg = micro
+		} else {
+			b.avg = (micro + b.avg) / 2
+		}
+	}
+	b.Unlock()
+}

common/counters/topics.go

@@ -5,7 +5,7 @@ import (
 	"sync/atomic"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
 	"github.com/pkg/errors"
 )
 
@@ -22,7 +22,7 @@ func NewTopicCounter() (*DefaultTopicCounter, error) {
 	acc := qgen.NewAcc()
 	co := &DefaultTopicCounter{
 		currentBucket: 0,
-		insert:        acc.Insert("topicchunks").Columns("count, createdAt").Fields("?,UTC_TIMESTAMP()").Prepare(),
+		insert:        acc.Insert("topicchunks").Columns("count,createdAt").Fields("?,UTC_TIMESTAMP()").Prepare(),
 	}
 	c.AddScheduledFifteenMinuteTask(co.Tick)
 	//c.AddScheduledSecondTask(co.Tick)
@@ -44,7 +44,7 @@ func (co *DefaultTopicCounter) Tick() (err error) {
 	atomic.AddInt64(&co.buckets[oldBucket], -previousViewChunk)
 	err = co.insertChunk(previousViewChunk)
 	if err != nil {
-		return errors.Wrap(errors.WithStack(err),"topics counter")
+		return errors.Wrap(errors.WithStack(err), "topics counter")
 	}
 	return nil
 }

common/reply_store.go

@@ -97,7 +97,7 @@ func (s *SQLReplyStore) Exists(id int) bool {
 }
 
 // TODO: Write a test for this
-func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (rid int, err error) {
+func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (id int, err error) {
 	if Config.DisablePostIP {
 		ip = ""
 	}
@@ -110,8 +110,8 @@ func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (rid int,
 	if err != nil {
 		return 0, err
 	}
-	rid = int(lastID)
+	id = int(lastID)
-	return rid, t.AddReply(rid, uid)
+	return id, t.AddReply(id, uid)
 }
 
 // TODO: Write a test for this

common/site.go

@@ -113,6 +113,7 @@ type config struct {
 	EnableCDNPush          bool
 	DisableNoavatarRange   bool
 	DisableDefaultNoavatar bool
+	DisableAnalytics       bool
 
 	RefNoTrack bool
 	RefNoRef   bool

docs/configuration.md

@@ -116,6 +116,8 @@ DisableNoavatarRange - This switch lets you disable the noavatar algorithm which
 
 DisableDefaultNoavatar - This switch lets you disable the default noavatar algorithm which may intercept noavatars for increased efficiency. Default: false
 
+DisableAnalytics - This switch lets you disable the analytics subsystem. Default: false
+
 RefNoTrack - This switch disables tracking the referrers of users who click from another site to your site and the referrers of any requests to resources from other sites as-well.
 
 RefNoRef - This switch makes it so that if a user clicks on a link, then the incoming site won't know which site they're coming from.

main.go

@@ -32,6 +32,7 @@ import (
 	qgen "github.com/Azareal/Gosora/query_gen"
 	"github.com/Azareal/Gosora/routes"
 	"github.com/fsnotify/fsnotify"
+
 	//"github.com/lucas-clemente/quic-go/http3"
 	"github.com/pkg/errors"
 )
@@ -272,21 +273,37 @@ func storeInit() (err error) {
 	}
 
 	log.Print("Initialising the view counters")
-	co.GlobalViewCounter, err = co.NewGlobalViewCounter(acc)
+	if !c.Config.DisableAnalytics {
-	if err != nil {
+		co.GlobalViewCounter, err = co.NewGlobalViewCounter(acc)
-		return errors.WithStack(err)
+		if err != nil {
-	}
+			return errors.WithStack(err)
-	co.AgentViewCounter, err = co.NewDefaultAgentViewCounter(acc)
+		}
-	if err != nil {
+		co.AgentViewCounter, err = co.NewDefaultAgentViewCounter(acc)
-		return errors.WithStack(err)
+		if err != nil {
-	}
+			return errors.WithStack(err)
-	co.OSViewCounter, err = co.NewDefaultOSViewCounter(acc)
+		}
-	if err != nil {
+		co.OSViewCounter, err = co.NewDefaultOSViewCounter(acc)
-		return errors.WithStack(err)
+		if err != nil {
-	}
+			return errors.WithStack(err)
-	co.LangViewCounter, err = co.NewDefaultLangViewCounter(acc)
+		}
-	if err != nil {
+		co.LangViewCounter, err = co.NewDefaultLangViewCounter(acc)
-		return errors.WithStack(err)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if !c.Config.RefNoTrack {
+			co.ReferrerTracker, err = co.NewDefaultReferrerTracker()
+			if err != nil {
+				return errors.WithStack(err)
+			}
+		}
+		co.MemoryCounter, err = co.NewMemoryCounter(acc)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		co.PerfCounter, err = co.NewDefaultPerfCounter(acc)
+		if err != nil {
+			return errors.WithStack(err)
+		}
 	}
 	co.RouteViewCounter, err = co.NewDefaultRouteViewCounter(acc)
 	if err != nil {
@@ -308,18 +325,6 @@ func storeInit() (err error) {
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	co.ReferrerTracker, err = co.NewDefaultReferrerTracker()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	co.MemoryCounter, err = co.NewMemoryCounter(acc)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	co.PerfCounter, err = co.NewDefaultPerfCounter(acc)
-	if err != nil {
-		return errors.WithStack(err)
-	}
 
 	log.Print("Initialising the meta store")
 	c.Meta, err = meta.NewDefaultMetaStore(acc)

patcher/patches.go

@@ -50,6 +50,7 @@ func init() {
 	addPatch(30, patch30)
 	addPatch(31, patch31)
 	addPatch(32, patch32)
+	addPatch(33, patch33)
 }
 
 func patch0(scanner *bufio.Scanner) (err error) {
@@ -428,22 +429,22 @@ func patch10(scanner *bufio.Scanner) error {
 
 	err = acc().Select("topics").Cols("tid").EachInt(func(tid int) error {
 		stid := itoa(tid)
-		count, err := acc().Count("attachments").Where("originTable = 'topics' and originID = " + stid).Total()
+		count, err := acc().Count("attachments").Where("originTable = 'topics' and originID=" + stid).Total()
 		if err != nil {
 			return err
 		}
 
 		hasReply := false
-		err = acc().Select("replies").Cols("rid").Where("tid = " + stid).Orderby("rid DESC").Limit("1").EachInt(func(rid int) error {
+		err = acc().Select("replies").Cols("rid").Where("tid=" + stid).Orderby("rid DESC").Limit("1").EachInt(func(rid int) error {
 			hasReply = true
-			_, err := acc().Update("topics").Set("lastReplyID = ?, attachCount = ?").Where("tid = "+stid).Exec(rid, count)
+			_, err := acc().Update("topics").Set("lastReplyID=?, attachCount=?").Where("tid="+stid).Exec(rid, count)
 			return err
 		})
 		if err != nil {
 			return err
 		}
 		if !hasReply {
-			_, err = acc().Update("topics").Set("attachCount = ?").Where("tid = " + stid).Exec(count)
+			_, err = acc().Update("topics").Set("attachCount=?").Where("tid=" + stid).Exec(count)
 		}
 		return err
 	})
@@ -894,14 +895,17 @@ func patch31(scanner *bufio.Scanner) error {
 	return nil
 }
 
-
 func patch32(scanner *bufio.Scanner) error {
 	return execStmt(qgen.Builder.CreateTable("perfchunks", "", "",
-	[]tC{
+		[]tC{
-		tC{"low", "int", 0, false, false, "0"},
+			tC{"low", "int", 0, false, false, "0"},
-		tC{"high", "int", 0, false, false, "0"},
+			tC{"high", "int", 0, false, false, "0"},
-		tC{"avg", "int", 0, false, false, "0"},
+			tC{"avg", "int", 0, false, false, "0"},
-		tC{"createdAt", "datetime", 0, false, false, ""},
+			tC{"createdAt", "datetime", 0, false, false, ""},
-	}, nil,
+		}, nil,
-))
+	))
+}
+
+func patch33(scanner *bufio.Scanner) error {
+	return execStmt(qgen.Builder.AddColumn("viewchunks", tC{"avg", "int", 0, false, false, "0"}, nil))
 }

query_gen/utils_test.go

@@ -31,6 +31,8 @@ func TestProcessWhere(t *testing.T) {
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
 	whs = processWhere("uid = '1'")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
+	whs = processWhere("uid = 't'")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "t"})
 	whs = processWhere("uid = ''")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
 	whs = processWhere("uid = '")
@@ -42,8 +44,12 @@ func TestProcessWhere(t *testing.T) {
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
 	whs = processWhere("uid=0")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
+	whs = processWhere("uid=20")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "20"})
 	whs = processWhere("uid='1'")
 	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
+	whs = processWhere("uid='t'")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "t"})
 
 	whs = processWhere("uid")
 	expectTokens(t, whs, MT{TokenColumn, "uid"})

router_gen/main.go

@@ -47,23 +47,24 @@ func main() {
 		allRouteNames = append(allRouteNames, RouteName{name, strings.Replace(name, "common.", "c.", -1)})
 		allRouteMap[name] = len(allRouteNames) - 1
 	}
+	mapIt("routes.Error")
 	countToIndents := func(indent int) (indentor string) {
 		for i := 0; i < indent; i++ {
 			indentor += "\t"
 		}
 		return indentor
 	}
-	runBefore := func(runnables []Runnable, indent int) (out string) {
+	runBefore := func(runnables []Runnable, indentCount int) (out string) {
-		indentor := countToIndents(indent)
+		indent := countToIndents(indentCount)
 		if len(runnables) > 0 {
 			for _, runnable := range runnables {
 				if runnable.Literal {
-					out += "\n\t" + indentor + runnable.Contents
+					out += "\n\t" + indent + runnable.Contents
 				} else {
-					out += "\n" + indentor + "err = c." + runnable.Contents + "(w,req,user)\n" +
+					out += "\n" + indent + "err = c." + runnable.Contents + "(w,req,user)\n" +
-						indentor + "if err != nil {\n" +
+						indent + "if err != nil {\n" +
-						indentor + "\treturn err\n" +
+						indent + "\treturn err\n" +
-						indentor + "}\n" + indentor
+						indent + "}\n" + indent
 				}
 			}
 		}
@@ -74,13 +75,13 @@ func main() {
 		mapIt(route.Name)
 		end := len(route.Path) - 1
 		out += "\n\t\tcase \"" + route.Path[0:end] + "\":"
+		//out += "\n\t\t\tid = " + strconv.Itoa(allRouteMap[route.Name])
 		out += runBefore(route.RunBefore, 4)
-		out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
 		if !route.Action && !route.NoHead {
-			out += "\n\t\t\thead, err := c.UserCheck(w,req,&user)"
+			out += "\n\t\t\th, err := c.UserCheck(w,req,&user)"
 			out += "\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}"
 			vcpy := route.Vars
-			route.Vars = []string{"head"}
+			route.Vars = []string{"h"}
 			route.Vars = append(route.Vars, vcpy...)
 		}
 		out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
@@ -88,6 +89,11 @@ func main() {
 			out += "," + item
 		}
 		out += `)`
+		if !route.Action && !route.NoHead {
+			out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
+		} else {
+			out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+		}
 	}
 
 	for _, group := range r.routeGroups {
@@ -105,6 +111,7 @@ func main() {
 			mapIt(route.Name)
 
 			out += "\n\t\t\t\tcase \"" + route.Path + "\":"
+			//out += "\n\t\t\t\t\tid = " + strconv.Itoa(allRouteMap[route.Name])
 			if len(route.RunBefore) > 0 {
 			skipRunnable:
 				for _, runnable := range route.RunBefore {
@@ -136,12 +143,11 @@ func main() {
 					}
 				}
 			}
-			out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
 			if !route.Action && !route.NoHead && !group.NoHead {
-				out += "\n\t\t\t\thead, err := c.UserCheck(w,req,&user)"
+				out += "\n\t\t\t\th, err := c.UserCheck(w,req,&user)"
 				out += "\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}"
 				vcpy := route.Vars
-				route.Vars = []string{"head"}
+				route.Vars = []string{"h"}
 				route.Vars = append(route.Vars, vcpy...)
 			}
 			out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
@@ -149,18 +155,23 @@ func main() {
 				out += "," + item
 			}
 			out += ")"
+			if !route.Action && !route.NoHead && !group.NoHead {
+				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
+			} else {
+				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+			}
 		}
 
 		if defaultRoute.Name != "" {
 			mapIt(defaultRoute.Name)
 			out += "\n\t\t\t\tdefault:"
+			//out += "\n\t\t\t\t\tid = " + strconv.Itoa(allRouteMap[defaultRoute.Name])
 			out += runBefore(defaultRoute.RunBefore, 4)
-			out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ")"
 			if !defaultRoute.Action && !defaultRoute.NoHead && !group.NoHead {
-				out += "\n\t\t\t\t\thead, err := c.UserCheck(w,req,&user)"
+				out += "\n\t\t\t\t\th, err := c.UserCheck(w,req,&user)"
 				out += "\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}"
 				vcpy := defaultRoute.Vars
-				defaultRoute.Vars = []string{"head"}
+				defaultRoute.Vars = []string{"h"}
 				defaultRoute.Vars = append(defaultRoute.Vars, vcpy...)
 			}
 			out += "\n\t\t\t\t\terr = " + strings.Replace(defaultRoute.Name, "common.", "c.", -1) + "(w,req,user"
@@ -168,6 +179,11 @@ func main() {
 				out += ", " + item
 			}
 			out += ")"
+			if !defaultRoute.Action && !defaultRoute.NoHead && !group.NoHead {
+				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ", h.StartedAt)"
+			} else {
+				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ")"
+			}
 		}
 		out += `
 			}`
@@ -180,6 +196,7 @@ func main() {
 	mapIt("routes.RobotsTxt")
 	mapIt("routes.SitemapXml")
 	mapIt("routes.OpenSearchXml")
+	mapIt("routes.Favicon")
 	mapIt("routes.BadRoute")
 	mapIt("routes.HTTPSRedirect")
 	tmplVars.AllRouteNames = allRouteNames
@@ -462,7 +479,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
 	var heads string
 	for key, value := range req.Header {
 		for _, vvalue := range value {
-			heads += "Header '" + c.SanitiseSingleLine(key) + "': " + c.SanitiseSingleLine(vvalue) + "!\n"
+			heads += "Header '" + c.SanitiseSingleLine(key) + "': " + c.SanitiseSingleLine(vvalue) + "\n"
 		}
 	}
 
@@ -472,7 +489,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
 		"Host: " + c.SanitiseSingleLine(req.Host) + "\n" + 
 		"URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + 
 		"URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + 
-		"Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + 
+		"Referer: " + c.SanitiseSingleLine(req.Referer()) + "\n" + 
 		"RemoteAddr: " + req.RemoteAddr + "\n")
 }
 
@@ -562,9 +579,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			break
 		}
 	}
-	lowerPath := strings.ToLower(req.URL.Path)
+	lp := strings.ToLower(req.URL.Path)
 	// TODO: Flag any requests which has a dot with anything but a number after that
-	if strings.Contains(req.URL.Path,"..") || strings.Contains(req.URL.Path,"--") || strings.Contains(lowerPath,".php") || strings.Contains(lowerPath,".asp") || strings.Contains(lowerPath,".cgi") || strings.Contains(lowerPath,".py") || strings.Contains(lowerPath,".sql") || strings.Contains(lowerPath,".action") {
+	if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") {
 		r.SuspiciousRequest(req,"Bad snippet in path")
 	}
 
@@ -572,6 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	if req.URL.Path == "/" {
 		req.URL.Path = c.Config.DefaultPath
 	}
+	//log.Print("URL.Path: ", req.URL.Path)
 	
 	var prefix, extraData string
 	prefix = req.URL.Path[0:strings.IndexByte(req.URL.Path[1:],'/') + 1]
@@ -603,10 +621,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		r.DumpRequest(req,"before routes.StaticFile")
 	}
 	// Increment the request counter
-	co.GlobalViewCounter.Bump()
+	if !c.Config.DisableAnalytics {
+		co.GlobalViewCounter.Bump()
+	}
 	
 	if prefix == "/s" { //old prefix: /static
-		co.RouteViewCounter.Bump({{index .AllRouteMap "routes.StaticFile"}})
+		if !c.Config.DisableAnalytics {
+			co.RouteViewCounter.Bump({{index .AllRouteMap "routes.StaticFile"}})
+		}
 		req.URL.Path += extraData
 		routes.StaticFile(w, req)
 		return
@@ -626,8 +648,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
 	// TODO: Add a setting to disable this?
 	// TODO: Use a more efficient detector instead of smashing every possible combination in
-	ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
 	var agent string
+	if !c.Config.DisableAnalytics {
+	
+	ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
 	if ua == "" {
 		co.AgentViewCounter.Bump({{.AllAgentMap.blank}})
 		if c.Dev.DebugMode {
@@ -736,8 +760,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// TODO: Do we want to track missing language headers too? Maybe as it's own type, e.g. "noheader"?
 	lang := req.Header.Get("Accept-Language")
 	if lang != "" {
-		lang = strings.TrimSpace(lang)
+		lLang := strings.Split(strings.TrimSpace(lang),"-")
-		lLang := strings.Split(lang,"-")
 		tLang := strings.Split(strings.Split(lLang[0],";")[0],",")
 		c.DebugDetail("tLang:", tLang)
 		var llLang string
@@ -764,12 +787,15 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			ref = strings.TrimPrefix(strings.TrimPrefix(ref,"http://"),"https://")
 			ref = strings.Split(ref,"/")[0]
 			portless := strings.Split(ref,":")[0]
+			// TODO: Handle c.Site.Host in uppercase too?
 			if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
 				co.ReferrerTracker.Bump(ref)
 			}
 		}
 	}
 
+	}
+	
 	// Deal with the session stuff, etc.
 	user, ok := c.PreRoute(w, req)
 	if !ok {
@@ -804,12 +830,15 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	if ferr != nil {
 		r.handleError(ferr,w,req,user)
 	}
+	/*if !c.Config.DisableAnalytics {
+		co.RouteViewCounter.Bump(id)
+	}*/
 
 	hTbl.VhookNoRet("router_end", w, req, user, prefix, extraData)
 	//c.StoppedServer("Profile end")
 }
 	
-func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
+func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) /*(id int, orerr */c.RouteError/*)*/ {
 	var err c.RouteError
 	switch(prefix) {` + out + `
 		/*case "/sitemaps": // TODO: Count these views
@@ -817,6 +846,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 			err = sitemapSwitch(w,req)*/
 		case "/uploads":
 			if extraData == "" {
+				co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
 				return c.NotFound(w,req,nil)
 			}
 			gzw, ok := w.(c.GzipResponseWriter)
@@ -826,10 +856,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 				h.Del("Content-Type")
 				h.Del("Content-Encoding")
 			}
-			co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
 			req.URL.Path += extraData
 			// TODO: Find a way to propagate errors up from this?
 			r.UploadHandler(w,req) // TODO: Count these views
+			co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
 			return nil
 		case "":
 			// Stop the favicons, robots.txt file, etc. resolving to the topics list
@@ -848,6 +878,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 					}
 					req.URL.Path = "/s/favicon.ico"
 					routes.StaticFile(w,req)
+					co.RouteViewCounter.Bump({{index .AllRouteMap "routes.Favicon"}})
 					return nil
 				case "opensearch.xml":
 					co.RouteViewCounter.Bump({{index .AllRouteMap "routes.OpenSearchXml"}})
@@ -856,17 +887,19 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 					co.RouteViewCounter.Bump({{index .AllRouteMap "routes.SitemapXml"}})
 					return routes.SitemapXml(w,req)*/
 			}
+			co.RouteViewCounter.Bump({{index .AllRouteMap "routes.Error"}})
 			return c.NotFound(w,req,nil)
 		default:
 			// A fallback for dynamic routes, e.g. ones declared by plugins
 			r.RLock()
-			handle, ok := r.extraRoutes[req.URL.Path]
+			h, ok := r.extraRoutes[req.URL.Path]
 			r.RUnlock()
 			
 			if ok {
-				co.RouteViewCounter.Bump({{index .AllRouteMap "routes.DynamicRoute"}}) // TODO: Be more specific about *which* dynamic route it is
 				req.URL.Path += extraData
-				return handle(w,req,user)
+				// TODO: Be more specific about *which* dynamic route it is
+				co.RouteViewCounter.Bump({{index .AllRouteMap "routes.DynamicRoute"}})
+				return h(w,req,user)
 			}
 
 			lp := strings.ToLower(req.URL.Path)
@@ -904,5 +937,8 @@ func writeFile(name, content string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	f.Close()
+	err = f.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
 }

router_gen/route_impl.go

@@ -63,8 +63,9 @@ func (r *RouteImpl) NoGzip() *RouteImpl {
 	return r.LitBeforeMultiline(`gzw, ok := w.(c.GzipResponseWriter)
 	if ok {
 		w = gzw.ResponseWriter
-		w.Header().Del("Content-Type")
+		h := w.Header()
-		w.Header().Del("Content-Encoding")
+		h.Del("Content-Type")
+		h.Del("Content-Encoding")
 	}`)
 }
 
@@ -77,15 +78,15 @@ func blankRoute() *RouteImpl {
 	return &RouteImpl{"", "", false, false, []string{}, []Runnable{}, nil}
 }
 
-func route(fname string, path string, action bool, special bool, args ...string) *RouteImpl {
+func route(fname, path string, action, special bool, args ...string) *RouteImpl {
 	return &RouteImpl{fname, path, action, special, args, []Runnable{}, nil}
 }
 
-func View(fname string, path string, args ...string) *RouteImpl {
+func View(fname, path string, args ...string) *RouteImpl {
 	return route(fname, path, false, false, args...)
 }
 
-func MView(fname string, path string, args ...string) *RouteImpl {
+func MView(fname, path string, args ...string) *RouteImpl {
 	route := route(fname, path, false, false, args...)
 	if !route.hasBefore("SuperModOnly", "AdminOnly") {
 		route.Before("MemberOnly")
@@ -93,7 +94,7 @@ func MView(fname string, path string, args ...string) *RouteImpl {
 	return route
 }
 
-func MemberView(fname string, path string, args ...string) *RouteImpl {
+func MemberView(fname, path string, args ...string) *RouteImpl {
 	route := route(fname, path, false, false, args...)
 	if !route.hasBefore("SuperModOnly", "AdminOnly") {
 		route.Before("MemberOnly")
@@ -101,7 +102,7 @@ func MemberView(fname string, path string, args ...string) *RouteImpl {
 	return route
 }
 
-func ModView(fname string, path string, args ...string) *RouteImpl {
+func ModView(fname, path string, args ...string) *RouteImpl {
 	route := route(fname, path, false, false, args...)
 	if !route.hasBefore("AdminOnly") {
 		route.Before("SuperModOnly")
@@ -109,7 +110,7 @@ func ModView(fname string, path string, args ...string) *RouteImpl {
 	return route
 }
 
-func Action(fname string, path string, args ...string) *RouteImpl {
+func Action(fname, path string, args ...string) *RouteImpl {
 	route := route(fname, path, true, false, args...)
 	route.Before("NoSessionMismatch")
 	if !route.hasBefore("SuperModOnly", "AdminOnly") {
@@ -118,11 +119,11 @@ func Action(fname string, path string, args ...string) *RouteImpl {
 	return route
 }
 
-func AnonAction(fname string, path string, args ...string) *RouteImpl {
+func AnonAction(fname, path string, args ...string) *RouteImpl {
 	return route(fname, path, true, false, args...).Before("ParseForm")
 }
 
-func Special(fname string, path string, args ...string) *RouteImpl {
+func Special(fname, path string, args ...string) *RouteImpl {
 	return route(fname, path, false, true, args...).LitBefore("req.URL.Path += extraData")
 }
 
@@ -131,7 +132,7 @@ type uploadAction struct {
 	Route *RouteImpl
 }
 
-func UploadAction(fname string, path string, args ...string) *uploadAction {
+func UploadAction(fname, path string, args ...string) *uploadAction {
 	route := route(fname, path, true, false, args...)
 	if !route.hasBefore("SuperModOnly", "AdminOnly") {
 		route.Before("MemberOnly")
@@ -154,6 +155,6 @@ type RouteSet struct {
 	Items []*RouteImpl
 }
 
-func Set(name string, path string, routes ...*RouteImpl) RouteSet {
+func Set(name, path string, routes ...*RouteImpl) RouteSet {
 	return RouteSet{name, path, routes}
 }

routes/stubs.go

@@ -7,6 +7,12 @@ func UploadedFile() {
 }
 func BadRoute() {
 }
+func Favicon() {
+}
+
+// TODO: Temporary stub for handling route group errors
+func Error() {
+}
 
 // Real implementation is in router_gen/main.go, this is just a stub to map the analytics onto
 func HTTPSRedirect() {

schema/mssql/query_viewchunks.sql

@@ -1,5 +1,6 @@
 CREATE TABLE [viewchunks] (
 	[count] int DEFAULT 0 not null,
+	[avg] int DEFAULT 0 not null,
 	[createdAt] datetime not null,
 	[route] nvarchar (200) not null
 );

schema/mysql/query_viewchunks.sql

@@ -1,5 +1,6 @@
 CREATE TABLE `viewchunks` (
 	`count` int DEFAULT 0 not null,
+	`avg` int DEFAULT 0 not null,
 	`createdAt` datetime not null,
 	`route` varchar(200) not null
 );

schema/pgsql/query_viewchunks.sql

@@ -1,5 +1,6 @@
 CREATE TABLE "viewchunks" (
 	`count` int DEFAULT 0 not null,
+	`avg` int DEFAULT 0 not null,
 	`createdAt` timestamp not null,
 	`route` varchar (200) not null
 );