Reply attachments can be managed too now.

Added eight database indices.
Fixed a bug where the second tick wouldn't fire.
Tweaked the .topic_forum in Nox by a pixel.
Replaced some _installer strings with blank strings for consistency with the builders.
Greatly reduced the number of allocations in the user agent parser.
Added ampersand entities in more attachment URLs to avoid accidental mangling.
.edit_source is now hidden for guests.
Guest noavatars are now pre-calculated to reduce the number of allocations.
Lazily initialised a couple of maps in ViewTopic to reduce the number of unnecessary allocations slightly.

Added the unsafe BytesToString function. Please don't use this, if you don't have to.
Added the AddIndex method to the adapter and associated components.
Added the /reply/attach/add/submit/ route.
Added the /reply/attach/remove/submit/ route.
Added the topic_alt_userinfo template.
Replaced Attachments.MiniTopicGet with MiniGetList.
Added Attachments.BulkMiniGetList.

Added a quick test for ReplyStore.Create.
Added BenchmarkPopulateTopicWithRouter.
Added BenchmarkTopicAdminFullPageRouteParallelWithRouter.
Added BenchmarkTopicGuestFullPageRouteParallelWithRouter.

You will need to run the updater or patcher for this commit.

cmd/query_gen/main.go

@@ -110,6 +110,15 @@ func writeStatements(adapter qgen.Adapter) error {
 }
 
 func seedTables(adapter qgen.Adapter) error {
+	qgen.Install.AddIndex("topics", "parentID", "parentID")
+	qgen.Install.AddIndex("replies", "tid", "tid")
+	qgen.Install.AddIndex("polls", "parentID", "parentID")
+	qgen.Install.AddIndex("likes", "targetItem", "targetItem")
+	qgen.Install.AddIndex("emails", "uid", "uid")
+	qgen.Install.AddIndex("attachments", "originID", "originID")
+	qgen.Install.AddIndex("attachments", "path", "path")
+	qgen.Install.AddIndex("activity_stream_matches", "watcher", "watcher")
+
 	qgen.Install.SimpleInsert("sync", "last_update", "UTC_TIMESTAMP()")
 	qgen.Install.SimpleInsert("settings", "name, content, type, constraints", "'activation_type','1','list','1-3'")
 	qgen.Install.SimpleInsert("settings", "name, content, type", "'bigpost_min_words','250','int'")

common/attachments.go

@@ -23,7 +23,8 @@ type MiniAttachment struct {
 
 type AttachmentStore interface {
 	Get(id int) (*MiniAttachment, error)
-	MiniTopicGet(id int) (alist []*MiniAttachment, err error)
+	MiniGetList(originTable string, originID int) (alist []*MiniAttachment, err error)
+	BulkMiniGetList(originTable string, ids []int) (amap map[int][]*MiniAttachment, err error)
 	Add(sectionID int, sectionTable string, originID int, originTable string, uploadedBy int, path string) (int, error)
 	GlobalCount() int
 	CountIn(originTable string, oid int) int
@@ -33,7 +34,7 @@ type AttachmentStore interface {
 
 type DefaultAttachmentStore struct {
 	get         *sql.Stmt
-	getByTopic  *sql.Stmt
+	getByObj    *sql.Stmt
 	add         *sql.Stmt
 	count       *sql.Stmt
 	countIn     *sql.Stmt
@@ -45,7 +46,7 @@ func NewDefaultAttachmentStore() (*DefaultAttachmentStore, error) {
 	acc := qgen.NewAcc()
 	return &DefaultAttachmentStore{
 		get:         acc.Select("attachments").Columns("originID, sectionID, uploadedBy, path").Where("attachID = ?").Prepare(),
-		getByTopic:  acc.Select("attachments").Columns("attachID, sectionID, uploadedBy, path").Where("originTable = 'topics' AND originID = ?").Prepare(),
+		getByObj:    acc.Select("attachments").Columns("attachID, sectionID, uploadedBy, path").Where("originTable = ? AND originID = ?").Prepare(),
 		add:         acc.Insert("attachments").Columns("sectionID, sectionTable, originID, originTable, uploadedBy, path").Fields("?,?,?,?,?,?").Prepare(),
 		count:       acc.Count("attachments").Prepare(),
 		countIn:     acc.Count("attachments").Where("originTable = ? and originID = ?").Prepare(),
@@ -54,12 +55,11 @@ func NewDefaultAttachmentStore() (*DefaultAttachmentStore, error) {
 	}, acc.FirstError()
 }
 
-// TODO: Make this more generic so we can use it for reply attachments too
+func (store *DefaultAttachmentStore) MiniGetList(originTable string, originID int) (alist []*MiniAttachment, err error) {
-func (store *DefaultAttachmentStore) MiniTopicGet(id int) (alist []*MiniAttachment, err error) {
+	rows, err := store.getByObj.Query(originTable, originID)
-	rows, err := store.getByTopic.Query(id)
 	defer rows.Close()
 	for rows.Next() {
-		attach := &MiniAttachment{OriginID: id}
+		attach := &MiniAttachment{OriginID: originID}
 		err := rows.Scan(&attach.ID, &attach.SectionID, &attach.UploadedBy, &attach.Path)
 		if err != nil {
 			return nil, err
@@ -75,6 +75,43 @@ func (store *DefaultAttachmentStore) MiniTopicGet(id int) (alist []*MiniAttachme
 	return alist, rows.Err()
 }
 
+func (store *DefaultAttachmentStore) BulkMiniGetList(originTable string, ids []int) (amap map[int][]*MiniAttachment, err error) {
+	if len(ids) == 0 {
+		return nil, sql.ErrNoRows
+	}
+	if len(ids) == 1 {
+		res, err := store.MiniGetList(originTable, ids[0])
+		return map[int][]*MiniAttachment{0: res}, err
+	}
+
+	amap = make(map[int][]*MiniAttachment)
+	var buffer []*MiniAttachment
+	var currentID int
+	rows, err := qgen.NewAcc().Select("attachments").Columns("attachID, sectionID, originID, uploadedBy, path").Where("originTable = ?").In("originID", ids).Orderby("originID ASC").Query(originTable)
+	defer rows.Close()
+	for rows.Next() {
+		attach := &MiniAttachment{}
+		err := rows.Scan(&attach.ID, &attach.SectionID, &attach.OriginID, &attach.UploadedBy, &attach.Path)
+		if err != nil {
+			return nil, err
+		}
+		extarr := strings.Split(attach.Path, ".")
+		if len(extarr) < 2 {
+			return nil, errors.New("corrupt attachment path")
+		}
+		attach.Ext = extarr[len(extarr)-1]
+		attach.Image = ImageFileExts.Contains(attach.Ext)
+		if attach.ID != currentID {
+			if len(buffer) > 0 {
+				amap[currentID] = buffer
+				buffer = nil
+			}
+		}
+		buffer = append(buffer, attach)
+	}
+	return amap, rows.Err()
+}
+
 func (store *DefaultAttachmentStore) Get(id int) (*MiniAttachment, error) {
 	attach := &MiniAttachment{ID: id}
 	err := store.get.QueryRow(id).Scan(&attach.OriginID, &attach.SectionID, &attach.UploadedBy, &attach.Path)

common/counters/systems.go

@@ -1,8 +1,11 @@
 package counters
 
-import "database/sql"
+import (
-import "github.com/Azareal/Gosora/common"
+	"database/sql"
-import "github.com/Azareal/Gosora/query_gen"
+
+	"github.com/Azareal/Gosora/common"
+	"github.com/Azareal/Gosora/query_gen"
+)
 
 var OSViewCounter *DefaultOSViewCounter
 

common/reply.go

@@ -39,8 +39,11 @@ type ReplyUser struct {
 	IPAddress     string
 	Liked         bool
 	LikeCount     int
+	AttachCount   int
 	ActionType    string
 	ActionIcon    string
+
+	Attachments []*MiniAttachment
 }
 
 type Reply struct {

common/site.go

@@ -129,6 +129,7 @@ func LoadConfig() error {
 
 func ProcessConfig() (err error) {
 	Config.Noavatar = strings.Replace(Config.Noavatar, "{site_url}", Site.URL, -1)
+	guestAvatar = GuestAvatar{buildNoavatar(0, 200), buildNoavatar(0, 48)}
 	Site.Host = Site.URL
 	if Site.Port != "80" && Site.Port != "443" {
 		Site.URL = strings.TrimSuffix(Site.URL, "/")

common/template_init.go

@@ -231,7 +231,7 @@ func CompileTemplates() error {
 	var replyList []ReplyUser
 	// TODO: Do we want the UID on this to be 0?
 	avatar, microAvatar = BuildAvatar(0, "")
-	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
+	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach})
 
 	var varList = make(map[string]tmpl.VarItem)
 	var compile = func(name string, expects string, expectsInt interface{}) (tmpl string, err error) {
@@ -456,7 +456,7 @@ func CompileJSTemplates() error {
 	var replyList []ReplyUser
 	// TODO: Do we really want the UID here to be zero?
 	avatar, microAvatar = BuildAvatar(0, "")
-	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
+	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, 1, "", "", miniAttach})
 
 	varList = make(map[string]tmpl.VarItem)
 	header.Title = "Topic Name"

common/templates/templates.go

@@ -1120,6 +1120,10 @@ func (c *CTemplateSet) compileIfVarSub(con CContext, varname string) (out string
 
 		cur = cur.FieldByName(bit)
 		out += "." + bit
+		if !cur.IsValid() {
+			fmt.Println("cur: ", cur)
+			panic(out + "^\n" + "Invalid value. Maybe, it doesn't exist?")
+		}
 		stepInterface()
 		if !cur.IsValid() {
 			fmt.Println("cur: ", cur)

common/user.go

@@ -446,6 +446,13 @@ func (user *User) InitPerms() {
 	}
 }
 
+var guestAvatar GuestAvatar
+
+type GuestAvatar struct {
+	Normal string
+	Micro  string
+}
+
 func buildNoavatar(uid int, width int) string {
 	return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1)
 }
@@ -464,6 +471,9 @@ func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar strin
 		}
 		return avatar, avatar
 	}
+	if uid == 0 {
+		return guestAvatar.Normal, guestAvatar.Micro
+	}
 	return buildNoavatar(uid, 200), buildNoavatar(uid, 48)
 }
 

gen_router.go

@@ -5,6 +5,7 @@ package main
 import (
 	"log"
 	"strings"
+	"bytes"
 	"strconv"
 	"compress/gzip"
 	"sync"
@@ -135,6 +136,8 @@ var RouteMap = map[string]interface{}{
 	"routes.ReplyEditSubmit": routes.ReplyEditSubmit,
 	"routes.ReplyDeleteSubmit": routes.ReplyDeleteSubmit,
 	"routes.ReplyLikeSubmit": routes.ReplyLikeSubmit,
+	"routes.AddAttachToReplySubmit": routes.AddAttachToReplySubmit,
+	"routes.RemoveAttachFromReplySubmit": routes.RemoveAttachFromReplySubmit,
 	"routes.ProfileReplyCreateSubmit": routes.ProfileReplyCreateSubmit,
 	"routes.ProfileReplyEditSubmit": routes.ProfileReplyEditSubmit,
 	"routes.ProfileReplyDeleteSubmit": routes.ProfileReplyDeleteSubmit,
@@ -270,24 +273,26 @@ var routeMapEnum = map[string]int{
 	"routes.ReplyEditSubmit": 110,
 	"routes.ReplyDeleteSubmit": 111,
 	"routes.ReplyLikeSubmit": 112,
-	"routes.ProfileReplyCreateSubmit": 113,
+	"routes.AddAttachToReplySubmit": 113,
-	"routes.ProfileReplyEditSubmit": 114,
+	"routes.RemoveAttachFromReplySubmit": 114,
-	"routes.ProfileReplyDeleteSubmit": 115,
+	"routes.ProfileReplyCreateSubmit": 115,
-	"routes.PollVote": 116,
+	"routes.ProfileReplyEditSubmit": 116,
-	"routes.PollResults": 117,
+	"routes.ProfileReplyDeleteSubmit": 117,
-	"routes.AccountLogin": 118,
+	"routes.PollVote": 118,
-	"routes.AccountRegister": 119,
+	"routes.PollResults": 119,
-	"routes.AccountLogout": 120,
+	"routes.AccountLogin": 120,
-	"routes.AccountLoginSubmit": 121,
+	"routes.AccountRegister": 121,
-	"routes.AccountLoginMFAVerify": 122,
+	"routes.AccountLogout": 122,
-	"routes.AccountLoginMFAVerifySubmit": 123,
+	"routes.AccountLoginSubmit": 123,
-	"routes.AccountRegisterSubmit": 124,
+	"routes.AccountLoginMFAVerify": 124,
-	"routes.DynamicRoute": 125,
+	"routes.AccountLoginMFAVerifySubmit": 125,
-	"routes.UploadedFile": 126,
+	"routes.AccountRegisterSubmit": 126,
-	"routes.StaticFile": 127,
+	"routes.DynamicRoute": 127,
-	"routes.RobotsTxt": 128,
+	"routes.UploadedFile": 128,
-	"routes.SitemapXml": 129,
+	"routes.StaticFile": 129,
-	"routes.BadRoute": 130,
+	"routes.RobotsTxt": 130,
+	"routes.SitemapXml": 131,
+	"routes.BadRoute": 132,
 }
 var reverseRouteMapEnum = map[int]string{ 
 	0: "routes.Overview",
@@ -403,24 +408,26 @@ var reverseRouteMapEnum = map[int]string{
 	110: "routes.ReplyEditSubmit",
 	111: "routes.ReplyDeleteSubmit",
 	112: "routes.ReplyLikeSubmit",
-	113: "routes.ProfileReplyCreateSubmit",
+	113: "routes.AddAttachToReplySubmit",
-	114: "routes.ProfileReplyEditSubmit",
+	114: "routes.RemoveAttachFromReplySubmit",
-	115: "routes.ProfileReplyDeleteSubmit",
+	115: "routes.ProfileReplyCreateSubmit",
-	116: "routes.PollVote",
+	116: "routes.ProfileReplyEditSubmit",
-	117: "routes.PollResults",
+	117: "routes.ProfileReplyDeleteSubmit",
-	118: "routes.AccountLogin",
+	118: "routes.PollVote",
-	119: "routes.AccountRegister",
+	119: "routes.PollResults",
-	120: "routes.AccountLogout",
+	120: "routes.AccountLogin",
-	121: "routes.AccountLoginSubmit",
+	121: "routes.AccountRegister",
-	122: "routes.AccountLoginMFAVerify",
+	122: "routes.AccountLogout",
-	123: "routes.AccountLoginMFAVerifySubmit",
+	123: "routes.AccountLoginSubmit",
-	124: "routes.AccountRegisterSubmit",
+	124: "routes.AccountLoginMFAVerify",
-	125: "routes.DynamicRoute",
+	125: "routes.AccountLoginMFAVerifySubmit",
-	126: "routes.UploadedFile",
+	126: "routes.AccountRegisterSubmit",
-	127: "routes.StaticFile",
+	127: "routes.DynamicRoute",
-	128: "routes.RobotsTxt",
+	128: "routes.UploadedFile",
-	129: "routes.SitemapXml",
+	129: "routes.StaticFile",
-	130: "routes.BadRoute",
+	130: "routes.RobotsTxt",
+	131: "routes.SitemapXml",
+	132: "routes.BadRoute",
 }
 var osMapEnum = map[string]int{ 
 	"unknown": 0,
@@ -501,32 +508,30 @@ var reverseAgentMapEnum = map[int]string{
 	28: "zgrab",
 }
 var markToAgent = map[string]string{ 
-	"OPR":"opera",
+	"OPR": "opera",
-	"Chrome":"chrome",
+	"Chrome": "chrome",
-	"Firefox":"firefox",
+	"Firefox": "firefox",
-	"MSIE":"internetexplorer",
+	"MSIE": "internetexplorer",
-	"Trident":"trident", // Hack to support IE11
+	"Trident": "trident",
-	"Edge":"edge",
+	"Edge": "edge",
-	"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
+	"Lynx": "lynx",
-	"SamsungBrowser":"samsung",
+	"SamsungBrowser": "samsung",
-	"UCBrowser":"ucbrowser",
+	"UCBrowser": "ucbrowser",
-
+	"Google": "googlebot",
-	"Google":"googlebot",
+	"Googlebot": "googlebot",
-	"Googlebot":"googlebot",
+	"yandex": "yandex",
-	"yandex": "yandex", // from the URL
+	"DuckDuckBot": "duckduckgo",
-	"DuckDuckBot":"duckduckgo",
+	"Baiduspider": "baidu",
-	"Baiduspider":"baidu",
+	"bingbot": "bing",
-	"bingbot":"bing",
+	"BingPreview": "bing",
-	"BingPreview":"bing",
+	"SeznamBot": "seznambot",
-	"SeznamBot":"seznambot",
+	"CloudFlare": "cloudflare",
-	"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
+	"Uptimebot": "uptimebot",
-	"Uptimebot":"uptimebot",
+	"Slackbot": "slackbot",
-	"Slackbot":"slackbot",
+	"Discordbot": "discord",
-	"Discordbot":"discord",
+	"Twitterbot": "twitter",
-	"Twitterbot":"twitter",
+	"Discourse": "discourse",
-	"Discourse":"discourse",
+	"zgrab": "zgrab",
-
-	"zgrab":"zgrab",
 }
 /*var agentRank = map[string]int{
 	"opera":9,
@@ -711,7 +716,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	counters.GlobalViewCounter.Bump()
 	
 	if prefix == "/static" {
-		counters.RouteViewCounter.Bump(127)
+		counters.RouteViewCounter.Bump(129)
 		req.URL.Path += extraData
 		routes.StaticFile(w, req)
 		return
@@ -738,41 +743,48 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			r.DumpRequest(req,"Blank UA: " + prepend)
 		}
 	} else {		
-		var runeEquals = func(a []rune, b []rune) bool {
-			if len(a) != len(b) {
-				return false
-			}
-			for i, item := range a {
-				if item != b[i] {
-					return false
-				}
-			}
-			return true
-		}
-		
 		// WIP UA Parser
-		var indices []int
 		var items []string
-		var buffer []rune
+		var buffer []byte
-		for index, item := range ua {
+		var os string
+		for _, item := range StringToBytes(ua) {
 			if (item > 64 && item < 91) || (item > 96 && item < 123) {
 				buffer = append(buffer, item)
-			} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && (runeEquals(buffer,[]rune("http")) || runeEquals(buffer,[]rune("rv")))) || item == ',' || item == '/' {
+			} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' {
 				if len(buffer) != 0 {
+					if len(buffer) > 2 {
+						// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
+						switch(BytesToString(buffer)) {
+						case "Windows":
+							os = "windows"
+						case "Linux":
+							os = "linux"
+						case "Mac":
+							os = "mac"
+						case "iPhone":
+							os = "iphone"
+						case "Android":
+							os = "android"
+						case "like":
+							// Skip this word
+						default:
 							items = append(items, string(buffer))
-					indices = append(indices, index - 1)
+						}
+					}
 					buffer = buffer[:0]
 				}
 			} else {
 				// TODO: Test this
 				items = items[:0]
-				indices = indices[:0]
 				r.SuspiciousRequest(req,"Illegal char in UA")
 				r.requestLogger.Print("UA Buffer: ", buffer)
 				r.requestLogger.Print("UA Buffer String: ", string(buffer))
 				break
 			}
 		}
+		if os == "" {
+			os = "unknown"
+		}
 
 		// Iterate over this in reverse as the real UA tends to be on the right side
 		var agent string
@@ -789,24 +801,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			r.requestLogger.Print("parsed agent: ", agent)
 		}
 
-		var os string
-		for _, mark := range items {
-			switch(mark) {
-			case "Windows":
-				os = "windows"
-			case "Linux":
-				os = "linux"
-			case "Mac":
-				os = "mac"
-			case "iPhone":
-				os = "iphone"
-			case "Android":
-				os = "android"
-			}
-		}
-		if os == "" {
-			os = "unknown"
-		}
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("os: ", os)
 			r.requestLogger.Printf("items: %+v\n",items)
@@ -1884,6 +1878,36 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 					
 					counters.RouteViewCounter.Bump(112)
 					err = routes.ReplyLikeSubmit(w,req,user,extraData)
+				case "/reply/attach/add/submit/":
+					err = common.MemberOnly(w,req,user)
+					if err != nil {
+						return err
+					}
+					
+					err = common.HandleUploadRoute(w,req,user,int(common.Config.MaxRequestSize))
+					if err != nil {
+					return err
+					}
+					err = common.NoUploadSessionMismatch(w,req,user)
+					if err != nil {
+						return err
+					}
+					
+					counters.RouteViewCounter.Bump(113)
+					err = routes.AddAttachToReplySubmit(w,req,user,extraData)
+				case "/reply/attach/remove/submit/":
+					err = common.NoSessionMismatch(w,req,user)
+					if err != nil {
+						return err
+					}
+					
+					err = common.MemberOnly(w,req,user)
+					if err != nil {
+						return err
+					}
+					
+					counters.RouteViewCounter.Bump(114)
+					err = routes.RemoveAttachFromReplySubmit(w,req,user,extraData)
 			}
 		case "/profile":
 			switch(req.URL.Path) {
@@ -1898,7 +1922,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(113)
+					counters.RouteViewCounter.Bump(115)
 					err = routes.ProfileReplyCreateSubmit(w,req,user)
 				case "/profile/reply/edit/submit/":
 					err = common.NoSessionMismatch(w,req,user)
@@ -1911,7 +1935,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(114)
+					counters.RouteViewCounter.Bump(116)
 					err = routes.ProfileReplyEditSubmit(w,req,user,extraData)
 				case "/profile/reply/delete/submit/":
 					err = common.NoSessionMismatch(w,req,user)
@@ -1924,7 +1948,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(115)
+					counters.RouteViewCounter.Bump(117)
 					err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData)
 			}
 		case "/poll":
@@ -1940,23 +1964,23 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(116)
+					counters.RouteViewCounter.Bump(118)
 					err = routes.PollVote(w,req,user,extraData)
 				case "/poll/results/":
-					counters.RouteViewCounter.Bump(117)
+					counters.RouteViewCounter.Bump(119)
 					err = routes.PollResults(w,req,user,extraData)
 			}
 		case "/accounts":
 			switch(req.URL.Path) {
 				case "/accounts/login/":
-					counters.RouteViewCounter.Bump(118)
+					counters.RouteViewCounter.Bump(120)
 				head, err := common.UserCheck(w,req,&user)
 				if err != nil {
 					return err
 				}
 					err = routes.AccountLogin(w,req,user,head)
 				case "/accounts/create/":
-					counters.RouteViewCounter.Bump(119)
+					counters.RouteViewCounter.Bump(121)
 				head, err := common.UserCheck(w,req,&user)
 				if err != nil {
 					return err
@@ -1973,7 +1997,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(120)
+					counters.RouteViewCounter.Bump(122)
 					err = routes.AccountLogout(w,req,user)
 				case "/accounts/login/submit/":
 					err = common.ParseForm(w,req,user)
@@ -1981,10 +2005,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(121)
+					counters.RouteViewCounter.Bump(123)
 					err = routes.AccountLoginSubmit(w,req,user)
 				case "/accounts/mfa_verify/":
-					counters.RouteViewCounter.Bump(122)
+					counters.RouteViewCounter.Bump(124)
 				head, err := common.UserCheck(w,req,&user)
 				if err != nil {
 					return err
@@ -1996,7 +2020,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(123)
+					counters.RouteViewCounter.Bump(125)
 					err = routes.AccountLoginMFAVerifySubmit(w,req,user)
 				case "/accounts/create/submit/":
 					err = common.ParseForm(w,req,user)
@@ -2004,7 +2028,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 						return err
 					}
 					
-					counters.RouteViewCounter.Bump(124)
+					counters.RouteViewCounter.Bump(126)
 					err = routes.AccountRegisterSubmit(w,req,user)
 			}
 		/*case "/sitemaps": // TODO: Count these views
@@ -2020,7 +2044,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 				w.Header().Del("Content-Type")
 				w.Header().Del("Content-Encoding")
 			}
-			counters.RouteViewCounter.Bump(126)
+			counters.RouteViewCounter.Bump(128)
 			req.URL.Path += extraData
 			// TODO: Find a way to propagate errors up from this?
 			r.UploadHandler(w,req) // TODO: Count these views
@@ -2030,10 +2054,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 			// TODO: Add support for favicons and robots.txt files
 			switch(extraData) {
 				case "robots.txt":
-					counters.RouteViewCounter.Bump(128)
+					counters.RouteViewCounter.Bump(130)
 					return routes.RobotsTxt(w,req)
 				/*case "sitemap.xml":
-					counters.RouteViewCounter.Bump(129)
+					counters.RouteViewCounter.Bump(131)
 					return routes.SitemapXml(w,req)*/
 			}
 			return common.NotFound(w,req,nil)
@@ -2044,7 +2068,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 			r.RUnlock()
 			
 			if ok {
-				counters.RouteViewCounter.Bump(125) // TODO: Be more specific about *which* dynamic route it is
+				counters.RouteViewCounter.Bump(127) // TODO: Be more specific about *which* dynamic route it is
 				req.URL.Path += extraData
 				return handle(w,req,user)
 			}
@@ -2055,7 +2079,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
 			} else {
 				r.DumpRequest(req,"Bad Route")
 			}
-			counters.RouteViewCounter.Bump(130)
+			counters.RouteViewCounter.Bump(132)
 			return common.NotFound(w,req,nil)
 	}
 	return err

general_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"runtime/debug"
 
 	"github.com/pkg/errors"
 
@@ -109,6 +110,7 @@ func init() {
 	}
 }
 
+const benchTidI = 1
 const benchTid = "1"
 
 // TODO: Swap out LocalError for a panic for this?
@@ -175,7 +177,7 @@ func BenchmarkTopicAdminRouteParallelWithRouter(b *testing.B) {
 	}
 	uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: common.Year}
 	sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: common.Year}
-	path := "/topic/hm."+benchTid
+	path := "/topic/hm." + benchTid
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
@@ -229,7 +231,7 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
 	}
 	uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: common.Year}
 	sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: common.Year}
-	path := "/topic/hm."+benchTid
+	path := "/topic/hm." + benchTid
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
@@ -246,7 +248,7 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
 				b.Fatal("HTTP Error!")
 			}
 
-{
+			{
 				w := httptest.NewRecorder()
 				req := httptest.NewRequest("GET", path, bytes.NewReader(nil))
 				req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
@@ -257,7 +259,7 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
 					b.Log(w.Body)
 					b.Fatal("HTTP Error!")
 				}
-}
+			}
 		}
 	})
 
@@ -428,6 +430,54 @@ func BenchmarkProfileGuestRouteParallelWithRouter(b *testing.B) {
 	obRoute(b, "/profile/admin.1")
 }
 
+func BenchmarkPopulateTopicWithRouter(b *testing.B) {
+	b.ReportAllocs()
+	topic, err := common.Topics.Get(benchTidI)
+	if err != nil {
+		debug.PrintStack()
+		b.Fatal(err)
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			for i := 0; i < 25; i++ {
+				_, err := common.Rstore.Create(topic, "hiii", "::1", 1)
+				if err != nil {
+					debug.PrintStack()
+					b.Fatal(err)
+				}
+			}
+		}
+	})
+}
+
+//var fullPage = false
+
+func BenchmarkTopicAdminFullPageRouteParallelWithRouter(b *testing.B) {
+	/*if !fullPage {
+		topic, err := common.Topics.Get(benchTidI)
+		panicIfErr(err)
+		for i := 0; i < 25; i++ {
+			_, err = common.Rstore.Create(topic, "hiii", "::1", 1)
+			panicIfErr(err)
+		}
+		fullPage = true
+	}*/
+	BenchmarkTopicAdminRouteParallel(b)
+}
+
+func BenchmarkTopicGuestFullPageRouteParallelWithRouter(b *testing.B) {
+	/*if !fullPage {
+		topic, err := common.Topics.Get(benchTidI)
+		panicIfErr(err)
+		for i := 0; i < 25; i++ {
+			_, err = common.Rstore.Create(topic, "hiii", "::1", 1)
+			panicIfErr(err)
+		}
+		fullPage = true
+	}*/
+	obRoute(b, "/topic/hm."+benchTid)
+}
+
 // TODO: Make these routes compatible with the changes to the router
 /*
 func BenchmarkForumsAdminRouteParallel(b *testing.B) {

main.go

@@ -371,14 +371,7 @@ func main() {
 	// TODO: Could we expand this to attachments and other things too?
 	thumbChan := make(chan bool)
 	go common.ThumbTask(thumbChan)
-
+	go tickLoop(thumbChan)
-	// TODO: Write tests for these
-	// Run this goroutine once every half second
-	halfSecondTicker := time.NewTicker(time.Second / 2)
-	secondTicker := time.NewTicker(time.Second)
-	fifteenMinuteTicker := time.NewTicker(15 * time.Minute)
-	hourTicker := time.NewTicker(time.Hour)
-	go tickLoop(thumbChan, halfSecondTicker, secondTicker, fifteenMinuteTicker, hourTicker)
 
 	// Resource Management Goroutine
 	go func() {
@@ -390,6 +383,7 @@ func main() {
 
 		var lastEvictedCount int
 		var couldNotDealloc bool
+		var secondTicker = time.NewTicker(time.Second)
 		for {
 			select {
 			case <-secondTicker.C:

misc_test.go

@@ -749,6 +749,10 @@ func TestReplyStore(t *testing.T) {
 	topic, err = common.Topics.Get(1)
 	expectNilErr(t, err)
 	expect(t, topic.PostCount == 3, fmt.Sprintf("TID #1's post count should be three, not %d", topic.PostCount))
+
+	rid, err = common.Rstore.Create(topic, "hiii", "::1", 1)
+	expectNilErr(t, err)
+	replyTest(rid, topic.ID, 1, "hiii", "::1")
 }
 
 func TestProfileReplyStore(t *testing.T) {

patcher/patches.go

@@ -23,6 +23,7 @@ func init() {
 	addPatch(9, patch9)
 	addPatch(10, patch10)
 	addPatch(11, patch11)
+	addPatch(12, patch12)
 }
 
 func patch0(scanner *bufio.Scanner) (err error) {
@@ -467,3 +468,39 @@ func patch11(scanner *bufio.Scanner) error {
 		return err
 	})*/
 }
+
+func patch12(scanner *bufio.Scanner) error {
+	err := execStmt(qgen.Builder.AddIndex("topics", "parentID", "parentID"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("replies", "tid", "tid"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("polls", "parentID", "parentID"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("likes", "targetItem", "targetItem"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("emails", "uid", "uid"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("attachments", "originID", "originID"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("attachments", "path", "path"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddIndex("activity_stream_matches", "watcher", "watcher"))
+	if err != nil {
+		return err
+	}
+	return nil
+}

public/global.js

@@ -427,6 +427,8 @@ function mainInit(){
 	$(".edit_item").click(function(event){
 		event.preventDefault();
 		let blockParent = this.closest('.editable_parent');
+		$(blockParent).find('.hide_on_edit').addClass("edit_opened");
+		$(blockParent).find('.show_on_edit').addClass("edit_opened");
 		let srcNode = blockParent.querySelector(".edit_source");
 		let block = blockParent.querySelector('.editable_block');
 		block.classList.add("in_edit");
@@ -438,6 +440,8 @@ function mainInit(){
 
 		$(".submit_edit").click(function(event){
 			event.preventDefault();
+			$(blockParent).find('.hide_on_edit').removeClass("edit_opened");
+			$(blockParent).find('.show_on_edit').removeClass("edit_opened");
 			block.classList.remove("in_edit");
 			let newContent = block.querySelector('textarea').value;
 			block.innerHTML = quickParse(newContent);
@@ -668,7 +672,7 @@ function mainInit(){
 					$(".attach_item_copy").unbind("click");
 					bindAttachItems()
 				});
-				req.open("POST","//"+window.location.host+"/topic/attach/add/submit/"+fileDock.getAttribute("tid"));
+				req.open("POST","//"+window.location.host+"/"+fileDock.getAttribute("type")+"/attach/add/submit/"+fileDock.getAttribute("id"));
 				req.send(formData);
 			});
 		} catch(e) {
@@ -714,14 +718,20 @@ function mainInit(){
 		}
 	}
 
-	var uploadFiles = document.getElementById("upload_files");
+	let uploadFiles = document.getElementById("upload_files");
 	if(uploadFiles != null) {
 		uploadFiles.addEventListener("change", uploadAttachHandler, false);
 	}
-	var uploadFilesOp = document.getElementById("upload_files_op");
+	let uploadFilesOp = document.getElementById("upload_files_op");
 	if(uploadFilesOp != null) {
 		uploadFilesOp.addEventListener("change", uploadAttachHandler2, false);
 	}
+	let uploadFilesPost = document.getElementsByClassName("upload_files_post");
+	if(uploadFilesPost != null) {
+		for(let i = 0; i < uploadFilesPost.length; i++) {
+			uploadFilesPost[i].addEventListener("change", uploadAttachHandler2, false);
+		}
+	}
 
 	function copyToClipboard(str) {
 		const el = document.createElement('textarea');
@@ -772,7 +782,7 @@ function mainInit(){
 		
 		let req = new XMLHttpRequest();
 		let fileDock = this.closest(".attach_edit_bay");
-		req.open("POST","//"+window.location.host+"/topic/attach/remove/submit/"+fileDock.getAttribute("tid"),true);
+		req.open("POST","//"+window.location.host+"/"+fileDock.getAttribute("type")+"/attach/remove/submit/"+fileDock.getAttribute("id"),true);
 		req.send(formData);
 	});
 	

query_gen/builder.go

@@ -112,6 +112,10 @@ func (build *builder) AddColumn(table string, column DBTableColumn) (stmt *sql.S
 	return build.prepare(build.adapter.AddColumn("", table, column))
 }
 
+func (build *builder) AddIndex(table string, iname string, colname string) (stmt *sql.Stmt, err error) {
+	return build.prepare(build.adapter.AddIndex("", table, iname, colname))
+}
+
 func (build *builder) SimpleInsert(table string, columns string, fields string) (stmt *sql.Stmt, err error) {
 	return build.prepare(build.adapter.SimpleInsert("", table, columns, fields))
 }

query_gen/install.go

@@ -54,7 +54,7 @@ func (install *installer) CreateTable(table string, charset string, collation st
 	if err != nil {
 		return err
 	}
-	res, err := install.adapter.CreateTable("_installer", table, charset, collation, columns, keys)
+	res, err := install.adapter.CreateTable("", table, charset, collation, columns, keys)
 	if err != nil {
 		return err
 	}
@@ -67,13 +67,31 @@ func (install *installer) CreateTable(table string, charset string, collation st
 	return nil
 }
 
+// TODO: Let plugins manipulate the parameters like in CreateTable
+func (install *installer) AddIndex(table string, iname string, colname string) error {
+	err := install.RunHook("AddIndexStart", table, iname, colname)
+	if err != nil {
+		return err
+	}
+	res, err := install.adapter.AddIndex("", table, iname, colname)
+	if err != nil {
+		return err
+	}
+	err = install.RunHook("AddIndexAfter", table, iname, colname)
+	if err != nil {
+		return err
+	}
+	install.instructions = append(install.instructions, DB_Install_Instruction{table, res, "index"})
+	return nil
+}
+
 // TODO: Let plugins manipulate the parameters like in CreateTable
 func (install *installer) SimpleInsert(table string, columns string, fields string) error {
 	err := install.RunHook("SimpleInsertStart", table, columns, fields)
 	if err != nil {
 		return err
 	}
-	res, err := install.adapter.SimpleInsert("_installer", table, columns, fields)
+	res, err := install.adapter.SimpleInsert("", table, columns, fields)
 	if err != nil {
 		return err
 	}

query_gen/mssql.go

@@ -146,6 +146,21 @@ func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTable
 	return querystr, nil
 }
 
+// TODO: Implement this
+// TODO: Test to make sure everything works here
+func (adapter *MssqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if iname == "" {
+		return "", errors.New("You need a name for the index")
+	}
+	if colname == "" {
+		return "", errors.New("You need a name for the column")
+	}
+	return "", errors.New("not implemented")
+}
+
 func (adapter *MssqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")
@@ -1134,7 +1149,7 @@ func _gen_mssql() (err error) {
 
 // Internal methods, not exposed in the interface
 func (adapter *MssqlAdapter) pushStatement(name string, stype string, querystr string) {
-	if name[0] == '_' {
+	if name == "" {
 		return
 	}
 	adapter.Buffer[name] = DBStmt{querystr, stype}

query_gen/mysql.go

@@ -185,6 +185,24 @@ func (adapter *MysqlAdapter) AddColumn(name string, table string, column DBTable
 	return querystr, nil
 }
 
+// TODO: Test to make sure everything works here
+func (adapter *MysqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if iname == "" {
+		return "", errors.New("You need a name for the index")
+	}
+	if colname == "" {
+		return "", errors.New("You need a name for the column")
+	}
+
+	querystr := "ALTER TABLE `" + table + "` ADD INDEX " + "`" + iname + "` (`" + colname + "`);"
+	// TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator
+	adapter.pushStatement(name, "add-index", querystr)
+	return querystr, nil
+}
+
 func (adapter *MysqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")

query_gen/pgsql.go

@@ -120,6 +120,21 @@ func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTable
 	return "", nil
 }
 
+// TODO: Implement this
+// TODO: Test to make sure everything works here
+func (adapter *PgsqlAdapter) AddIndex(name string, table string, iname string, colname string) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if iname == "" {
+		return "", errors.New("You need a name for the index")
+	}
+	if colname == "" {
+		return "", errors.New("You need a name for the column")
+	}
+	return "", errors.New("not implemented")
+}
+
 // TODO: Test this
 // ! We need to get the last ID out of this somehow, maybe add returning to every query? Might require some sort of wrapper over the sql statements
 func (adapter *PgsqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {

query_gen/querygen.go

@@ -109,6 +109,7 @@ type Adapter interface {
 	// TODO: Some way to add indices and keys
 	// TODO: Test this
 	AddColumn(name string, table string, column DBTableColumn) (string, error)
+	AddIndex(name string, table string, iname string, colname string) (string, error)
 	SimpleInsert(name string, table string, columns string, fields string) (string, error)
 	SimpleUpdate(up *updatePrebuilder) (string, error)
 	SimpleUpdateSelect(up *updatePrebuilder) (string, error) // ! Experimental

router_gen/main.go

@@ -16,6 +16,8 @@ type TmplVars struct {
 	AllRouteMap       map[string]int
 	AllAgentNames     []string
 	AllAgentMap       map[string]int
+	AllAgentMarkNames []string
+	AllAgentMarks     map[string]string
 	AllOSNames        []string
 	AllOSMap          map[string]int
 }
@@ -227,6 +229,64 @@ func main() {
 		tmplVars.AllAgentMap[agent] = id
 	}
 
+	tmplVars.AllAgentMarkNames = []string{
+		"OPR",
+		"Chrome",
+		"Firefox",
+		"MSIE",
+		"Trident",
+		"Edge",
+		"Lynx",
+		"SamsungBrowser",
+		"UCBrowser",
+
+		"Google",
+		"Googlebot",
+		"yandex",
+		"DuckDuckBot",
+		"Baiduspider",
+		"bingbot",
+		"BingPreview",
+		"SeznamBot",
+		"CloudFlare",
+		"Uptimebot",
+		"Slackbot",
+		"Discordbot",
+		"Twitterbot",
+		"Discourse",
+
+		"zgrab",
+	}
+
+	tmplVars.AllAgentMarks = map[string]string{
+		"OPR":            "opera",
+		"Chrome":         "chrome",
+		"Firefox":        "firefox",
+		"MSIE":           "internetexplorer",
+		"Trident":        "trident", // Hack to support IE11
+		"Edge":           "edge",
+		"Lynx":           "lynx", // There's a rare android variant of lynx which isn't covered by this
+		"SamsungBrowser": "samsung",
+		"UCBrowser":      "ucbrowser",
+
+		"Google":      "googlebot",
+		"Googlebot":   "googlebot",
+		"yandex":      "yandex", // from the URL
+		"DuckDuckBot": "duckduckgo",
+		"Baiduspider": "baidu",
+		"bingbot":     "bing",
+		"BingPreview": "bing",
+		"SeznamBot":   "seznambot",
+		"CloudFlare":  "cloudflare", // Track alwayson specifically in case there are other bots?
+		"Uptimebot":   "uptimebot",
+		"Slackbot":    "slackbot",
+		"Discordbot":  "discord",
+		"Twitterbot":  "twitter",
+		"Discourse":   "discourse",
+
+		"zgrab": "zgrab",
+	}
+
 	var fileData = `// Code generated by Gosora's Router Generator. DO NOT EDIT.
 /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */
 package main
@@ -234,6 +294,7 @@ package main
 import (
 	"log"
 	"strings"
+	"bytes"
 	"strconv"
 	"compress/gzip"
 	"sync"
@@ -273,33 +334,8 @@ var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
 var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}}
 	{{$index}}: "{{$element}}",{{end}}
 }
-var markToAgent = map[string]string{
+var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}}
-	"OPR":"opera",
+	"{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}}
-	"Chrome":"chrome",
-	"Firefox":"firefox",
-	"MSIE":"internetexplorer",
-	"Trident":"trident", // Hack to support IE11
-	"Edge":"edge",
-	"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
-	"SamsungBrowser":"samsung",
-	"UCBrowser":"ucbrowser",
-
-	"Google":"googlebot",
-	"Googlebot":"googlebot",
-	"yandex": "yandex", // from the URL
-	"DuckDuckBot":"duckduckgo",
-	"Baiduspider":"baidu",
-	"bingbot":"bing",
-	"BingPreview":"bing",
-	"SeznamBot":"seznambot",
-	"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
-	"Uptimebot":"uptimebot",
-	"Slackbot":"slackbot",
-	"Discordbot":"discord",
-	"Twitterbot":"twitter",
-	"Discourse":"discourse",
-
-	"zgrab":"zgrab",
 }
 /*var agentRank = map[string]int{
 	"opera":9,
@@ -511,41 +547,48 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			r.DumpRequest(req,"Blank UA: " + prepend)
 		}
 	} else {		
-		var runeEquals = func(a []rune, b []rune) bool {
-			if len(a) != len(b) {
-				return false
-			}
-			for i, item := range a {
-				if item != b[i] {
-					return false
-				}
-			}
-			return true
-		}
-		
 		// WIP UA Parser
-		var indices []int
 		var items []string
-		var buffer []rune
+		var buffer []byte
-		for index, item := range ua {
+		var os string
+		for _, item := range StringToBytes(ua) {
 			if (item > 64 && item < 91) || (item > 96 && item < 123) {
 				buffer = append(buffer, item)
-			} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && (runeEquals(buffer,[]rune("http")) || runeEquals(buffer,[]rune("rv")))) || item == ',' || item == '/' {
+			} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' {
 				if len(buffer) != 0 {
+					if len(buffer) > 2 {
+						// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
+						switch(BytesToString(buffer)) {
+						case "Windows":
+							os = "windows"
+						case "Linux":
+							os = "linux"
+						case "Mac":
+							os = "mac"
+						case "iPhone":
+							os = "iphone"
+						case "Android":
+							os = "android"
+						case "like":
+							// Skip this word
+						default:
 							items = append(items, string(buffer))
-					indices = append(indices, index - 1)
+						}
+					}
 					buffer = buffer[:0]
 				}
 			} else {
 				// TODO: Test this
 				items = items[:0]
-				indices = indices[:0]
 				r.SuspiciousRequest(req,"Illegal char in UA")
 				r.requestLogger.Print("UA Buffer: ", buffer)
 				r.requestLogger.Print("UA Buffer String: ", string(buffer))
 				break
 			}
 		}
+		if os == "" {
+			os = "unknown"
+		}
 
 		// Iterate over this in reverse as the real UA tends to be on the right side
 		var agent string
@@ -562,24 +605,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			r.requestLogger.Print("parsed agent: ", agent)
 		}
 
-		var os string
-		for _, mark := range items {
-			switch(mark) {
-			case "Windows":
-				os = "windows"
-			case "Linux":
-				os = "linux"
-			case "Mac":
-				os = "mac"
-			case "iPhone":
-				os = "iphone"
-			case "Android":
-				os = "android"
-			}
-		}
-		if os == "" {
-			os = "unknown"
-		}
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("os: ", os)
 			r.requestLogger.Printf("items: %+v\n",items)

router_gen/routes.go

@@ -104,6 +104,8 @@ func replyRoutes() *RouteGroup {
 		Action("routes.ReplyLikeSubmit", "/reply/like/submit/", "extraData"),
 		//MemberView("routes.ReplyEdit","/reply/edit/","extraData"), // No js fallback
 		//MemberView("routes.ReplyDelete","/reply/delete/","extraData"), // No js confirmation page? We could have a confirmation modal for the JS case
+		UploadAction("routes.AddAttachToReplySubmit", "/reply/attach/add/submit/", "extraData").MaxSizeVar("int(common.Config.MaxRequestSize)"),
+		Action("routes.RemoveAttachFromReplySubmit", "/reply/attach/remove/submit/", "extraData"),
 	)
 }
 

routes/profile.go

@@ -100,7 +100,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User, heade
 		replyLikeCount := 0
 		// TODO: Add a hook here
 
-		replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
+		replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, 0, "", "", nil})
 	}
 	err = rows.Err()
 	if err != nil {

routes/reply.go

@@ -3,12 +3,14 @@ package routes
 import (
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/Azareal/Gosora/common"
 	"github.com/Azareal/Gosora/common/counters"
+	"github.com/Azareal/Gosora/common/phrases"
 	"github.com/Azareal/Gosora/query_gen"
 )
 
@@ -333,6 +335,106 @@ func ReplyDeleteSubmit(w http.ResponseWriter, r *http.Request, user common.User,
 	return nil
 }
 
+// TODO: Avoid uploading this again if the attachment already exists? They'll resolve to the same hash either way, but we could save on some IO / bandwidth here
+// TODO: Enforce the max request limit on all of this topic's attachments
+// TODO: Test this route
+func AddAttachToReplySubmit(w http.ResponseWriter, r *http.Request, user common.User, srid string) common.RouteError {
+	rid, err := strconv.Atoi(srid)
+	if err != nil {
+		return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r)
+	}
+
+	reply, err := common.Rstore.Get(rid)
+	if err == sql.ErrNoRows {
+		return common.PreErrorJS("You can't attach to something which doesn't exist!", w, r)
+	} else if err != nil {
+		return common.InternalErrorJS(err, w, r)
+	}
+
+	topic, err := common.Topics.Get(reply.ParentID)
+	if err != nil {
+		return common.NotFoundJS(w, r)
+	}
+
+	_, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID)
+	if ferr != nil {
+		return ferr
+	}
+	if !user.Perms.ViewTopic || !user.Perms.EditReply || !user.Perms.UploadFiles {
+		return common.NoPermissionsJS(w, r, user)
+	}
+	if topic.IsClosed && !user.Perms.CloseTopic {
+		return common.NoPermissionsJS(w, r, user)
+	}
+
+	// Handle the file attachments
+	pathMap, rerr := uploadAttachment(w, r, user, topic.ParentID, "forums", rid, "replies")
+	if rerr != nil {
+		// TODO: This needs to be a JS error...
+		return rerr
+	}
+	if len(pathMap) == 0 {
+		return common.InternalErrorJS(errors.New("no paths for attachment add"), w, r)
+	}
+
+	var elemStr string
+	for path, aids := range pathMap {
+		elemStr += "\"" + path + "\":\"" + aids + "\","
+	}
+	if len(elemStr) > 1 {
+		elemStr = elemStr[:len(elemStr)-1]
+	}
+
+	w.Write([]byte(`{"success":"1","elems":[{` + elemStr + `}]}`))
+	return nil
+}
+
+// TODO: Reduce the amount of duplication between this and RemoveAttachFromTopicSubmit
+func RemoveAttachFromReplySubmit(w http.ResponseWriter, r *http.Request, user common.User, srid string) common.RouteError {
+	rid, err := strconv.Atoi(srid)
+	if err != nil {
+		return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r)
+	}
+
+	reply, err := common.Rstore.Get(rid)
+	if err == sql.ErrNoRows {
+		return common.PreErrorJS("You can't attach from something which doesn't exist!", w, r)
+	} else if err != nil {
+		return common.InternalErrorJS(err, w, r)
+	}
+
+	topic, err := common.Topics.Get(reply.ParentID)
+	if err != nil {
+		return common.NotFoundJS(w, r)
+	}
+
+	_, ferr := common.SimpleForumUserCheck(w, r, &user, topic.ParentID)
+	if ferr != nil {
+		return ferr
+	}
+	if !user.Perms.ViewTopic || !user.Perms.EditReply {
+		return common.NoPermissionsJS(w, r, user)
+	}
+	if topic.IsClosed && !user.Perms.CloseTopic {
+		return common.NoPermissionsJS(w, r, user)
+	}
+
+	for _, said := range strings.Split(r.PostFormValue("aids"), ",") {
+		aid, err := strconv.Atoi(said)
+		if err != nil {
+			return common.LocalErrorJS(phrases.GetErrorPhrase("id_must_be_integer"), w, r)
+		}
+		rerr := deleteAttachment(w, r, user, aid, true)
+		if rerr != nil {
+			// TODO: This needs to be a JS error...
+			return rerr
+		}
+	}
+
+	w.Write(successJSONBytes)
+	return nil
+}
+
 // TODO: Move the profile reply routes to their own file?
 func ProfileReplyCreateSubmit(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
 	if !user.Perms.ViewTopic || !user.Perms.CreateReply {

routes/topic.go

@@ -32,7 +32,7 @@ var topicStmts TopicStmts
 func init() {
 	common.DbInits.Add(func(acc *qgen.Accumulator) error {
 		topicStmts = TopicStmts{
-			getReplies:    acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"),
+			getReplies:    acc.SimpleLeftJoin("replies", "users", "replies.rid, replies.content, replies.createdBy, replies.createdAt, replies.lastEdit, replies.lastEditBy, users.avatar, users.name, users.group, users.url_prefix, users.url_name, users.level, replies.ipaddress, replies.likeCount, replies.attachCount, replies.actionType", "replies.createdBy = users.uid", "replies.tid = ?", "replies.rid ASC", "?,?"),
 			getLikedTopic: acc.Select("likes").Columns("targetItem").Where("sentBy = ? && targetItem = ? && targetType = 'topics'").Prepare(),
 			// TODO: Less race-y attachment count updates
 			updateAttachs: acc.Update("topics").Set("attachCount = ?").Where("tid = ?").Prepare(),
@@ -109,7 +109,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header
 	}
 
 	if topic.AttachCount > 0 {
-		attachs, err := common.Attachments.MiniTopicGet(topic.ID)
+		attachs, err := common.Attachments.MiniGetList("topics", topic.ID)
 		if err != nil {
 			// TODO: We might want to be a little permissive here in-case of a desync?
 			return common.InternalError(err, w, r)
@@ -124,9 +124,18 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header
 
 	// Get the replies if we have any...
 	if topic.PostCount > 0 {
-		var likedMap = make(map[int]int)
+		var likedMap map[int]int
+		if user.Liked > 0 {
+			likedMap = make(map[int]int)
+		}
 		var likedQueryList = []int{user.ID}
 
+		var attachMap map[int]int
+		if user.Perms.EditReply {
+			attachMap = make(map[int]int)
+		}
+		var attachQueryList = []int{}
+
 		rows, err := topicStmts.getReplies.Query(topic.ID, offset, common.Config.ItemsPerPage)
 		if err == sql.ErrNoRows {
 			return common.LocalError("Bad Page. Some of the posts may have been deleted or you got here by directly typing in the page number.", w, r, user)
@@ -138,7 +147,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header
 		// TODO: Factor the user fields out and embed a user struct instead
 		replyItem := common.ReplyUser{ClassName: ""}
 		for rows.Next() {
-			err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType)
+			err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.AttachCount, &replyItem.ActionType)
 			if err != nil {
 				return common.InternalError(err, w, r)
 			}
@@ -196,6 +205,10 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header
 				likedMap[replyItem.ID] = len(tpage.ItemList)
 				likedQueryList = append(likedQueryList, replyItem.ID)
 			}
+			if user.Perms.EditReply && replyItem.AttachCount > 0 {
+				attachMap[replyItem.ID] = len(tpage.ItemList)
+				attachQueryList = append(attachQueryList, replyItem.ID)
+			}
 
 			header.Hooks.VhookNoRet("topic_reply_row_assign", &tpage, &replyItem)
 			// TODO: Use a pointer instead to make it easier to abstract this loop? What impact would this have on escape analysis?
@@ -228,6 +241,16 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, header
 				return common.InternalError(err, w, r)
 			}
 		}
+
+		if user.Perms.EditReply && len(attachQueryList) > 0 {
+			amap, err := common.Attachments.BulkMiniGetList("replies", attachQueryList)
+			if err != nil && err != sql.ErrNoRows {
+				return common.InternalError(err, w, r)
+			}
+			for id, attach := range amap {
+				tpage.ItemList[attachMap[id]].Attachments = attach
+			}
+		}
 	}
 
 	rerr := renderTemplate("topic", w, r, header, tpage)

schema/mysql/inserts.sql

@@ -1,3 +1,11 @@
+ALTER TABLE `topics` ADD INDEX `parentID` (`parentID`);;
+ALTER TABLE `replies` ADD INDEX `tid` (`tid`);;
+ALTER TABLE `polls` ADD INDEX `parentID` (`parentID`);;
+ALTER TABLE `likes` ADD INDEX `targetItem` (`targetItem`);;
+ALTER TABLE `emails` ADD INDEX `uid` (`uid`);;
+ALTER TABLE `attachments` ADD INDEX `originID` (`originID`);;
+ALTER TABLE `attachments` ADD INDEX `path` (`path`);;
+ALTER TABLE `activity_stream_matches` ADD INDEX `watcher` (`watcher`);;
 INSERT INTO `sync`(`last_update`) VALUES (UTC_TIMESTAMP());
 INSERT INTO `settings`(`name`,`content`,`type`,`constraints`) VALUES ('activation_type','1','list','1-3');
 INSERT INTO `settings`(`name`,`content`,`type`) VALUES ('bigpost_min_words','250','int');

templates/topic_alt.html

@@ -36,14 +36,7 @@
 	{{if .Poll.ID}}
 	<form id="poll_{{.Poll.ID}}_form" action="/poll/vote/{{.Poll.ID}}?session={{.CurrentUser.Session}}" method="post"></form>
 	<article class="rowitem passive deletable_block editable_parent post_item poll_item top_post hide_on_edit">
-		{{/** TODO: De-dupe userinfo with a common template **/}}
+		{{template "topic_alt_userinfo.html" .Topic }}
-		<div class="userinfo" aria-label="{{lang "topic.userinfo_aria"}}">
-			<div class="avatar_item" style="background-image: url({{.Topic.Avatar}}), url(/static/white-dot.jpg);background-position: 0px -10px;">&nbsp;</div>
-			<div class="user_meta">
-				<a href="{{.Topic.UserLink}}" class="the_name" rel="author">{{.Topic.CreatedByName}}</a>
-				{{if .Topic.Tag}}<div class="tag_block"><div class="tag_pre"></div><div class="post_tag">{{.Topic.Tag}}</div><div class="tag_post"></div></div>{{else}}<div class="tag_block"><div class="tag_pre"></div><div class="post_tag post_level">{{level .Topic.Level}}</div><div class="tag_post"></div></div>{{end}}
-			</div>
-		</div>
 		<div id="poll_voter_{{.Poll.ID}}" class="content_container poll_voter">
 			<div class="topic_content user_content">
 				{{range .Poll.QuickOptions}}
@@ -73,10 +66,10 @@
 			<div class="hide_on_edit topic_content user_content" itemprop="text">{{.Topic.ContentHTML}}</div>
 			{{if .CurrentUser.Loggedin}}{{if .CurrentUser.Perms.EditTopic}}<textarea name="topic_content" class="show_on_edit topic_content_input">{{.Topic.Content}}</textarea>
 
-			{{if .Topic.Attachments}}<div class="show_on_edit attach_edit_bay" tid="{{.Topic.ID}}">
+			{{if .Topic.Attachments}}<div class="show_on_edit attach_edit_bay" type="topic" id="{{.Topic.ID}}">
 				{{range .Topic.Attachments}}
 				<div class="attach_item{{if .Image}} attach_image_holder{{end}}">
-					{{if .Image}}<img src="//{{$.Header.Site.URL}}/attachs/{{.Path}}?sectionID={{.SectionID}}&sectionType=forums" height=24 width=24 />{{end}}
+					{{if .Image}}<img src="//{{$.Header.Site.URL}}/attachs/{{.Path}}?sectionID={{.SectionID}}&amp;sectionType=forums" height=24 width=24 />{{end}}
 					<span class="attach_item_path" aid="{{.ID}}" fullPath="//{{$.Header.Site.URL}}/attachs/{{.Path}}">{{.Path}}</span>
 					<button class="attach_item_select">{{lang "topic.select_button_text"}}</button>
 					<button class="attach_item_copy">{{lang "topic.copy_button_text"}}</button>

templates/topic_alt_posts.html

@@ -5,8 +5,28 @@
 			<span class="action_icon" style="font-size: 18px;padding-right: 5px;" aria-hidden="true">{{.ActionIcon}}</span>
 			<span itemprop="text">{{.ActionType}}</span>
 		{{else}}
-		<div class="edit_source auto_hide">{{.Content}}</div>
 		<div class="editable_block user_content" itemprop="text">{{.ContentHtml}}</div>
+		{{if $.CurrentUser.Loggedin}}{{if $.CurrentUser.Perms.EditReply}}
+		<div class="edit_source auto_hide">{{.Content}}</div>
+
+		{{if .Attachments}}<div class="show_on_edit show_on_block_edit attach_edit_bay" type="reply" id="{{.ID}}">
+				{{range .Attachments}}
+				<div class="attach_item{{if .Image}} attach_image_holder{{end}}">
+					{{if .Image}}<img src="//{{$.Header.Site.URL}}/attachs/{{.Path}}?sectionID={{.SectionID}}&amp;sectionType=forums" height=24 width=24 />{{end}}
+					<span class="attach_item_path" aid="{{.ID}}" fullPath="//{{$.Header.Site.URL}}/attachs/{{.Path}}">{{.Path}}</span>
+					<button class="attach_item_select">{{lang "topic.select_button_text"}}</button>
+					<button class="attach_item_copy">{{lang "topic.copy_button_text"}}</button>
+				</div>
+				{{end}}
+				<div class="attach_item attach_item_buttons">
+					{{if $.CurrentUser.Perms.UploadFiles}}
+					<input name="upload_files" class="upload_files_post" id="upload_files_post_{{.ID}}" multiple type="file" style="display: none;" />
+					<label for="upload_files_post_{{.ID}}" class="formbutton add_file_button">{{lang "topic.upload_button_text"}}</label>{{end}}
+					<button class="attach_item_delete">{{lang "topic.delete_button_text"}}</button>
+				</div>
+		</div>{{end}}
+		{{end}}{{end}}
+
 		<div class="controls button_container{{if .LikeCount}} has_likes{{end}}">
 			<div class="action_button_left">
 			{{if $.CurrentUser.Loggedin}}

templates/topic_posts.html

@@ -8,6 +8,7 @@
 	<article {{scope "post"}} id="post-{{.ID}}" itemscope itemtype="http://schema.org/CreativeWork" class="rowitem passive deletable_block editable_parent post_item {{.ClassName}}" style="background-image: url({{.Avatar}}), url(/static/{{$.Header.Theme.Name}}/post-avatar-bg.jpg);background-position: 0px {{if le .ContentLines 5}}-1{{end}}0px;background-repeat:no-repeat, repeat-y;">
 		{{/** TODO: We might end up with <br>s in the inline editor, fix this **/}}
 		<p class="editable_block user_content" itemprop="text" style="margin:0;padding:0;">{{.ContentHtml}}</p>
+		{{if $.CurrentUser.Loggedin}}{{if $.CurrentUser.Perms.EditReply}}<div class="auto_hide edit_source">{{.Content}}</div>{{end}}{{end}}
 
 		<span class="controls{{if .LikeCount}} has_likes{{end}}">
 

themes/nox/public/main.css

@@ -616,7 +616,7 @@ button, .formbutton, .panel_right_button:not(.has_inner_button) {
 }
 .topic_item .topic_forum {
 	font-size: 19px;
-	line-height: 31px;
+	line-height: 30px;
 	color: #cccccc;
 }
 .topic_view_count {

tickloop.go

@@ -45,7 +45,13 @@ func runHook(name string) {
 	}
 }
 
-func tickLoop(thumbChan chan bool, halfSecondTicker *time.Ticker, secondTicker *time.Ticker, fifteenMinuteTicker *time.Ticker, hourTicker *time.Ticker) {
+func tickLoop(thumbChan chan bool) {
+	// TODO: Write tests for these
+	// Run this goroutine once every half second
+	halfSecondTicker := time.NewTicker(time.Second / 2)
+	secondTicker := time.NewTicker(time.Second)
+	fifteenMinuteTicker := time.NewTicker(15 * time.Minute)
+	hourTicker := time.NewTicker(time.Hour)
 	for {
 		select {
 		case <-halfSecondTicker.C:

tmpl_stub.go

@@ -23,3 +23,12 @@ func StringToBytes(s string) (bytes []byte) {
 	runtime.KeepAlive(&s)
 	return bytes
 }
+
+func BytesToString(bytes []byte) (s string) {
+	slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
+	str := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	str.Data = slice.Data
+	str.Len = slice.Len
+	runtime.KeepAlive(&bytes)
+	return s
+}