Basic search now works for the Search & Filter Widget. ElasticSearch has been temporarily delayed, so I can push through this update.

Added the three month time range to the analytics panes.
Began work on adding new graphs to the analytics panes.
Began work on the ElasticSearch adapter for the search system.
Added the currently limited AddKey method to the database adapters.
Expanded upon the column parsing logic in the database adapters to ease the use of InsertSelects.

Added the BulkGet method to TopicCache.
Added the BulkGetMap method to TopicStore.
TopicStore methods should now properly retrieve lastReplyBy.
Added the panel_analytics_script template to de-dupe part of the analytics logic. We plan to tidy this up further, but for now, it'll suffice.
Added plugin_sendmail and plugin_hyperdrive to the continuous integration test list.
Tweaked the width and heights of the textareas for the Widget Editor.
Added the AddKey method to *qgen.builder
Fixed a bug where using the inline forum editor would crash Gosora and wouldn't set the preset permissions for that forum properly.
Added DotBot to the user agent analytics.
Invisibles should be better handled when they're encountered now in user agent strings.
Unknown language ISO Codes in headers now have the requests fully logged for debugging purposes.
Shortened some of the pointer receiver names.
Shortened some variable names.

Added the dotbot phrase.
Added the panel_statistics_time_range_three_months phrase.

Added gopkg.in/olivere/elastic.v6 as a dependency.

You will need to run the patcher or updater for this commit.

.travis.yml

@@ -13,6 +13,8 @@ before_install:
   - mv ./config/config_example.json ./config/config.json
   - ./update-deps-linux
   - ./dev-update-travis
+  - mv ./experimental/plugin_sendmail.go ..
+  - mv ./experimental/plugin_hyperdrive.go ..
 install: true
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter

cmd/elasticsearch/setup.go

@@ -0,0 +1,206 @@
+// Work in progress
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/Azareal/Gosora/common"
+	"github.com/Azareal/Gosora/query_gen"
+	"gopkg.in/olivere/elastic.v6"
+)
+
+func main() {
+	log.Print("Loading the configuration data")
+	err := common.LoadConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Print("Processing configuration data")
+	err = common.ProcessConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if common.DbConfig.Adapter != "mysql" && common.DbConfig.Adapter != "" {
+		log.Fatal("Only MySQL is supported for upgrades right now, please wait for a newer build of the patcher")
+	}
+
+	err = prepMySQL()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	client, err := elastic.NewClient(elastic.SetErrorLog(log.New(os.Stdout, "ES ", log.LstdFlags)))
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, _, err = client.Ping("http://127.0.0.1:9200").Do(context.Background())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = setupIndices(client)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = setupData(client)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func prepMySQL() error {
+	return qgen.Builder.Init("mysql", map[string]string{
+		"host":      common.DbConfig.Host,
+		"port":      common.DbConfig.Port,
+		"name":      common.DbConfig.Dbname,
+		"username":  common.DbConfig.Username,
+		"password":  common.DbConfig.Password,
+		"collation": "utf8mb4_general_ci",
+	})
+}
+
+type ESIndexBase struct {
+	Mappings ESIndexMappings `json:"mappings"`
+}
+
+type ESIndexMappings struct {
+	Doc ESIndexDoc `json:"_doc"`
+}
+
+type ESIndexDoc struct {
+	Properties map[string]map[string]string `json:"properties"`
+}
+
+type ESDocMap map[string]map[string]string
+
+func (d ESDocMap) Add(column string, cType string) {
+	d["column"] = map[string]string{"type": cType}
+}
+
+func setupIndices(client *elastic.Client) error {
+	exists, err := client.IndexExists("topics").Do(context.Background())
+	if err != nil {
+		return err
+	}
+	if exists {
+		deleteIndex, err := client.DeleteIndex("topics").Do(context.Background())
+		if err != nil {
+			return err
+		}
+		if !deleteIndex.Acknowledged {
+			return errors.New("delete not acknowledged")
+		}
+	}
+
+	docMap := make(ESDocMap)
+	docMap.Add("tid", "integer")
+	docMap.Add("title", "text")
+	docMap.Add("content", "text")
+	docMap.Add("createdBy", "integer")
+	docMap.Add("ip", "ip")
+	docMap.Add("suggest", "completion")
+	indexBase := ESIndexBase{ESIndexMappings{ESIndexDoc{docMap}}}
+	oBytes, err := json.Marshal(indexBase)
+	if err != nil {
+		return err
+	}
+	createIndex, err := client.CreateIndex("topics").Body(string(oBytes)).Do(context.Background())
+	if err != nil {
+		return err
+	}
+	if !createIndex.Acknowledged {
+		return errors.New("not acknowledged")
+	}
+
+	exists, err = client.IndexExists("replies").Do(context.Background())
+	if err != nil {
+		return err
+	}
+	if exists {
+		deleteIndex, err := client.DeleteIndex("replies").Do(context.Background())
+		if err != nil {
+			return err
+		}
+		if !deleteIndex.Acknowledged {
+			return errors.New("delete not acknowledged")
+		}
+	}
+
+	docMap = make(ESDocMap)
+	docMap.Add("rid", "integer")
+	docMap.Add("tid", "integer")
+	docMap.Add("content", "text")
+	docMap.Add("createdBy", "integer")
+	docMap.Add("ip", "ip")
+	docMap.Add("suggest", "completion")
+	indexBase = ESIndexBase{ESIndexMappings{ESIndexDoc{docMap}}}
+	oBytes, err = json.Marshal(indexBase)
+	if err != nil {
+		return err
+	}
+	createIndex, err = client.CreateIndex("replies").Body(string(oBytes)).Do(context.Background())
+	if err != nil {
+		return err
+	}
+	if !createIndex.Acknowledged {
+		return errors.New("not acknowledged")
+	}
+
+	return nil
+}
+
+type ESTopic struct {
+	ID        int    `json:"tid"`
+	Title     string `json:"title"`
+	Content   string `json:"content"`
+	CreatedBy int    `json:"createdBy"`
+	IPAddress string `json:"ip"`
+}
+
+type ESReply struct {
+	ID        int    `json:"rid"`
+	TID       int    `json:"tid"`
+	Content   string `json:"content"`
+	CreatedBy int    `json:"createdBy"`
+	IPAddress string `json:"ip"`
+}
+
+func setupData(client *elastic.Client) error {
+	err := qgen.NewAcc().Select("topics").Cols("tid, title, content, createdBy, ipaddress").Each(func(rows *sql.Rows) error {
+		var tid, createdBy int
+		var title, content, ip string
+		err := rows.Scan(&tid, &title, &content, &createdBy, &ip)
+		if err != nil {
+			return err
+		}
+
+		topic := ESTopic{tid, title, content, createdBy, ip}
+		_, err = client.Index().Index("topics").Type("_doc").Id(strconv.Itoa(tid)).BodyJson(topic).Do(context.Background())
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	return qgen.NewAcc().Select("replies").Cols("rid, tid, content, createdBy, ipaddress").Each(func(rows *sql.Rows) error {
+		var rid, tid, createdBy int
+		var content, ip string
+		err := rows.Scan(&rid, &tid, &content, &createdBy, &ip)
+		if err != nil {
+			return err
+		}
+
+		reply := ESReply{rid, tid, content, createdBy, ip}
+		_, err = client.Index().Index("replies").Type("_doc").Id(strconv.Itoa(rid)).BodyJson(reply).Do(context.Background())
+		return err
+	})
+}

cmd/query_gen/tables.go

@@ -242,6 +242,7 @@ func createTables(adapter qgen.Adapter) error {
 		},
 		[]tblKey{
 			tblKey{"tid", "primary"},
+			tblKey{"content", "fulltext"},
 		},
 	)
 
@@ -265,6 +266,7 @@ func createTables(adapter qgen.Adapter) error {
 		},
 		[]tblKey{
 			tblKey{"rid", "primary"},
+			tblKey{"content", "fulltext"},
 		},
 	)
 

common/common.go

@@ -8,7 +8,9 @@ package common // import "github.com/Azareal/Gosora/common"
 
 import (
 	"database/sql"
+	"io"
 	"log"
+	"os"
 	"sync/atomic"
 	"time"
 
@@ -112,6 +114,8 @@ func StoppedServer(msg ...interface{}) {
 
 var StopServerChan = make(chan []interface{})
 
+var LogWriter = io.MultiWriter(os.Stderr)
+
 func DebugDetail(args ...interface{}) {
 	if Dev.SuperDebug {
 		log.Print(args...)

common/counters/langs.go

@@ -149,19 +149,23 @@ func (counter *DefaultLangViewCounter) insertChunk(count int, id int) error {
 	return err
 }
 
-func (counter *DefaultLangViewCounter) Bump(langCode string) {
+func (counter *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
+	validCode = true
 	id, ok := counter.codesToIndices[langCode]
 	if !ok {
 		// TODO: Tell the caller that the code's invalid
 		id = 0 // Unknown
+		validCode = false
 	}
 
 	// TODO: Test this check
 	common.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id])
 	if len(counter.buckets) <= id || id < 0 {
-		return
+		return validCode
 	}
 	counter.buckets[id].Lock()
 	counter.buckets[id].counter++
 	counter.buckets[id].Unlock()
+
+	return validCode
 }

common/forum.go

@@ -82,7 +82,7 @@ func (forum *Forum) Update(name string, desc string, active bool, preset string)
 	if err != nil {
 		return err
 	}
-	if forum.Preset != preset || preset == "custom" || preset == "" {
+	if forum.Preset != preset && preset != "custom" && preset != "" {
 		err = PermmapToQuery(PresetToPermmap(preset), forum.ID)
 		if err != nil {
 			return err

common/forum_perms.go

@@ -99,7 +99,6 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error {
 	if err != nil {
 		return err
 	}
-
 	_, err = deleteForumPermsByForumTx.Exec(fid)
 	if err != nil {
 		return err
@@ -112,13 +111,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error {
 
 	addForumPermsToForumAdminsTx, err := qgen.Builder.SimpleInsertSelectTx(tx,
 		qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""},
-		qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 1", "", ""},
+		qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 1", "", ""},
 	)
 	if err != nil {
 		return err
 	}
-
+	_, err = addForumPermsToForumAdminsTx.Exec(fid, perms)
-	_, err = addForumPermsToForumAdminsTx.Exec(fid, "", perms)
 	if err != nil {
 		return err
 	}
@@ -130,12 +128,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error {
 
 	addForumPermsToForumStaffTx, err := qgen.Builder.SimpleInsertSelectTx(tx,
 		qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""},
-		qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 0 AND is_mod = 1", "", ""},
+		qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 0 AND is_mod = 1", "", ""},
 	)
 	if err != nil {
 		return err
 	}
-	_, err = addForumPermsToForumStaffTx.Exec(fid, "", perms)
+	_, err = addForumPermsToForumStaffTx.Exec(fid, perms)
 	if err != nil {
 		return err
 	}
@@ -147,12 +145,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error {
 
 	addForumPermsToForumMembersTx, err := qgen.Builder.SimpleInsertSelectTx(tx,
 		qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""},
-		qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 0 AND is_mod = 0 AND is_banned = 0", "", ""},
+		qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 0 AND is_mod = 0 AND is_banned = 0", "", ""},
 	)
 	if err != nil {
 		return err
 	}
-	_, err = addForumPermsToForumMembersTx.Exec(fid, "", perms)
+	_, err = addForumPermsToForumMembersTx.Exec(fid, perms)
 	if err != nil {
 		return err
 	}

common/null_topic_cache.go

@@ -10,34 +10,37 @@ func NewNullTopicCache() *NullTopicCache {
 }
 
 // nolint
-func (mts *NullTopicCache) Get(id int) (*Topic, error) {
+func (c *NullTopicCache) Get(id int) (*Topic, error) {
 	return nil, ErrNoRows
 }
-func (mts *NullTopicCache) GetUnsafe(id int) (*Topic, error) {
+func (c *NullTopicCache) GetUnsafe(id int) (*Topic, error) {
 	return nil, ErrNoRows
 }
-func (mts *NullTopicCache) Set(_ *Topic) error {
+func (c *NullTopicCache) BulkGet(ids []int) (list []*Topic) {
+	return make([]*Topic, len(ids))
+}
+func (c *NullTopicCache) Set(_ *Topic) error {
 	return nil
 }
-func (mts *NullTopicCache) Add(_ *Topic) error {
+func (c *NullTopicCache) Add(_ *Topic) error {
 	return nil
 }
-func (mts *NullTopicCache) AddUnsafe(_ *Topic) error {
+func (c *NullTopicCache) AddUnsafe(_ *Topic) error {
 	return nil
 }
-func (mts *NullTopicCache) Remove(id int) error {
+func (c *NullTopicCache) Remove(id int) error {
 	return nil
 }
-func (mts *NullTopicCache) RemoveUnsafe(id int) error {
+func (c *NullTopicCache) RemoveUnsafe(id int) error {
 	return nil
 }
-func (mts *NullTopicCache) Flush() {
+func (c *NullTopicCache) Flush() {
 }
-func (mts *NullTopicCache) Length() int {
+func (c *NullTopicCache) Length() int {
 	return 0
 }
-func (mts *NullTopicCache) SetCapacity(_ int) {
+func (c *NullTopicCache) SetCapacity(_ int) {
 }
-func (mts *NullTopicCache) GetCapacity() int {
+func (c *NullTopicCache) GetCapacity() int {
 	return 0
 }

common/null_user_cache.go

@@ -10,41 +10,41 @@ func NewNullUserCache() *NullUserCache {
 }
 
 // nolint
-func (mus *NullUserCache) DeallocOverflow(evictPriority bool) (evicted int) {
+func (c *NullUserCache) DeallocOverflow(evictPriority bool) (evicted int) {
 	return 0
 }
-func (mus *NullUserCache) Get(id int) (*User, error) {
+func (c *NullUserCache) Get(id int) (*User, error) {
 	return nil, ErrNoRows
 }
-func (mus *NullUserCache) BulkGet(ids []int) (list []*User) {
+func (c *NullUserCache) BulkGet(ids []int) (list []*User) {
 	return make([]*User, len(ids))
 }
-func (mus *NullUserCache) GetUnsafe(id int) (*User, error) {
+func (c *NullUserCache) GetUnsafe(id int) (*User, error) {
 	return nil, ErrNoRows
 }
-func (mus *NullUserCache) Set(_ *User) error {
+func (c *NullUserCache) Set(_ *User) error {
 	return nil
 }
-func (mus *NullUserCache) Add(_ *User) error {
+func (c *NullUserCache) Add(_ *User) error {
 	return nil
 }
-func (mus *NullUserCache) AddUnsafe(_ *User) error {
+func (c *NullUserCache) AddUnsafe(_ *User) error {
 	return nil
 }
-func (mus *NullUserCache) Remove(id int) error {
+func (c *NullUserCache) Remove(id int) error {
 	return nil
 }
-func (mus *NullUserCache) RemoveUnsafe(id int) error {
+func (c *NullUserCache) RemoveUnsafe(id int) error {
 	return nil
 }
-func (mus *NullUserCache) BulkRemove(ids []int) {}
+func (c *NullUserCache) BulkRemove(ids []int) {}
-func (mus *NullUserCache) Flush() {
+func (c *NullUserCache) Flush() {
 }
-func (mus *NullUserCache) Length() int {
+func (c *NullUserCache) Length() int {
 	return 0
 }
-func (mus *NullUserCache) SetCapacity(_ int) {
+func (c *NullUserCache) SetCapacity(_ int) {
 }
-func (mus *NullUserCache) GetCapacity() int {
+func (c *NullUserCache) GetCapacity() int {
 	return 0
 }

common/pages.go

@@ -32,10 +32,12 @@ type Header struct {
 	ZoneData    interface{}
 	Path        string
 	MetaDesc    string
-	StartedAt   time.Time
+	//OGImage string
-	Elapsed1    string
+	//OGDesc string
-	Writer      http.ResponseWriter
+	StartedAt time.Time
-	ExtData     ExtData
+	Elapsed1  string
+	Writer    http.ResponseWriter
+	ExtData   ExtData
 }
 
 func (header *Header) AddScript(name string) {
@@ -255,9 +257,14 @@ type PanelCustomPageEditPage struct {
 	Page *CustomPage
 }
 
-type PanelTimeGraph struct {
+/*type PanelTimeGraph struct {
 	Series []int64 // The counts on the left
 	Labels []int64 // unixtimes for the bottom, gets converted into 1:00, 2:00, etc. with JS
+}*/
+type PanelTimeGraph struct {
+	Series  [][]int64 // The counts on the left
+	Labels  []int64   // unixtimes for the bottom, gets converted into 1:00, 2:00, etc. with JS
+	Legends []string
 }
 
 type PanelAnalyticsItem struct {
@@ -267,9 +274,9 @@ type PanelAnalyticsItem struct {
 
 type PanelAnalyticsPage struct {
 	*BasePanelPage
-	PrimaryGraph PanelTimeGraph
+	Graph     PanelTimeGraph
-	ViewItems    []PanelAnalyticsItem
+	ViewItems []PanelAnalyticsItem
-	TimeRange    string
+	TimeRange string
 }
 
 type PanelAnalyticsRoutesItem struct {
@@ -297,20 +304,27 @@ type PanelAnalyticsAgentsPage struct {
 
 type PanelAnalyticsRoutePage struct {
 	*BasePanelPage
-	Route        string
+	Route     string
-	PrimaryGraph PanelTimeGraph
+	Graph     PanelTimeGraph
-	ViewItems    []PanelAnalyticsItem
+	ViewItems []PanelAnalyticsItem
-	TimeRange    string
+	TimeRange string
 }
 
 type PanelAnalyticsAgentPage struct {
 	*BasePanelPage
 	Agent         string
 	FriendlyAgent string
-	PrimaryGraph  PanelTimeGraph
+	Graph         PanelTimeGraph
 	TimeRange     string
 }
 
+type PanelAnalyticsDuoPage struct {
+	*BasePanelPage
+	ItemList  []PanelAnalyticsAgentsItem
+	Graph     PanelTimeGraph
+	TimeRange string
+}
+
 type PanelThemesPage struct {
 	*BasePanelPage
 	PrimaryThemes []*Theme

common/search.go

@@ -3,70 +3,131 @@ package common
 import (
 	"database/sql"
 	"errors"
+	"strconv"
 
 	"github.com/Azareal/Gosora/query_gen"
 )
 
-//var RepliesSearch Searcher
+var RepliesSearch Searcher
 
 type Searcher interface {
-	Query(q string) ([]int, error)
+	Query(q string, zones []int) ([]int, error)
-}
-
-type ZoneSearcher interface {
-	QueryZone(q string, zoneID int) ([]int, error)
 }
 
 // TODO: Implement this
 // Note: This is slow compared to something like ElasticSearch and very limited
 type SQLSearcher struct {
-	queryReplies     *sql.Stmt
+	queryReplies *sql.Stmt
-	queryTopics      *sql.Stmt
+	queryTopics  *sql.Stmt
-	queryZoneReplies *sql.Stmt
+	queryZone    *sql.Stmt
-	queryZoneTopics  *sql.Stmt
 }
 
 // TODO: Support things other than MySQL
+// TODO: Use LIMIT?
 func NewSQLSearcher(acc *qgen.Accumulator) (*SQLSearcher, error) {
 	if acc.GetAdapter().GetName() != "mysql" {
 		return nil, errors.New("SQLSearcher only supports MySQL at this time")
 	}
 	return &SQLSearcher{
-		queryReplies:     acc.RawPrepare("SELECT `rid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
+		queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
-		queryTopics:      acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title,content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
+		queryTopics:  acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
-		queryZoneReplies: acc.RawPrepare("SELECT `rid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE) AND `parentID` = ?;"),
+		queryZone:    acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` = ?;"),
-		queryZoneTopics:  acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title,content) AGAINST (? IN NATURAL LANGUAGE MODE) AND `parentID` = ?;"),
 	}, acc.FirstError()
 }
 
-func (searcher *SQLSearcher) Query(q string) ([]int, error) {
+func (search *SQLSearcher) queryAll(q string) ([]int, error) {
-	return nil, nil
+	var ids []int
+	var run = func(stmt *sql.Stmt, q ...interface{}) error {
+		rows, err := stmt.Query(q...)
+		if err == sql.ErrNoRows {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		defer rows.Close()
 
-	/*
+		for rows.Next() {
-		rows, err := stmt.Query(q)
+			var id int
+			err := rows.Scan(&id)
+			if err != nil {
+				return err
+			}
+			ids = append(ids, id)
+		}
+		return rows.Err()
+	}
+
+	err := run(search.queryReplies, q)
+	if err != nil {
+		return nil, err
+	}
+	err = run(search.queryTopics, q, q)
+	if err != nil {
+		return nil, err
+	}
+	if len(ids) == 0 {
+		err = sql.ErrNoRows
+	}
+	return ids, err
+}
+
+func (search *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
+	if len(zones) == 0 {
+		return nil, nil
+	}
+	var run = func(rows *sql.Rows, err error) error {
+		if err == sql.ErrNoRows {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		defer rows.Close()
+
+		for rows.Next() {
+			var id int
+			err := rows.Scan(&id)
+			if err != nil {
+				return err
+			}
+			ids = append(ids, id)
+		}
+		return rows.Err()
+	}
+
+	if len(zones) == 1 {
+		err = run(search.queryZone.Query(q, q, q, zones[0]))
+	} else {
+		var zList string
+		for _, zone := range zones {
+			zList += strconv.Itoa(zone) + ","
+		}
+		zList = zList[:len(zList)-1]
+
+		acc := qgen.NewAcc()
+		stmt := acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` IN(" + zList + ");")
+		err := acc.FirstError()
 		if err != nil {
 			return nil, err
 		}
-		defer rows.Close()
+		err = run(stmt.Query(q, q, q))
-	*/
+	}
-}
+	if err != nil {
-
+		return nil, err
-func (searcher *SQLSearcher) QueryZone(q string, zoneID int) ([]int, error) {
+	}
-	return nil, nil
+	if len(ids) == 0 {
+		err = sql.ErrNoRows
+	}
+	return ids, err
 }
 
 // TODO: Implement this
 type ElasticSearchSearcher struct {
 }
 
-func NewElasticSearchSearcher() *ElasticSearchSearcher {
+func NewElasticSearchSearcher() (*ElasticSearchSearcher, error) {
-	return &ElasticSearchSearcher{}
+	return &ElasticSearchSearcher{}, nil
 }
 
-func (searcher *ElasticSearchSearcher) Query(q string) ([]int, error) {
+func (search *ElasticSearchSearcher) Query(q string, zones []int) ([]int, error) {
-	return nil, nil
-}
-
-func (searcher *ElasticSearchSearcher) QueryZone(q string, zoneID int) ([]int, error) {
 	return nil, nil
 }

common/template_init.go

@@ -359,12 +359,9 @@ func compileTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName string
 			writeTemplate(name, tmpl)
 		}
 	}
-	/*writeTemplate("profile", profileTmpl)
+	/*writeTemplate("login", loginTmpl)
-	writeTemplate("forums", forumsTmpl)
-	writeTemplate("login", loginTmpl)
 	writeTemplate("register", registerTmpl)
 	writeTemplate("ip_search", ipSearchTmpl)
-	writeTemplate("account", accountTmpl)
 	writeTemplate("error", errorTmpl)*/
 	return nil
 }

common/topic.go

@@ -146,6 +146,28 @@ func (row *TopicsRow) WebSockets() *WsTopicsRow {
 	return &WsTopicsRow{row.ID, row.Link, row.Title, row.CreatedBy, row.IsClosed, row.Sticky, row.CreatedAt, row.LastReplyAt, RelativeTime(row.LastReplyAt), row.LastReplyBy, row.LastReplyID, row.ParentID, row.ViewCount, row.PostCount, row.LikeCount, row.AttachCount, row.ClassName, row.Creator.WebSockets(), row.LastUser.WebSockets(), row.ForumName, row.ForumLink}
 }
 
+// TODO: Stop relying on so many struct types?
+// ! Not quite safe as Topic doesn't contain all the data needed to constructs a TopicsRow
+func (t *Topic) TopicsRow() *TopicsRow {
+	lastPage := 1
+	var creator *User = nil
+	contentLines := 1
+	var lastUser *User = nil
+	forumName := ""
+	forumLink := ""
+
+	return &TopicsRow{t.ID, t.Link, t.Title, t.Content, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, t.LastReplyBy, t.LastReplyID, t.ParentID, t.Status, t.IPAddress, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, lastPage, t.ClassName, t.Data, creator, "", contentLines, lastUser, forumName, forumLink}
+}
+
+// ! Not quite safe as Topic doesn't contain all the data needed to constructs a WsTopicsRow
+/*func (t *Topic) WsTopicsRows() *WsTopicsRow {
+	var creator *User = nil
+	var lastUser *User = nil
+	forumName := ""
+	forumLink := ""
+	return &WsTopicsRow{t.ID, t.Link, t.Title, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, RelativeTime(t.LastReplyAt), t.LastReplyBy, t.LastReplyID, t.ParentID, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, t.ClassName, creator, lastUser, forumName, forumLink}
+}*/
+
 type TopicStmts struct {
 	addReplies      *sql.Stmt
 	updateLastReply *sql.Stmt

common/topic_cache.go

@@ -9,6 +9,7 @@ import (
 type TopicCache interface {
 	Get(id int) (*Topic, error)
 	GetUnsafe(id int) (*Topic, error)
+	BulkGet(ids []int) (list []*Topic)
 	Set(item *Topic) error
 	Add(item *Topic) error
 	AddUnsafe(item *Topic) error
@@ -57,6 +58,17 @@ func (mts *MemoryTopicCache) GetUnsafe(id int) (*Topic, error) {
 	return item, ErrNoRows
 }
 
+// BulkGet fetches multiple topics by their IDs. Indices without topics will be set to nil, so make sure you check for those, we might want to change this behaviour to make it less confusing.
+func (c *MemoryTopicCache) BulkGet(ids []int) (list []*Topic) {
+	list = make([]*Topic, len(ids))
+	c.RLock()
+	for i, id := range ids {
+		list[i] = c.items[id]
+	}
+	c.RUnlock()
+	return list
+}
+
 // Set overwrites the value of a topic in the cache, whether it's present or not. May return a capacity overflow error.
 func (mts *MemoryTopicCache) Set(item *Topic) error {
 	mts.Lock()

common/topic_list.go

@@ -143,6 +143,7 @@ func (tList *DefaultTopicList) GetListByGroup(group *Group, page int, orderby st
 }
 
 func (tList *DefaultTopicList) GetListByCanSee(canSee []int, page int, orderby string, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, paginator Paginator, err error) {
+	// TODO: Optimise this by filtering canSee and then fetching the forums?
 	// We need a list of the visible forums for Quick Topic
 	// ? - Would it be useful, if we could post in social groups from /topics/?
 	for _, fid := range canSee {
@@ -269,27 +270,27 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter
 	var reqUserList = make(map[int]bool)
 	for rows.Next() {
 		// TODO: Embed Topic structs in TopicsRow to make it easier for us to reuse this work in the topic cache
-		topicItem := TopicsRow{ID: 0}
+		topic := TopicsRow{ID: 0}
-		err := rows.Scan(&topicItem.ID, &topicItem.Title, &topicItem.Content, &topicItem.CreatedBy, &topicItem.IsClosed, &topicItem.Sticky, &topicItem.CreatedAt, &topicItem.LastReplyAt, &topicItem.LastReplyBy, &topicItem.LastReplyID, &topicItem.ParentID, &topicItem.ViewCount, &topicItem.PostCount, &topicItem.LikeCount)
+		err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.IsClosed, &topic.Sticky, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyBy, &topic.LastReplyID, &topic.ParentID, &topic.ViewCount, &topic.PostCount, &topic.LikeCount)
 		if err != nil {
 			return nil, Paginator{nil, 1, 1}, err
 		}
 
-		topicItem.Link = BuildTopicURL(NameToSlug(topicItem.Title), topicItem.ID)
+		topic.Link = BuildTopicURL(NameToSlug(topic.Title), topic.ID)
 		// TODO: Pass forum to something like topicItem.Forum and use that instead of these two properties? Could be more flexible.
-		forum := Forums.DirtyGet(topicItem.ParentID)
+		forum := Forums.DirtyGet(topic.ParentID)
-		topicItem.ForumName = forum.Name
+		topic.ForumName = forum.Name
-		topicItem.ForumLink = forum.Link
+		topic.ForumLink = forum.Link
 
 		// TODO: Create a specialised function with a bit less overhead for getting the last page for a post count
-		_, _, lastPage := PageOffset(topicItem.PostCount, 1, Config.ItemsPerPage)
+		_, _, lastPage := PageOffset(topic.PostCount, 1, Config.ItemsPerPage)
-		topicItem.LastPage = lastPage
+		topic.LastPage = lastPage
 
 		// TODO: Rename this Vhook to better reflect moving the topic list from /routes/ to /common/
-		GetHookTable().Vhook("topics_topic_row_assign", &topicItem, &forum)
+		GetHookTable().Vhook("topics_topic_row_assign", &topic, &forum)
-		topicList = append(topicList, &topicItem)
+		topicList = append(topicList, &topic)
-		reqUserList[topicItem.CreatedBy] = true
+		reqUserList[topic.CreatedBy] = true
-		reqUserList[topicItem.LastReplyBy] = true
+		reqUserList[topic.LastReplyBy] = true
 	}
 	err = rows.Err()
 	if err != nil {
@@ -312,9 +313,9 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter
 
 	// Second pass to the add the user data
 	// TODO: Use a pointer to TopicsRow instead of TopicsRow itself?
-	for _, topicItem := range topicList {
+	for _, topic := range topicList {
-		topicItem.Creator = userList[topicItem.CreatedBy]
+		topic.Creator = userList[topic.CreatedBy]
-		topicItem.LastUser = userList[topicItem.LastReplyBy]
+		topic.LastUser = userList[topic.LastReplyBy]
 	}
 
 	pageList := Paginate(topicCount, Config.ItemsPerPage, 5)

common/topic_store.go

@@ -9,6 +9,7 @@ package common
 import (
 	"database/sql"
 	"errors"
+	"strconv"
 	"strings"
 
 	"github.com/Azareal/Gosora/query_gen"
@@ -27,6 +28,7 @@ type TopicStore interface {
 	DirtyGet(id int) *Topic
 	Get(id int) (*Topic, error)
 	BypassGet(id int) (*Topic, error)
+	BulkGetMap(ids []int) (list map[int]*Topic, err error)
 	Exists(id int) bool
 	Create(fid int, topicName string, content string, uid int, ipaddress string) (tid int, err error)
 	AddLastTopic(item *Topic, fid int) error // unimplemented
@@ -57,7 +59,7 @@ func NewDefaultTopicStore(cache TopicCache) (*DefaultTopicStore, error) {
 	}
 	return &DefaultTopicStore{
 		cache:      cache,
-		get:        acc.Select("topics").Columns("title, content, createdBy, createdAt, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid = ?").Prepare(),
+		get:        acc.Select("topics").Columns("title, content, createdBy, createdAt, lastReplyBy, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid = ?").Prepare(),
 		exists:     acc.Select("topics").Columns("tid").Where("tid = ?").Prepare(),
 		topicCount: acc.Count("topics").Prepare(),
 		create:     acc.Insert("topics").Columns("parentID, title, content, parsed_content, createdAt, lastReplyAt, lastReplyBy, ipaddress, words, createdBy").Fields("?,?,?,?,UTC_TIMESTAMP(),UTC_TIMESTAMP(),?,?,?,?").Prepare(),
@@ -71,7 +73,7 @@ func (mts *DefaultTopicStore) DirtyGet(id int) *Topic {
 	}
 
 	topic = &Topic{ID: id}
-	err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
+	err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
 	if err == nil {
 		topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
 		_ = mts.cache.Add(topic)
@@ -88,7 +90,7 @@ func (mts *DefaultTopicStore) Get(id int) (topic *Topic, err error) {
 	}
 
 	topic = &Topic{ID: id}
-	err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
+	err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
 	if err == nil {
 		topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
 		_ = mts.cache.Add(topic)
@@ -99,14 +101,89 @@ func (mts *DefaultTopicStore) Get(id int) (topic *Topic, err error) {
 // BypassGet will always bypass the cache and pull the topic directly from the database
 func (mts *DefaultTopicStore) BypassGet(id int) (*Topic, error) {
 	topic := &Topic{ID: id}
-	err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
+	err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
 	topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
 	return topic, err
 }
 
+// TODO: Avoid duplicating much of this logic from user_store.go
+func (s *DefaultTopicStore) BulkGetMap(ids []int) (list map[int]*Topic, err error) {
+	var idCount = len(ids)
+	list = make(map[int]*Topic)
+	if idCount == 0 {
+		return list, nil
+	}
+
+	var stillHere []int
+	sliceList := s.cache.BulkGet(ids)
+	if len(sliceList) > 0 {
+		for i, sliceItem := range sliceList {
+			if sliceItem != nil {
+				list[sliceItem.ID] = sliceItem
+			} else {
+				stillHere = append(stillHere, ids[i])
+			}
+		}
+		ids = stillHere
+	}
+
+	// If every user is in the cache, then return immediately
+	if len(ids) == 0 {
+		return list, nil
+	} else if len(ids) == 1 {
+		topic, err := s.Get(ids[0])
+		if err != nil {
+			return list, err
+		}
+		list[topic.ID] = topic
+		return list, nil
+	}
+
+	// TODO: Add a function for the qlist stuff
+	var qlist string
+	var idList []interface{}
+	for _, id := range ids {
+		idList = append(idList, strconv.Itoa(id))
+		qlist += "?,"
+	}
+	qlist = qlist[0 : len(qlist)-1]
+
+	rows, err := qgen.NewAcc().Select("topics").Columns("tid, title, content, createdBy, createdAt, lastReplyBy, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid IN(" + qlist + ")").Query(idList...)
+	if err != nil {
+		return list, err
+	}
+	for rows.Next() {
+		topic := &Topic{}
+		err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
+		if err != nil {
+			return list, err
+		}
+		topic.Link = BuildTopicURL(NameToSlug(topic.Title), topic.ID)
+		s.cache.Set(topic)
+		list[topic.ID] = topic
+	}
+
+	// Did we miss any topics?
+	if idCount > len(list) {
+		var sidList string
+		for _, id := range ids {
+			_, ok := list[id]
+			if !ok {
+				sidList += strconv.Itoa(id) + ","
+			}
+		}
+		if sidList != "" {
+			sidList = sidList[0 : len(sidList)-1]
+			err = errors.New("Unable to find topics with the following IDs: " + sidList)
+		}
+	}
+
+	return list, err
+}
+
 func (mts *DefaultTopicStore) Reload(id int) error {
 	topic := &Topic{ID: id}
-	err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
+	err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data)
 	if err == nil {
 		topic.Link = BuildTopicURL(NameToSlug(topic.Title), id)
 		_ = mts.cache.Set(topic)

common/user_store.go

@@ -132,7 +132,6 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User
 		if err != nil {
 			return nil, err
 		}
-
 		user.Init()
 		store.cache.Set(user)
 		users = append(users, user)
@@ -176,25 +175,23 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
 
 	// TODO: Add a function for the qlist stuff
 	var qlist string
-	var uidList []interface{}
+	var idList []interface{}
 	for _, id := range ids {
-		uidList = append(uidList, strconv.Itoa(id))
+		idList = append(idList, strconv.Itoa(id))
 		qlist += "?,"
 	}
 	qlist = qlist[0 : len(qlist)-1]
 
-	rows, err := qgen.NewAcc().Select("users").Columns("uid, name, group, active, is_super_admin, session, email, avatar, message, url_prefix, url_name, level, score, liked, last_ip, temp_group").Where("uid IN(" + qlist + ")").Query(uidList...)
+	rows, err := qgen.NewAcc().Select("users").Columns("uid, name, group, active, is_super_admin, session, email, avatar, message, url_prefix, url_name, level, score, liked, last_ip, temp_group").Where("uid IN(" + qlist + ")").Query(idList...)
 	if err != nil {
 		return list, err
 	}
-
 	for rows.Next() {
 		user := &User{Loggedin: true}
 		err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 		if err != nil {
 			return list, err
 		}
-
 		user.Init()
 		mus.cache.Set(user)
 		list[user.ID] = user
@@ -211,7 +208,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
 		}
 		if sidList != "" {
 			sidList = sidList[0 : len(sidList)-1]
-			err = errors.New("Unable to find the users with the following IDs: " + sidList)
+			err = errors.New("Unable to find users with the following IDs: " + sidList)
 		}
 	}
 
@@ -233,7 +230,6 @@ func (mus *DefaultUserStore) Reload(id int) error {
 		mus.cache.Remove(id)
 		return err
 	}
-
 	user.Init()
 	_ = mus.cache.Set(user)
 	TopicListThaw.Thaw()
@@ -276,7 +272,6 @@ func (mus *DefaultUserStore) Create(username string, password string, email stri
 	if err != nil {
 		return 0, err
 	}
-
 	lastID, err := res.LastInsertId()
 	return int(lastID), err
 }

docs/updating.md

@@ -39,7 +39,7 @@ If this is the first time you've done an update as the `gosora` user, then you m
 
 Replace that name and email with whatever you like. This name and email only applies to the `gosora` user. If you see a zillion modified files pop-up, then that is due to you changing their permissions, don't worry about it.
 
-If you get an access denied error, then you might need to run `chown -R gosora /home/gosora` and `chgrp -R www-data /home/gosora` to fix the ownership of the files.
+If you get an access denied error, then you might need to run `chown -R gosora /home/gosora` and `chgrp -R www-data /home/gosora` (with the corresponding user you setup for your instance) to fix the ownership of the files.
 
 If you want to manually patch Gosora rather than relying on the above scripts to do it, you'll first want to save your changes with `git stash`, and then, you'll overwrite the files with the new ones with `git pull origin master`, and then, you'll re-apply your changes with `git stash apply`.
 

gen_router.go

@@ -487,7 +487,8 @@ var agentMapEnum = map[string]int{
 	"malformed": 26,
 	"suspicious": 27,
 	"semrush": 28,
-	"zgrab": 29,
+	"dotbot": 29,
+	"zgrab": 30,
 }
 var reverseAgentMapEnum = map[int]string{ 
 	0: "unknown",
@@ -519,7 +520,8 @@ var reverseAgentMapEnum = map[int]string{
 	26: "malformed",
 	27: "suspicious",
 	28: "semrush",
-	29: "zgrab",
+	29: "dotbot",
+	30: "zgrab",
 }
 var markToAgent = map[string]string{ 
 	"OPR": "opera",
@@ -546,6 +548,7 @@ var markToAgent = map[string]string{
 	"Twitterbot": "twitter",
 	"Discourse": "discourse",
 	"SemrushBot": "semrush",
+	"DotBot": "dotbot",
 	"zgrab": "zgrab",
 }
 /*var agentRank = map[string]int{
@@ -641,7 +644,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
 	var heads string
 	for key, value := range req.Header {
 		for _, vvalue := range value {
-			heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!!\n"
+			heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!\n"
 		}
 	}
 
@@ -791,7 +794,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			} else {
 				// TODO: Test this
 				items = items[:0]
-				r.SuspiciousRequest(req,"Illegal char in UA")
+				r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA")
 				r.requestLogger.Print("UA Buffer: ", buffer)
 				r.requestLogger.Print("UA Buffer String: ", string(buffer))
 				break
@@ -815,7 +818,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("parsed agent: ", agent)
 		}
-
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("os: ", os)
 			r.requestLogger.Printf("items: %+v\n",items)
@@ -861,7 +863,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		lang = strings.TrimSpace(lang)
 		lLang := strings.Split(lang,"-")
 		common.DebugDetail("lLang:", lLang)
-		counters.LangViewCounter.Bump(lLang[0])
+		validCode := counters.LangViewCounter.Bump(lLang[0])
+		if !validCode {
+			r.DumpRequest(req,"Invalid ISO Code")
+		}
 	} else {
 		counters.LangViewCounter.Bump("none")
 	}

go.mod

@@ -5,18 +5,21 @@ require (
 	github.com/Azareal/gopsutil v0.0.0-20170716174751-0763ca4e911d
 	github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
 	github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f
+	github.com/fortytw2/leaktest v1.3.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-sql-driver/mysql v1.4.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/lib/pq v1.0.0
 	github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329
+	github.com/olivere/elastic v6.2.16+incompatible // indirect
 	github.com/oschwald/geoip2-golang v1.2.1
 	github.com/oschwald/maxminddb-golang v1.3.0 // indirect
 	github.com/pkg/errors v0.8.0
 	github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d
 	golang.org/x/crypto v0.0.0-20181025213731-e84da0312774
 	google.golang.org/appengine v1.2.0 // indirect
+	gopkg.in/olivere/elastic.v6 v6.2.16
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/src-d/go-git.v4 v4.7.1
 )

go.sum

@@ -16,6 +16,8 @@ github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo=
 github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
@@ -45,6 +47,8 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/olivere/elastic v6.2.16+incompatible h1:+mQIHbkADkOgq9tFqnbyg7uNFVV6swGU07EoK1u0nEQ=
+github.com/olivere/elastic v6.2.16+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
 github.com/oschwald/geoip2-golang v1.2.1 h1:3iz+jmeJc6fuCyWeKgtXSXu7+zvkxJbHFXkMT5FVebU=
 github.com/oschwald/geoip2-golang v1.2.1/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
 github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE=
@@ -79,6 +83,8 @@ google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH2
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/olivere/elastic.v6 v6.2.16 h1:SvZm4VE4auXSIWpuG2630o+NA1hcIFFzzcHFQpCsv/w=
+gopkg.in/olivere/elastic.v6 v6.2.16/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4=
 gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
 gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=

langs/english.json

@@ -191,6 +191,7 @@
 		"lynx":"Lynx",
 		
 		"semrush":"SemrushBot",
+		"dotbot":"DotBot",
 		"zgrab":"Zgrab Application Scanner",
 		"suspicious":"Suspicious",
 		"unknown":"Unknown",
@@ -829,6 +830,7 @@
 		"panel_statistics_topic_counts_head":"Topic Counts",
 		"panel_statistics_requests_head":"Requests",
 
+		"panel_statistics_time_range_three_months":"3 months",
 		"panel_statistics_time_range_one_month":"1 month",
 		"panel_statistics_time_range_one_week":"1 week",
 		"panel_statistics_time_range_two_days":"2 days",

main.go

@@ -1,7 +1,7 @@
 /*
 *
 *	Gosora Main File
-*	Copyright Azareal 2016 - 2019
+*	Copyright Azareal 2016 - 2020
 *
  */
 // Package main contains the main initialisation logic for Gosora
@@ -34,7 +34,6 @@ import (
 )
 
 var router *GenRouter
-var logWriter = io.MultiWriter(os.Stderr)
 
 // TODO: Wrap the globals in here so we can pass pointers to them to subpackages
 var globs *Globs
@@ -144,6 +143,10 @@ func afterDBInit() (err error) {
 	if err != nil {
 		return errors.WithStack(err)
 	}
+	common.RepliesSearch, err = common.NewSQLSearcher(acc)
+	if err != nil {
+		return errors.WithStack(err)
+	}
 	common.Subscriptions, err = common.NewDefaultSubscriptionStore()
 	if err != nil {
 		return errors.WithStack(err)
@@ -227,8 +230,8 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	logWriter = io.MultiWriter(os.Stderr, f)
+	common.LogWriter = io.MultiWriter(os.Stderr, f)
-	log.SetOutput(logWriter)
+	log.SetOutput(common.LogWriter)
 	log.Print("Running Gosora v" + common.SoftwareVersion.String())
 	fmt.Println("")
 

patcher/patches.go

@@ -25,6 +25,7 @@ func init() {
 	addPatch(11, patch11)
 	addPatch(12, patch12)
 	addPatch(13, patch13)
+	addPatch(14, patch14)
 }
 
 func patch0(scanner *bufio.Scanner) (err error) {
@@ -514,3 +515,20 @@ func patch13(scanner *bufio.Scanner) error {
 
 	return nil
 }
+
+func patch14(scanner *bufio.Scanner) error {
+	err := execStmt(qgen.Builder.AddKey("topics", "title", tblKey{"title", "fulltext"}))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddKey("topics", "content", tblKey{"content", "fulltext"}))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddKey("replies", "content", tblKey{"content", "fulltext"}))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

public/analytics.js

@@ -2,7 +2,9 @@
 
 })*/
 
-function buildStatsChart(rawLabels, seriesData, timeRange) {
+// TODO: Fully localise this
+// TODO: Load rawLabels and seriesData dynamically rather than potentially fiddling with nonces for the CSP?
+function buildStatsChart(rawLabels, seriesData, timeRange, legendNames) {
 	let labels = [];
 	if(timeRange=="one-month") {
 		labels = ["today","01 days"];
@@ -28,12 +30,20 @@ function buildStatsChart(rawLabels, seriesData, timeRange) {
 		}
 	}
 	labels = labels.reverse()
-	seriesData = seriesData.reverse();
+	for(let i = 0; i < seriesData.length;i++) {
+		seriesData[i] = seriesData[i].reverse();
+	}
 
+	let config = {
+		height: '250px',
+	};
+	if(legendNames.length > 0) config.plugins = [
+		Chartist.plugins.legend({
+			legendNames: legendNames,
+		})
+    ];
 	Chartist.Line('.ct_chart', {
 		labels: labels,
-		series: [seriesData],
+		series: seriesData,
-	}, {
+	}, config);
-		height: '250px',
-	});
 }

public/chartist/chartist-plugin-legend.css

@@ -0,0 +1,75 @@
+.ct-legend {
+    position: relative;
+           z-index: 10;
+           list-style: none;
+           text-align: center;
+       }
+       .ct-legend li {
+           position: relative;
+           padding-left: 23px;
+           margin-right: 10px;
+           margin-bottom: 3px;
+           cursor: pointer;
+           display: inline-block;
+       }
+       .ct-legend li:before {
+           width: 12px;
+           height: 12px;
+           position: absolute;
+           left: 0;
+           content: '';
+           border: 3px solid transparent;
+           border-radius: 2px;
+       }
+       .ct-legend li.inactive:before {
+           background: transparent;
+       }
+       .ct-legend.ct-legend-inside {
+           position: absolute;
+           top: 0;
+           right: 0;
+       }
+       .ct-legend.ct-legend-inside li{
+           display: block;
+           margin: 0;
+       }
+       .ct-legend .ct-series-0:before {
+           background-color: #d70206;
+           border-color: #d70206;
+       }
+       .ct-legend .ct-series-1:before {
+           background-color: #f05b4f;
+           border-color: #f05b4f;
+       }
+       .ct-legend .ct-series-2:before {
+           background-color: #f4c63d;
+           border-color: #f4c63d;
+       }
+       .ct-legend .ct-series-3:before {
+           background-color: #d17905;
+           border-color: #d17905;
+       }
+       .ct-legend .ct-series-4:before {
+           background-color: #453d3f;
+           border-color: #453d3f;
+       }
+       .ct-legend .ct-series-5:before {
+           background-color: #59922b;
+           border-color: #59922b;
+       }
+       .ct-legend .ct-series-6:before {
+           background-color: #0544d3;
+           border-color: #0544d3;
+       }
+
+       .ct-chart-line-multipleseries .ct-legend .ct-series-0:before {
+          background-color: #d70206;
+          border-color: #d70206;
+       }
+       .ct-chart-line-multipleseries .ct-legend .ct-series-1:before {
+          background-color: #f4c63d;
+          border-color: #f4c63d;
+       }
+       .ct-chart-line-multipleseries .ct-legend li.inactive:before {
+          background: transparent;
+        }

public/chartist/chartist-plugin-legend.min.js

@@ -0,0 +1,2 @@
+//https://github.com/CodeYellowBV/chartist-plugin-legend
+!function(e,t){"function"==typeof define&&define.amd?define(["chartist"],function(s){return e.returnExportsGlobal=t(s)}):"object"==typeof exports?module.exports=t(require("chartist")):e["Chartist.plugins.legend"]=t(e.Chartist)}(this,function(e){"use strict";var t={className:"",classNames:!1,removeAll:!1,legendNames:!1,clickable:!0,onClick:null,position:"top"};return e.plugins=e.plugins||{},e.plugins.legend=function(s){function a(e,t){return e-t}if(s&&s.position){if(!("top"===s.position||"bottom"===s.position||s.position instanceof HTMLElement))throw Error("The position you entered is not a valid position");if(s.position instanceof HTMLElement){var i=s.position;delete s.position}}return s=e.extend({},t,s),i&&(s.position=i),function(t){var i=t.container.querySelector(".ct-legend");if(i&&i.parentNode.removeChild(i),s.clickable){var n=t.data.series.map(function(s,a){return"object"!=typeof s&&(s={value:s}),s.className=s.className||t.options.classNames.series+"-"+e.alphaNumerate(a),s});t.data.series=n}var o=document.createElement("ul"),l=t instanceof e.Pie;o.className="ct-legend",t instanceof e.Pie&&o.classList.add("ct-legend-inside"),"string"==typeof s.className&&s.className.length>0&&o.classList.add(s.className),t.options.width&&(o.style.cssText="width: "+t.options.width+"px;margin: 0 auto;");var r=[],c=t.data.series.slice(0),d=t.data.series,p=l&&t.data.labels&&t.data.labels.length;if(p){var u=t.data.labels.slice(0);d=t.data.labels}d=s.legendNames||d;var f=Array.isArray(s.classNames)&&s.classNames.length===d.length;d.forEach(function(e,t){var a=document.createElement("li");a.className="ct-series-"+t,f&&(a.className+=" "+s.classNames[t]),a.setAttribute("data-legend",t),a.textContent=e.name||e,o.appendChild(a)}),t.on("created",function(e){if(s.position instanceof HTMLElement)s.position.insertBefore(o,null);else switch(s.position){case"top":t.container.insertBefore(o,t.container.childNodes[0]);break;case"bottom":t.container.insertBefore(o,null)}}),s.clickable&&o.addEventListener("click",function(e){var i=e.target;if(i.parentNode===o&&i.hasAttribute("data-legend")){e.preventDefault();var n=parseInt(i.getAttribute("data-legend")),l=r.indexOf(n);l>-1?(r.splice(l,1),i.classList.remove("inactive")):s.removeAll?(r.push(n),i.classList.add("inactive")):t.data.series.length>1?(r.push(n),i.classList.add("inactive")):(r=[],Array.prototype.slice.call(o.childNodes).forEach(function(e){e.classList.remove("inactive")}));var d=c.slice(0);if(p)var f=u.slice(0);r.sort(a).reverse(),r.forEach(function(e){d.splice(e,1),p&&f.splice(e,1)}),s.onClick&&s.onClick(t,e),t.data.series=d,p&&(t.data.labels=f),t.update()}})}},e.plugins.legend});

public/global.js

@@ -392,8 +392,7 @@ function mainInit(){
 
 	$(".link_label").click(function(event) {
 		event.preventDefault();
-		let forSelect = $(this).attr("data-for");
+		let linkSelect = $('#'+$(this).attr("data-for"));
-		let linkSelect = $('#'+forSelect);
 		if(!linkSelect.hasClass("link_opened")) {
 			event.stopPropagation();
 			linkSelect.addClass("link_opened");
@@ -415,9 +414,7 @@ function mainInit(){
 			this.outerHTML = Template_paginator({PageList: pageList, Page: page, LastPage: lastPage});
 			ok = true;
 		});
-		if(!ok) {
+		if(!ok) $(Template_paginator({PageList: pageList, Page: page, LastPage: lastPage})).insertAfter("#topic_list");
-			$(Template_paginator({PageList: pageList, Page: page, LastPage: lastPage})).insertAfter("#topic_list");
-		}
 	}
 
 	function rebindPaginator() {
@@ -496,6 +493,41 @@ function mainInit(){
 	if (document.getElementById("topicsItemList")!==null) rebindPaginator();
 	if (document.getElementById("forumItemList")!==null) rebindPaginator();
 
+	// TODO: Show a search button when JS is disabled?
+	$(".widget_search_input").keypress(function(e) {
+		if (e.keyCode != '13') return;
+		event.preventDefault();
+		// TODO: Take mostviewed into account
+		let url = "//"+window.location.host+window.location.pathname;
+		let urlParams = new URLSearchParams(window.location.search);
+		urlParams.set("q",this.value);
+		let q = "?";
+		for(let item of urlParams.entries()) q += item[0]+"="+item[1]+"&";
+		if(q.length>1) q = q.slice(0,-1);
+
+		// TODO: Try to de-duplicate some of these fetch calls
+		fetch(url+q+"&js=1", {credentials: "same-origin"})
+			.then((resp) => resp.json())
+			.then((data) => {
+				if(!"Topics" in data) throw("no Topics in data");
+				let topics = data["Topics"];
+
+				// TODO: Fix the data race where the function hasn't been loaded yet
+				let out = "";
+				for(let i = 0; i < topics.length;i++) out += Template_topics_topic(topics[i]);
+				$(".topic_list").html(out);
+
+				let obj = {Title: document.title, Url: url+q};
+				history.pushState(obj, obj.Title, obj.Url);
+				rebuildPaginator(data.LastPage);
+				rebindPaginator();
+		}).catch((ex) => {
+			console.log("Unable to get script '"+url+q+"&js=1"+"'");
+			console.log("ex: ", ex);
+			console.trace();
+		});
+	});
+
 	$(".open_edit").click((event) => {
 		event.preventDefault();
 		$('.hide_on_edit').addClass("edit_opened");

query_gen/builder.go

@@ -116,6 +116,10 @@ func (build *builder) AddIndex(table string, iname string, colname string) (stmt
 	return build.prepare(build.adapter.AddIndex("", table, iname, colname))
 }
 
+func (build *builder) AddKey(table string, column string, key DBTableKey) (stmt *sql.Stmt, err error) {
+	return build.prepare(build.adapter.AddKey("", table, column, key))
+}
+
 func (build *builder) SimpleInsert(table string, columns string, fields string) (stmt *sql.Stmt, err error) {
 	return build.prepare(build.adapter.SimpleInsert("", table, columns, fields))
 }

query_gen/mssql.go

@@ -136,7 +136,7 @@ func (adapter *MssqlAdapter) parseColumn(column DBTableColumn) (col DBTableColum
 
 // TODO: Test this, not sure if some things work
 // TODO: Add support for keys
-func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTableColumn,key *DBTableKey) (string, error) {
+func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")
 	}
@@ -162,6 +162,18 @@ func (adapter *MssqlAdapter) AddIndex(name string, table string, iname string, c
 	return "", errors.New("not implemented")
 }
 
+// TODO: Implement this
+// TODO: Test to make sure everything works here
+func (adapter *MssqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if column == "" {
+		return "", errors.New("You need a name for the column")
+	}
+	return "", errors.New("not implemented")
+}
+
 func (adapter *MssqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")

query_gen/mysql.go

@@ -213,6 +213,22 @@ func (adapter *MysqlAdapter) AddIndex(name string, table string, iname string, c
 	return querystr, nil
 }
 
+// TODO: Test to make sure everything works here
+// Only supports FULLTEXT right now
+func (adapter *MysqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if key.Type != "fulltext" {
+		return "", errors.New("Only fulltext is supported by AddKey right now")
+	}
+	querystr := "ALTER TABLE `" + table + "` ADD FULLTEXT(`" + column + "`)"
+
+	// TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator
+	adapter.pushStatement(name, "add-key", querystr)
+	return querystr, nil
+}
+
 func (adapter *MysqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")
@@ -689,11 +705,23 @@ func (adapter *MysqlAdapter) buildLimit(limit string) (querystr string) {
 
 func (adapter *MysqlAdapter) buildJoinColumns(columns string) (querystr string) {
 	for _, column := range processColumns(columns) {
+		// TODO: Move the stirng and number logic to processColumns?
+		// TODO: Error if [0] doesn't exist
+		firstChar := column.Left[0]
+		if firstChar == '\'' {
+			column.Type = "string"
+		} else {
+			_, err := strconv.Atoi(string(firstChar))
+			if err == nil {
+				column.Type = "number"
+			}
+		}
+
 		// Escape the column names, just in case we've used a reserved keyword
 		var source = column.Left
 		if column.Table != "" {
 			source = "`" + column.Table + "`.`" + source + "`"
-		} else if column.Type != "function" {
+		} else if column.Type != "function" && column.Type != "number" && column.Type != "substitute" && column.Type != "string" {
 			source = "`" + source + "`"
 		}
 

query_gen/pgsql.go

@@ -113,7 +113,7 @@ func (adapter *PgsqlAdapter) CreateTable(name string, table string, charset stri
 }
 
 // TODO: Implement this
-func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTableColumn,key *DBTableKey) (string, error) {
+func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")
 	}
@@ -135,6 +135,18 @@ func (adapter *PgsqlAdapter) AddIndex(name string, table string, iname string, c
 	return "", errors.New("not implemented")
 }
 
+// TODO: Implement this
+// TODO: Test to make sure everything works here
+func (adapter *PgsqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) {
+	if table == "" {
+		return "", errors.New("You need a name for this table")
+	}
+	if column == "" {
+		return "", errors.New("You need a name for the column")
+	}
+	return "", errors.New("not implemented")
+}
+
 // TODO: Test this
 // ! We need to get the last ID out of this somehow, maybe add returning to every query? Might require some sort of wrapper over the sql statements
 func (adapter *PgsqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) {

query_gen/querygen.go

@@ -110,6 +110,7 @@ type Adapter interface {
 	// TODO: Test this
 	AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error)
 	AddIndex(name string, table string, iname string, colname string) (string, error)
+	AddKey(name string, table string, column string, key DBTableKey) (string, error)
 	SimpleInsert(name string, table string, columns string, fields string) (string, error)
 	SimpleUpdate(up *updatePrebuilder) (string, error)
 	SimpleUpdateSelect(up *updatePrebuilder) (string, error) // ! Experimental

query_gen/utils.go

@@ -2,17 +2,17 @@
 *
 *	Query Generator Library
 *	WIP Under Construction
-*	Copyright Azareal 2017 - 2019
+*	Copyright Azareal 2017 - 2020
 *
  */
 package qgen
 
-//import "fmt"
 import (
 	"os"
 	"strings"
 )
 
+// TODO: Add support for numbers and strings?
 func processColumns(colstr string) (columns []DBColumn) {
 	if colstr == "" {
 		return columns

router_gen/main.go

@@ -222,6 +222,7 @@ func main() {
 		"malformed",
 		"suspicious",
 		"semrush",
+		"dotbot",
 		"zgrab",
 	}
 
@@ -257,6 +258,7 @@ func main() {
 		"Discourse",
 
 		"SemrushBot",
+		"DotBot",
 		"zgrab",
 	}
 
@@ -287,6 +289,7 @@ func main() {
 		"Discourse":   "discourse",
 
 		"SemrushBot": "semrush",
+		"DotBot":     "dotbot",
 		"zgrab":      "zgrab",
 	}
 
@@ -433,7 +436,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
 	var heads string
 	for key, value := range req.Header {
 		for _, vvalue := range value {
-			heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!!\n"
+			heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!\n"
 		}
 	}
 
@@ -583,7 +586,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			} else {
 				// TODO: Test this
 				items = items[:0]
-				r.SuspiciousRequest(req,"Illegal char in UA")
+				r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA")
 				r.requestLogger.Print("UA Buffer: ", buffer)
 				r.requestLogger.Print("UA Buffer String: ", string(buffer))
 				break
@@ -607,7 +610,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("parsed agent: ", agent)
 		}
-
 		if common.Dev.SuperDebug {
 			r.requestLogger.Print("os: ", os)
 			r.requestLogger.Printf("items: %+v\n",items)
@@ -653,7 +655,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		lang = strings.TrimSpace(lang)
 		lLang := strings.Split(lang,"-")
 		common.DebugDetail("lLang:", lLang)
-		counters.LangViewCounter.Bump(lLang[0])
+		validCode := counters.LangViewCounter.Bump(lLang[0])
+		if !validCode {
+			r.DumpRequest(req,"Invalid ISO Code")
+		}
 	} else {
 		counters.LangViewCounter.Bump("none")
 	}

routes/panel/analytics.go

@@ -30,6 +30,13 @@ func analyticsTimeRange(rawTimeRange string) (timeRange AnalyticsTimeRange, err
 	timeRange.Range = "six-hours"
 
 	switch rawTimeRange {
+	// This might be pushing it, we might want to come up with a more efficient scheme for dealing with large timeframes like this
+	case "three-months":
+		timeRange.Quantity = 90
+		timeRange.Unit = "day"
+		timeRange.Slices = 30
+		timeRange.SliceWidth = 60 * 60 * 24 * 3
+		timeRange.Range = "three-months"
 	case "one-month":
 		timeRange.Quantity = 30
 		timeRange.Unit = "day"
@@ -59,7 +66,6 @@ func analyticsTimeRange(rawTimeRange string) (timeRange AnalyticsTimeRange, err
 		timeRange.Slices = 24
 		timeRange.Range = "twelve-hours"
 	case "six-hours", "":
-		timeRange.Range = "six-hours"
 	default:
 		return timeRange, errors.New("Unknown time range")
 	}
@@ -89,7 +95,6 @@ func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64
 		if err != nil {
 			return viewMap, err
 		}
-
 		var unixCreatedAt = createdAt.Unix()
 		// TODO: Bulk log this
 		if common.Dev.SuperDebug {
@@ -97,7 +102,6 @@ func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64
 			log.Print("createdAt: ", createdAt)
 			log.Print("unixCreatedAt: ", unixCreatedAt)
 		}
-
 		for _, value := range labelList {
 			if unixCreatedAt > value {
 				viewMap[value] += count
@@ -113,7 +117,6 @@ func PreAnalyticsDetail(w http.ResponseWriter, r *http.Request, user *common.Use
 	if ferr != nil {
 		return nil, ferr
 	}
-
 	basePage.AddSheet("chartist/chartist.min.css")
 	basePage.AddScript("chartist/chartist.min.js")
 	basePage.AddScript("analytics.js")
@@ -125,7 +128,6 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co
 	if ferr != nil {
 		return ferr
 	}
-
 	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
 	if err != nil {
 		return common.LocalError(err.Error(), w, r, user)
@@ -138,7 +140,6 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -150,7 +151,7 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co
 		viewList = append(viewList, viewMap[value])
 		viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range}
@@ -162,7 +163,6 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use
 	if ferr != nil {
 		return ferr
 	}
-
 	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
 	if err != nil {
 		return common.LocalError(err.Error(), w, r, user)
@@ -175,7 +175,6 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -187,7 +186,7 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use
 		viewList = append(viewList, viewMap[value])
 		viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	pi := common.PanelAnalyticsRoutePage{basePage, common.SanitiseSingleLine(route), graph, viewItems, timeRange.Range}
@@ -199,13 +198,11 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use
 	if ferr != nil {
 		return ferr
 	}
-
 	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
 	if err != nil {
 		return common.LocalError(err.Error(), w, r, user)
 	}
 	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)
-
 	// ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff
 	agent = common.SanitiseSingleLine(agent)
 
@@ -215,7 +212,6 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -225,7 +221,7 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use
 	for _, value := range revLabelList {
 		viewList = append(viewList, viewMap[value])
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	friendlyAgent, ok := phrases.GetUserAgentPhrase(agent)
@@ -259,7 +255,6 @@ func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user common.Use
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -269,7 +264,7 @@ func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user common.Use
 	for _, value := range revLabelList {
 		viewList = append(viewList, viewMap[value])
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	forum, err := common.Forums.Get(fid)
@@ -286,7 +281,6 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us
 	if ferr != nil {
 		return ferr
 	}
-
 	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
 	if err != nil {
 		return common.LocalError(err.Error(), w, r, user)
@@ -300,7 +294,6 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -310,7 +303,7 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us
 	for _, value := range revLabelList {
 		viewList = append(viewList, viewMap[value])
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	friendlySystem, ok := phrases.GetOSPhrase(system)
@@ -350,7 +343,7 @@ func AnalyticsLanguageViews(w http.ResponseWriter, r *http.Request, user common.
 	for _, value := range revLabelList {
 		viewList = append(viewList, viewMap[value])
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
 
 	friendlyLang, ok := phrases.GetHumanLangPhrase(lang)
@@ -379,7 +372,6 @@ func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user common.
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -389,9 +381,8 @@ func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user common.
 	for _, value := range revLabelList {
 		viewList = append(viewList, viewMap[value])
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
-
 	pi := common.PanelAnalyticsAgentPage{basePage, common.SanitiseSingleLine(domain), "", graph, timeRange.Range}
 	return renderTemplate("panel_analytics_referrer_views", w, r, basePage.Header, &pi)
 }
@@ -412,7 +403,6 @@ func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) c
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -424,9 +414,8 @@ func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) c
 		viewList = append(viewList, viewMap[value])
 		viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
-
 	pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range}
 	return renderTemplate("panel_analytics_topics", w, r, basePage.Header, &pi)
 }
@@ -447,7 +436,6 @@ func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user common.User) co
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -459,13 +447,39 @@ func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user common.User) co
 		viewList = append(viewList, viewMap[value])
 		viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
 	}
-	graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
+	graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
 	common.DebugLogf("graph: %+v\n", graph)
-
 	pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range}
 	return renderTemplate("panel_analytics_posts", w, r, basePage.Header, &pi)
 }
 
+/*func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[int64]int64, error) {
+	defer rows.Close()
+	for rows.Next() {
+		var count int64
+		var createdAt time.Time
+		err := rows.Scan(&count, &createdAt)
+		if err != nil {
+			return viewMap, err
+		}
+
+		var unixCreatedAt = createdAt.Unix()
+		// TODO: Bulk log this
+		if common.Dev.SuperDebug {
+			log.Print("count: ", count)
+			log.Print("createdAt: ", createdAt)
+			log.Print("unixCreatedAt: ", unixCreatedAt)
+		}
+		for _, value := range labelList {
+			if unixCreatedAt > value {
+				viewMap[value] += count
+				break
+			}
+		}
+	}
+	return viewMap, rows.Err()
+}*/
+
 func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) {
 	nameMap := make(map[string]int)
 	defer rows.Close()
@@ -476,7 +490,6 @@ func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) {
 		if err != nil {
 			return nameMap, err
 		}
-
 		// TODO: Bulk log this
 		if common.Dev.SuperDebug {
 			log.Print("count: ", count)
@@ -487,6 +500,46 @@ func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) {
 	return nameMap, rows.Err()
 }
 
+func analyticsRowsToDuoMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[string]map[int64]int64, map[string]int, error) {
+	vMap := make(map[string]map[int64]int64)
+	nameMap := make(map[string]int)
+	defer rows.Close()
+	for rows.Next() {
+		var count int64
+		var name string
+		var createdAt time.Time
+		err := rows.Scan(&count, &name, &createdAt)
+		if err != nil {
+			return vMap, nameMap, err
+		}
+
+		// TODO: Bulk log this
+		var unixCreatedAt = createdAt.Unix()
+		if common.Dev.SuperDebug {
+			log.Print("count: ", count)
+			log.Print("name: ", name)
+			log.Print("createdAt: ", createdAt)
+			log.Print("unixCreatedAt: ", unixCreatedAt)
+		}
+		vvMap, ok := vMap[name]
+		if !ok {
+			vvMap = make(map[int64]int64)
+			for key, val := range viewMap {
+				vvMap[key] = val
+			}
+			vMap[name] = vvMap
+		}
+		for _, value := range labelList {
+			if unixCreatedAt > value {
+				vvMap[value] += count
+				break
+			}
+		}
+		nameMap[name] += int(count)
+	}
+	return vMap, nameMap, rows.Err()
+}
+
 func AnalyticsForums(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
 	basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics")
 	if ferr != nil {
@@ -501,7 +554,6 @@ func AnalyticsForums(w http.ResponseWriter, r *http.Request, user common.User) c
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	forumMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -543,7 +595,6 @@ func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) c
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	routeMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -562,26 +613,78 @@ func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) c
 	return renderTemplate("panel_analytics_routes", w, r, basePage.Header, &pi)
 }
 
+type OVItem struct {
+	name    string
+	count   int
+	viewMap map[int64]int64
+}
+
+// Trialling multi-series charts
 func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
-	basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics")
+	basePage, ferr := PreAnalyticsDetail(w, r, &user)
 	if ferr != nil {
 		return ferr
 	}
+	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
+	basePage.AddSheet("chartist/chartist-plugin-legend.css")
+
 	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
 	if err != nil {
 		return common.LocalError(err.Error(), w, r, user)
 	}
+	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)
 
-	rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, browser").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
+	rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, browser, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
+	vMap, agentMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
-	agentMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)
 	}
 
+	// Order the map
+	var ovList []OVItem
+	for name, viewMap := range vMap {
+		var totcount int
+		for _, count := range viewMap {
+			totcount += int(count)
+		}
+		ovList = append(ovList, OVItem{name, totcount, viewMap})
+	}
+	// Use bubble sort for now as there shouldn't be too many items
+	for i := 0; i < len(ovList)-1; i++ {
+		for j := 0; j < len(ovList)-1; j++ {
+			if ovList[j].count > ovList[j+1].count {
+				temp := ovList[j]
+				ovList[j] = ovList[j+1]
+				ovList[j+1] = temp
+			}
+		}
+	}
+
+	var vList [][]int64
+	var legendList []string
+	var i int
+	for _, ovitem := range ovList {
+		var viewList []int64
+		for _, value := range revLabelList {
+			viewList = append(viewList, ovitem.viewMap[value])
+		}
+		vList = append(vList, viewList)
+		lName, ok := phrases.GetUserAgentPhrase(ovitem.name)
+		if !ok {
+			lName = ovitem.name
+		}
+		legendList = append(legendList, lName)
+		if i >= 6 {
+			break
+		}
+		i++
+	}
+	graph := common.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
+	common.DebugLogf("graph: %+v\n", graph)
+
 	// TODO: Sort this slice
 	var agentItems []common.PanelAnalyticsAgentsItem
 	for agent, count := range agentMap {
@@ -596,7 +699,7 @@ func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) c
 		})
 	}
 
-	pi := common.PanelAnalyticsAgentsPage{basePage, agentItems, timeRange.Range}
+	pi := common.PanelAnalyticsDuoPage{basePage, agentItems, graph, timeRange.Range}
 	return renderTemplate("panel_analytics_agents", w, r, basePage.Header, &pi)
 }
 
@@ -614,7 +717,6 @@ func AnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User)
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	osMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -652,7 +754,6 @@ func AnalyticsLanguages(w http.ResponseWriter, r *http.Request, user common.User
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	langMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)
@@ -691,7 +792,6 @@ func AnalyticsReferrers(w http.ResponseWriter, r *http.Request, user common.User
 	if err != nil && err != sql.ErrNoRows {
 		return common.InternalError(err, w, r)
 	}
-
 	refMap, err := analyticsRowsToNameMap(rows)
 	if err != nil {
 		return common.InternalError(err, w, r)

routes/topic_list.go

@@ -1,6 +1,7 @@
 package routes
 
 import (
+	"database/sql"
 	"log"
 	"net/http"
 	"strconv"
@@ -10,72 +11,6 @@ import (
 	"github.com/Azareal/Gosora/common/phrases"
 )
 
-// TODO: Implement search
-func TopicList(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError {
-	group, err := common.Groups.Get(user.Group)
-	if err != nil {
-		log.Printf("Group #%d doesn't exist despite being used by common.User #%d", user.Group, user.ID)
-		return common.LocalError("Something weird happened", w, r, user)
-	}
-
-	// Get the current page
-	page, _ := strconv.Atoi(r.FormValue("page"))
-	sfids := r.FormValue("fids")
-	var fids []int
-	if sfids != "" {
-		for _, sfid := range strings.Split(sfids, ",") {
-			fid, err := strconv.Atoi(sfid)
-			if err != nil {
-				return common.LocalError("Invalid fid", w, r, user)
-			}
-			fids = append(fids, fid)
-		}
-	}
-
-	// TODO: Pass a struct back rather than passing back so many variables
-	var topicList []*common.TopicsRow
-	var forumList []common.Forum
-	var paginator common.Paginator
-	if user.IsSuperAdmin {
-		topicList, forumList, paginator, err = common.TopicList.GetList(page, "", fids)
-	} else {
-		topicList, forumList, paginator, err = common.TopicList.GetListByGroup(group, page, "", fids)
-	}
-	if err != nil {
-		return common.InternalError(err, w, r)
-	}
-	// ! Need an inline error not a page level error
-	if len(topicList) == 0 {
-		return common.NotFound(w, r, header)
-	}
-
-	// TODO: Reduce the amount of boilerplate here
-	if r.FormValue("js") == "1" {
-		outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON()
-		if err != nil {
-			return common.InternalError(err, w, r)
-		}
-		w.Write(outBytes)
-		return nil
-	}
-
-	header.Title = phrases.GetTitlePhrase("topics")
-	header.Zone = "topics"
-	header.Path = "/topics/"
-	header.MetaDesc = header.Settings["meta_desc"].(string)
-	if len(fids) == 1 {
-		forum, err := common.Forums.Get(fids[0])
-		if err != nil {
-			return common.LocalError("Invalid fid forum", w, r, user)
-		}
-		header.Title = forum.Name
-		header.ZoneID = forum.ID
-	}
-
-	pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{"lastupdated", false}, paginator}
-	return renderTemplate("topics", w, r, header, pi)
-}
-
 func wsTopicList(topicList []*common.TopicsRow, lastPage int) *common.WsTopicList {
 	wsTopicList := make([]*common.WsTopicsRow, len(topicList))
 	for i, topicRow := range topicList {
@@ -84,7 +19,16 @@ func wsTopicList(topicList []*common.TopicsRow, lastPage int) *common.WsTopicLis
 	return &common.WsTopicList{wsTopicList, lastPage}
 }
 
+func TopicList(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError {
+	return TopicListCommon(w, r, user, header, "lastupdated")
+}
+
 func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError {
+	return TopicListCommon(w, r, user, header, "mostviewed")
+}
+
+// TODO: Implement search
+func TopicListCommon(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header, torder string) common.RouteError {
 	header.Title = phrases.GetTitlePhrase("topics")
 	header.Zone = "topics"
 	header.Path = "/topics/"
@@ -118,10 +62,111 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use
 		}
 	}
 
-	// TODO: Pass a struct back rather than passing back so many variables
+	//(t *Topic) WsTopicsRows() *WsTopicsRow
+	// TODO: Allow multiple forums in searches
+	// TODO: Simplify this block after initially landing search
 	var topicList []*common.TopicsRow
 	var forumList []common.Forum
 	var paginator common.Paginator
+	q := r.FormValue("q")
+	if q != "" {
+		var canSee []int
+		if user.IsSuperAdmin {
+			canSee, err = common.Forums.GetAllVisibleIDs()
+			if err != nil {
+				return common.InternalError(err, w, r)
+			}
+		} else {
+			canSee = group.CanSee
+		}
+
+		var cfids []int
+		if len(fids) > 0 {
+			var inSlice = func(haystack []int, needle int) bool {
+				for _, item := range haystack {
+					if needle == item {
+						return true
+					}
+				}
+				return false
+			}
+			for _, fid := range fids {
+				if inSlice(canSee, fid) {
+					forum := common.Forums.DirtyGet(fid)
+					if forum.Name != "" && forum.Active && (forum.ParentType == "" || forum.ParentType == "forum") {
+						// TODO: Add a hook here for plugin_guilds?
+						cfids = append(cfids, fid)
+					}
+				}
+			}
+		} else {
+			cfids = canSee
+		}
+
+		tids, err := common.RepliesSearch.Query(q, cfids)
+		if err != nil && err != sql.ErrNoRows {
+			return common.InternalError(err, w, r)
+		}
+		//fmt.Printf("tids %+v\n", tids)
+		// TODO: Handle the case where there aren't any items...
+		// TODO: Add a BulkGet method which returns a slice?
+		tMap, err := common.Topics.BulkGetMap(tids)
+		if err != nil {
+			return common.InternalError(err, w, r)
+		}
+		var reqUserList = make(map[int]bool)
+		for _, topic := range tMap {
+			reqUserList[topic.CreatedBy] = true
+			reqUserList[topic.LastReplyBy] = true
+			topicList = append(topicList, topic.TopicsRow())
+		}
+		//fmt.Printf("reqUserList %+v\n", reqUserList)
+
+		// Convert the user ID map to a slice, then bulk load the users
+		var idSlice = make([]int, len(reqUserList))
+		var i int
+		for userID := range reqUserList {
+			idSlice[i] = userID
+			i++
+		}
+
+		// TODO: What if a user is deleted via the Control Panel?
+		//fmt.Printf("idSlice %+v\n", idSlice)
+		userList, err := common.Users.BulkGetMap(idSlice)
+		if err != nil {
+			return nil // TODO: Implement this!
+		}
+
+		// TODO: De-dupe this logic in common/topic_list.go?
+		for _, topic := range topicList {
+			topic.Link = common.BuildTopicURL(common.NameToSlug(topic.Title), topic.ID)
+			// TODO: Pass forum to something like topic.Forum and use that instead of these two properties? Could be more flexible.
+			forum := common.Forums.DirtyGet(topic.ParentID)
+			topic.ForumName = forum.Name
+			topic.ForumLink = forum.Link
+
+			// TODO: Create a specialised function with a bit less overhead for getting the last page for a post count
+			_, _, lastPage := common.PageOffset(topic.PostCount, 1, common.Config.ItemsPerPage)
+			topic.LastPage = lastPage
+			topic.Creator = userList[topic.CreatedBy]
+			topic.LastUser = userList[topic.LastReplyBy]
+		}
+
+		// TODO: Reduce the amount of boilerplate here
+		if r.FormValue("js") == "1" {
+			outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON()
+			if err != nil {
+				return common.InternalError(err, w, r)
+			}
+			w.Write(outBytes)
+			return nil
+		}
+
+		pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{torder, false}, paginator}
+		return renderTemplate("topics", w, r, header, pi)
+	}
+
+	// TODO: Pass a struct back rather than passing back so many variables
 	if user.IsSuperAdmin {
 		topicList, forumList, paginator, err = common.TopicList.GetList(page, "most-viewed", fids)
 	} else {
@@ -135,7 +180,6 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use
 		return common.NotFound(w, r, header)
 	}
 
-	//MarshalJSON() ([]byte, error)
 	// TODO: Reduce the amount of boilerplate here
 	if r.FormValue("js") == "1" {
 		outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON()
@@ -146,6 +190,6 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use
 		return nil
 	}
 
-	pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{"mostviewed", false}, paginator}
+	pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{torder, false}, paginator}
 	return renderTemplate("topics", w, r, header, pi)
 }

schema/mssql/query_replies.sql

@@ -14,5 +14,6 @@ CREATE TABLE [replies] (
 	[words] int DEFAULT 1 not null,
 	[actionType] nvarchar (20) DEFAULT '' not null,
 	[poll] int DEFAULT 0 not null,
-	primary key([rid])
+	primary key([rid]),
+	fulltext key([content])
 );

schema/mssql/query_topics.sql

@@ -20,5 +20,6 @@ CREATE TABLE [topics] (
 	[css_class] nvarchar (100) DEFAULT '' not null,
 	[poll] int DEFAULT 0 not null,
 	[data] nvarchar (200) DEFAULT '' not null,
-	primary key([tid])
+	primary key([tid]),
+	fulltext key([content])
 );

schema/mysql/query_replies.sql

@@ -14,5 +14,6 @@ CREATE TABLE `replies` (
 	`words` int DEFAULT 1 not null,
 	`actionType` varchar(20) DEFAULT '' not null,
 	`poll` int DEFAULT 0 not null,
-	primary key(`rid`)
+	primary key(`rid`),
+	fulltext key(`content`)
 ) CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;

schema/mysql/query_topics.sql

@@ -20,5 +20,6 @@ CREATE TABLE `topics` (
 	`css_class` varchar(100) DEFAULT '' not null,
 	`poll` int DEFAULT 0 not null,
 	`data` varchar(200) DEFAULT '' not null,
-	primary key(`tid`)
+	primary key(`tid`),
+	fulltext key(`content`)
 ) CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;

schema/pgsql/query_replies.sql

@@ -14,5 +14,6 @@ CREATE TABLE "replies" (
 	`words` int DEFAULT 1 not null,
 	`actionType` varchar (20) DEFAULT '' not null,
 	`poll` int DEFAULT 0 not null,
-	primary key(`rid`)
+	primary key(`rid`),
+	fulltext key(`content`)
 );

schema/pgsql/query_topics.sql

@@ -20,5 +20,6 @@ CREATE TABLE "topics" (
 	`css_class` varchar (100) DEFAULT '' not null,
 	`poll` int DEFAULT 0 not null,
 	`data` varchar (200) DEFAULT '' not null,
-	primary key(`tid`)
+	primary key(`tid`),
+	fulltext key(`content`)
 );

templates/panel_analytics_agent_views.html

@@ -15,13 +15,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_agents.html

@@ -10,6 +10,9 @@
 				</div>
 			</div>
 		</form>
+		<div id="panel_analytics_agents_chart" class="colstack_graph_holder">
+			<div class="ct_chart"></div>
+		</div>
 		<div id="panel_analytics_agents" class="colstack_item rowlist">
 			{{range .ItemList}}
 			<div class="rowitem panel_compactrow editable_parent">
@@ -20,4 +23,5 @@
 		</div>
 	</main>
 </div>
+{{template "panel_analytics_script.html" . }}
 {{template "footer.html" . }}

templates/panel_analytics_forum_views.html

@@ -15,13 +15,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_lang_views.html

@@ -16,12 +16,5 @@
 	</main>
 </div>
 <script>
-let rawLabels = [{{range .PrimaryGraph.Labels}}
+{{template "panel_analytics_script.html" . }}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_posts.html

@@ -26,13 +26,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_referrer_views.html

@@ -15,13 +15,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_route_views.html

@@ -26,13 +26,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_script.html

@@ -0,0 +1,14 @@
+
+<script>
+let rawLabels = [{{range .Graph.Labels}}
+{{.}},{{end}}
+];
+let seriesData = [{{range .Graph.Series}}[{{range .}}
+{{.}},{{end}}
+],{{end}}
+];
+let legendNames = [{{range .Graph.Legends}}
+{{.}},{{end}}
+];
+buildStatsChart(rawLabels, seriesData.reverse(), "{{.TimeRange}}",legendNames.reverse());
+</script>

templates/panel_analytics_system_views.html

@@ -15,13 +15,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_time_range.html

@@ -1,4 +1,5 @@
 <select class="timeRangeSelector to_right" name="timeRange">
+	<option val="three-months"{{if eq .TimeRange "three-months"}} selected{{end}}>{{lang "panel_statistics_time_range_three_months"}}</option>
 	<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>{{lang "panel_statistics_time_range_one_month"}}</option>
 	<option val="one-week"{{if eq .TimeRange "one-week"}} selected{{end}}>{{lang "panel_statistics_time_range_one_week"}}</option>
 	<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>{{lang "panel_statistics_time_range_two_days"}}</option>

templates/panel_analytics_topics.html

@@ -26,13 +26,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_analytics_views.html

@@ -26,13 +26,5 @@
 		</div>
 	</main>
 </div>
-<script>
+{{template "panel_analytics_script.html" . }}
-let rawLabels = [{{range .PrimaryGraph.Labels}}
-{{.}},{{end}}
-];
-let seriesData = [{{range .PrimaryGraph.Series}}
-{{.}},{{end}}
-];
-buildStatsChart(rawLabels, seriesData, "{{.TimeRange}}");
-</script>
 {{template "footer.html" . }}

templates/panel_themes_widgets_widget.html

@@ -35,13 +35,13 @@
 <div class="formrow w_simple w_about">
 	<div class="formitem formlabel"><a>{{lang "panel_themes_widgets_body"}}</a></div>
 	<div class="formitem">
-		<textarea name="wtext">{{index .Data "Text"}}</textarea>
+		<textarea name="wtext" class="wtext">{{index .Data "Text"}}</textarea>
 	</div>
 </div>
 <div class="formrow w_default">
 	<div class="formitem formlabel"><a>{{lang "panel_themes_widgets_raw_body"}}</a></div>
 	<div class="formitem">
-		<textarea name="wbody">{{.RawBody}}</textarea>
+		<textarea name="wbody" class="rwtext">{{.RawBody}}</textarea>
 	</div>
 </div>
 </div>

templates/widget_search_and_filter.html

@@ -1,5 +1,5 @@
 <div class="search widget_search">
-	<input name="widget_search" placeholder="Search" />
+	<input class="widget_search_input" name="widget_search" placeholder="Search" />
 </div>
 <div class="rowblock filter_list widget_filter">
 {{range .Forums}}	<div class="rowitem filter_item{{if .Selected}} filter_selected{{end}}" data-fid="{{.ID}}"><a href="/topics/?fids={{.ID}}" rel="nofollow">{{.Name}}</a></div>

themes/nox/public/panel.css

@@ -125,6 +125,7 @@ button, .formbutton, .panel_right_button:not(.has_inner_button), #panel_users .p
 	padding: 16px;
 	padding-bottom: 0px;
 	padding-left: 0px;
+	margin-bottom: 10px;
 }
 .colstack_graph_holder .ct-label {
 	color: rgb(195,195,195);
@@ -299,6 +300,10 @@ button, .formbutton, .panel_right_button:not(.has_inner_button), #panel_users .p
 .wtype_about .w_about, .wtype_simple .w_simple, .wtype_wol .w_wol, .wtype_default .w_default {
 	display: block;
 }
+.wtext, .rwtext {
+	width: 100%;
+	height: 80px;
+}
 
 #panel_debug .grid_stat:not(.grid_stat_head) {
 	margin-bottom: 5px;