From 2296008655fb765033dc76a9977e827f3aa127e1 Mon Sep 17 00:00:00 2001 From: Azareal Date: Sat, 23 Feb 2019 16:29:19 +1000 Subject: [PATCH] Basic search now works for the Search & Filter Widget. ElasticSearch has been temporarily delayed, so I can push through this update. Added the three month time range to the analytics panes. Began work on adding new graphs to the analytics panes. Began work on the ElasticSearch adapter for the search system. Added the currently limited AddKey method to the database adapters. Expanded upon the column parsing logic in the database adapters to ease the use of InsertSelects. Added the BulkGet method to TopicCache. Added the BulkGetMap method to TopicStore. TopicStore methods should now properly retrieve lastReplyBy. Added the panel_analytics_script template to de-dupe part of the analytics logic. We plan to tidy this up further, but for now, it'll suffice. Added plugin_sendmail and plugin_hyperdrive to the continuous integration test list. Tweaked the width and heights of the textareas for the Widget Editor. Added the AddKey method to *qgen.builder Fixed a bug where using the inline forum editor would crash Gosora and wouldn't set the preset permissions for that forum properly. Added DotBot to the user agent analytics. Invisibles should be better handled when they're encountered now in user agent strings. Unknown language ISO Codes in headers now have the requests fully logged for debugging purposes. Shortened some of the pointer receiver names. Shortened some variable names. Added the dotbot phrase. Added the panel_statistics_time_range_three_months phrase. Added gopkg.in/olivere/elastic.v6 as a dependency. You will need to run the patcher or updater for this commit. --- .travis.yml | 2 + cmd/elasticsearch/setup.go | 206 ++++++++++++++++++ cmd/query_gen/tables.go | 2 + common/common.go | 4 + common/counters/langs.go | 8 +- common/forum.go | 2 +- common/forum_perms.go | 14 +- common/null_topic_cache.go | 25 ++- common/null_user_cache.go | 28 +-- common/pages.go | 40 ++-- common/search.go | 123 ++++++++--- common/template_init.go | 5 +- common/topic.go | 22 ++ common/topic_cache.go | 12 + common/topic_list.go | 31 +-- common/topic_store.go | 87 +++++++- common/user_store.go | 13 +- docs/updating.md | 2 +- gen_router.go | 17 +- go.mod | 3 + go.sum | 6 + langs/english.json | 2 + main.go | 11 +- patcher/patches.go | 18 ++ public/analytics.js | 22 +- public/chartist/chartist-plugin-legend.css | 75 +++++++ public/chartist/chartist-plugin-legend.min.js | 2 + public/global.js | 42 +++- query_gen/builder.go | 4 + query_gen/mssql.go | 14 +- query_gen/mysql.go | 30 ++- query_gen/pgsql.go | 14 +- query_gen/querygen.go | 1 + query_gen/utils.go | 4 +- router_gen/main.go | 13 +- routes/panel/analytics.go | 180 +++++++++++---- routes/topic_list.go | 182 ++++++++++------ schema/mssql/query_replies.sql | 3 +- schema/mssql/query_topics.sql | 3 +- schema/mysql/query_replies.sql | 3 +- schema/mysql/query_topics.sql | 3 +- schema/pgsql/query_replies.sql | 3 +- schema/pgsql/query_topics.sql | 3 +- templates/panel_analytics_agent_views.html | 10 +- templates/panel_analytics_agents.html | 4 + templates/panel_analytics_forum_views.html | 10 +- templates/panel_analytics_lang_views.html | 9 +- templates/panel_analytics_posts.html | 10 +- templates/panel_analytics_referrer_views.html | 10 +- templates/panel_analytics_route_views.html | 10 +- templates/panel_analytics_script.html | 14 ++ templates/panel_analytics_system_views.html | 10 +- templates/panel_analytics_time_range.html | 1 + templates/panel_analytics_topics.html | 10 +- templates/panel_analytics_views.html | 10 +- templates/panel_themes_widgets_widget.html | 4 +- templates/widget_search_and_filter.html | 2 +- themes/nox/public/panel.css | 5 + 58 files changed, 1061 insertions(+), 342 deletions(-) create mode 100644 cmd/elasticsearch/setup.go create mode 100644 public/chartist/chartist-plugin-legend.css create mode 100644 public/chartist/chartist-plugin-legend.min.js create mode 100644 templates/panel_analytics_script.html diff --git a/.travis.yml b/.travis.yml index d1598d48..071b3a0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ before_install: - mv ./config/config_example.json ./config/config.json - ./update-deps-linux - ./dev-update-travis + - mv ./experimental/plugin_sendmail.go .. + - mv ./experimental/plugin_hyperdrive.go .. install: true before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter diff --git a/cmd/elasticsearch/setup.go b/cmd/elasticsearch/setup.go new file mode 100644 index 00000000..24cf9248 --- /dev/null +++ b/cmd/elasticsearch/setup.go @@ -0,0 +1,206 @@ +// Work in progress +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "log" + "os" + "strconv" + + "github.com/Azareal/Gosora/common" + "github.com/Azareal/Gosora/query_gen" + "gopkg.in/olivere/elastic.v6" +) + +func main() { + log.Print("Loading the configuration data") + err := common.LoadConfig() + if err != nil { + log.Fatal(err) + } + + log.Print("Processing configuration data") + err = common.ProcessConfig() + if err != nil { + log.Fatal(err) + } + + if common.DbConfig.Adapter != "mysql" && common.DbConfig.Adapter != "" { + log.Fatal("Only MySQL is supported for upgrades right now, please wait for a newer build of the patcher") + } + + err = prepMySQL() + if err != nil { + log.Fatal(err) + } + + client, err := elastic.NewClient(elastic.SetErrorLog(log.New(os.Stdout, "ES ", log.LstdFlags))) + if err != nil { + log.Fatal(err) + } + _, _, err = client.Ping("http://127.0.0.1:9200").Do(context.Background()) + if err != nil { + log.Fatal(err) + } + + err = setupIndices(client) + if err != nil { + log.Fatal(err) + } + + err = setupData(client) + if err != nil { + log.Fatal(err) + } +} + +func prepMySQL() error { + return qgen.Builder.Init("mysql", map[string]string{ + "host": common.DbConfig.Host, + "port": common.DbConfig.Port, + "name": common.DbConfig.Dbname, + "username": common.DbConfig.Username, + "password": common.DbConfig.Password, + "collation": "utf8mb4_general_ci", + }) +} + +type ESIndexBase struct { + Mappings ESIndexMappings `json:"mappings"` +} + +type ESIndexMappings struct { + Doc ESIndexDoc `json:"_doc"` +} + +type ESIndexDoc struct { + Properties map[string]map[string]string `json:"properties"` +} + +type ESDocMap map[string]map[string]string + +func (d ESDocMap) Add(column string, cType string) { + d["column"] = map[string]string{"type": cType} +} + +func setupIndices(client *elastic.Client) error { + exists, err := client.IndexExists("topics").Do(context.Background()) + if err != nil { + return err + } + if exists { + deleteIndex, err := client.DeleteIndex("topics").Do(context.Background()) + if err != nil { + return err + } + if !deleteIndex.Acknowledged { + return errors.New("delete not acknowledged") + } + } + + docMap := make(ESDocMap) + docMap.Add("tid", "integer") + docMap.Add("title", "text") + docMap.Add("content", "text") + docMap.Add("createdBy", "integer") + docMap.Add("ip", "ip") + docMap.Add("suggest", "completion") + indexBase := ESIndexBase{ESIndexMappings{ESIndexDoc{docMap}}} + oBytes, err := json.Marshal(indexBase) + if err != nil { + return err + } + createIndex, err := client.CreateIndex("topics").Body(string(oBytes)).Do(context.Background()) + if err != nil { + return err + } + if !createIndex.Acknowledged { + return errors.New("not acknowledged") + } + + exists, err = client.IndexExists("replies").Do(context.Background()) + if err != nil { + return err + } + if exists { + deleteIndex, err := client.DeleteIndex("replies").Do(context.Background()) + if err != nil { + return err + } + if !deleteIndex.Acknowledged { + return errors.New("delete not acknowledged") + } + } + + docMap = make(ESDocMap) + docMap.Add("rid", "integer") + docMap.Add("tid", "integer") + docMap.Add("content", "text") + docMap.Add("createdBy", "integer") + docMap.Add("ip", "ip") + docMap.Add("suggest", "completion") + indexBase = ESIndexBase{ESIndexMappings{ESIndexDoc{docMap}}} + oBytes, err = json.Marshal(indexBase) + if err != nil { + return err + } + createIndex, err = client.CreateIndex("replies").Body(string(oBytes)).Do(context.Background()) + if err != nil { + return err + } + if !createIndex.Acknowledged { + return errors.New("not acknowledged") + } + + return nil +} + +type ESTopic struct { + ID int `json:"tid"` + Title string `json:"title"` + Content string `json:"content"` + CreatedBy int `json:"createdBy"` + IPAddress string `json:"ip"` +} + +type ESReply struct { + ID int `json:"rid"` + TID int `json:"tid"` + Content string `json:"content"` + CreatedBy int `json:"createdBy"` + IPAddress string `json:"ip"` +} + +func setupData(client *elastic.Client) error { + err := qgen.NewAcc().Select("topics").Cols("tid, title, content, createdBy, ipaddress").Each(func(rows *sql.Rows) error { + var tid, createdBy int + var title, content, ip string + err := rows.Scan(&tid, &title, &content, &createdBy, &ip) + if err != nil { + return err + } + + topic := ESTopic{tid, title, content, createdBy, ip} + _, err = client.Index().Index("topics").Type("_doc").Id(strconv.Itoa(tid)).BodyJson(topic).Do(context.Background()) + return err + }) + if err != nil { + return err + } + + return qgen.NewAcc().Select("replies").Cols("rid, tid, content, createdBy, ipaddress").Each(func(rows *sql.Rows) error { + var rid, tid, createdBy int + var content, ip string + err := rows.Scan(&rid, &tid, &content, &createdBy, &ip) + if err != nil { + return err + } + + reply := ESReply{rid, tid, content, createdBy, ip} + _, err = client.Index().Index("replies").Type("_doc").Id(strconv.Itoa(rid)).BodyJson(reply).Do(context.Background()) + return err + }) +} diff --git a/cmd/query_gen/tables.go b/cmd/query_gen/tables.go index 397e57f6..4a71bf0b 100644 --- a/cmd/query_gen/tables.go +++ b/cmd/query_gen/tables.go @@ -242,6 +242,7 @@ func createTables(adapter qgen.Adapter) error { }, []tblKey{ tblKey{"tid", "primary"}, + tblKey{"content", "fulltext"}, }, ) @@ -265,6 +266,7 @@ func createTables(adapter qgen.Adapter) error { }, []tblKey{ tblKey{"rid", "primary"}, + tblKey{"content", "fulltext"}, }, ) diff --git a/common/common.go b/common/common.go index 3ebc81ab..8aa1d3ba 100644 --- a/common/common.go +++ b/common/common.go @@ -8,7 +8,9 @@ package common // import "github.com/Azareal/Gosora/common" import ( "database/sql" + "io" "log" + "os" "sync/atomic" "time" @@ -112,6 +114,8 @@ func StoppedServer(msg ...interface{}) { var StopServerChan = make(chan []interface{}) +var LogWriter = io.MultiWriter(os.Stderr) + func DebugDetail(args ...interface{}) { if Dev.SuperDebug { log.Print(args...) diff --git a/common/counters/langs.go b/common/counters/langs.go index c19ac687..9f78ccbc 100644 --- a/common/counters/langs.go +++ b/common/counters/langs.go @@ -149,19 +149,23 @@ func (counter *DefaultLangViewCounter) insertChunk(count int, id int) error { return err } -func (counter *DefaultLangViewCounter) Bump(langCode string) { +func (counter *DefaultLangViewCounter) Bump(langCode string) (validCode bool) { + validCode = true id, ok := counter.codesToIndices[langCode] if !ok { // TODO: Tell the caller that the code's invalid id = 0 // Unknown + validCode = false } // TODO: Test this check common.DebugDetail("counter.buckets[", id, "]: ", counter.buckets[id]) if len(counter.buckets) <= id || id < 0 { - return + return validCode } counter.buckets[id].Lock() counter.buckets[id].counter++ counter.buckets[id].Unlock() + + return validCode } diff --git a/common/forum.go b/common/forum.go index f9c87e65..2c6bb11e 100644 --- a/common/forum.go +++ b/common/forum.go @@ -82,7 +82,7 @@ func (forum *Forum) Update(name string, desc string, active bool, preset string) if err != nil { return err } - if forum.Preset != preset || preset == "custom" || preset == "" { + if forum.Preset != preset && preset != "custom" && preset != "" { err = PermmapToQuery(PresetToPermmap(preset), forum.ID) if err != nil { return err diff --git a/common/forum_perms.go b/common/forum_perms.go index 5fb52adc..a2e09270 100644 --- a/common/forum_perms.go +++ b/common/forum_perms.go @@ -99,7 +99,6 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error { if err != nil { return err } - _, err = deleteForumPermsByForumTx.Exec(fid) if err != nil { return err @@ -112,13 +111,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error { addForumPermsToForumAdminsTx, err := qgen.Builder.SimpleInsertSelectTx(tx, qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""}, - qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 1", "", ""}, + qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 1", "", ""}, ) if err != nil { return err } - - _, err = addForumPermsToForumAdminsTx.Exec(fid, "", perms) + _, err = addForumPermsToForumAdminsTx.Exec(fid, perms) if err != nil { return err } @@ -130,12 +128,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error { addForumPermsToForumStaffTx, err := qgen.Builder.SimpleInsertSelectTx(tx, qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""}, - qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 0 AND is_mod = 1", "", ""}, + qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 0 AND is_mod = 1", "", ""}, ) if err != nil { return err } - _, err = addForumPermsToForumStaffTx.Exec(fid, "", perms) + _, err = addForumPermsToForumStaffTx.Exec(fid, perms) if err != nil { return err } @@ -147,12 +145,12 @@ func PermmapToQuery(permmap map[string]*ForumPerms, fid int) error { addForumPermsToForumMembersTx, err := qgen.Builder.SimpleInsertSelectTx(tx, qgen.DBInsert{"forums_permissions", "gid, fid, preset, permissions", ""}, - qgen.DBSelect{"users_groups", "gid, ? AS fid, ? AS preset, ? AS permissions", "is_admin = 0 AND is_mod = 0 AND is_banned = 0", "", ""}, + qgen.DBSelect{"users_groups", "gid, ?, '', ?", "is_admin = 0 AND is_mod = 0 AND is_banned = 0", "", ""}, ) if err != nil { return err } - _, err = addForumPermsToForumMembersTx.Exec(fid, "", perms) + _, err = addForumPermsToForumMembersTx.Exec(fid, perms) if err != nil { return err } diff --git a/common/null_topic_cache.go b/common/null_topic_cache.go index 27ddd851..fc76c9b5 100644 --- a/common/null_topic_cache.go +++ b/common/null_topic_cache.go @@ -10,34 +10,37 @@ func NewNullTopicCache() *NullTopicCache { } // nolint -func (mts *NullTopicCache) Get(id int) (*Topic, error) { +func (c *NullTopicCache) Get(id int) (*Topic, error) { return nil, ErrNoRows } -func (mts *NullTopicCache) GetUnsafe(id int) (*Topic, error) { +func (c *NullTopicCache) GetUnsafe(id int) (*Topic, error) { return nil, ErrNoRows } -func (mts *NullTopicCache) Set(_ *Topic) error { +func (c *NullTopicCache) BulkGet(ids []int) (list []*Topic) { + return make([]*Topic, len(ids)) +} +func (c *NullTopicCache) Set(_ *Topic) error { return nil } -func (mts *NullTopicCache) Add(_ *Topic) error { +func (c *NullTopicCache) Add(_ *Topic) error { return nil } -func (mts *NullTopicCache) AddUnsafe(_ *Topic) error { +func (c *NullTopicCache) AddUnsafe(_ *Topic) error { return nil } -func (mts *NullTopicCache) Remove(id int) error { +func (c *NullTopicCache) Remove(id int) error { return nil } -func (mts *NullTopicCache) RemoveUnsafe(id int) error { +func (c *NullTopicCache) RemoveUnsafe(id int) error { return nil } -func (mts *NullTopicCache) Flush() { +func (c *NullTopicCache) Flush() { } -func (mts *NullTopicCache) Length() int { +func (c *NullTopicCache) Length() int { return 0 } -func (mts *NullTopicCache) SetCapacity(_ int) { +func (c *NullTopicCache) SetCapacity(_ int) { } -func (mts *NullTopicCache) GetCapacity() int { +func (c *NullTopicCache) GetCapacity() int { return 0 } diff --git a/common/null_user_cache.go b/common/null_user_cache.go index 73279d50..47dd68c0 100644 --- a/common/null_user_cache.go +++ b/common/null_user_cache.go @@ -10,41 +10,41 @@ func NewNullUserCache() *NullUserCache { } // nolint -func (mus *NullUserCache) DeallocOverflow(evictPriority bool) (evicted int) { +func (c *NullUserCache) DeallocOverflow(evictPriority bool) (evicted int) { return 0 } -func (mus *NullUserCache) Get(id int) (*User, error) { +func (c *NullUserCache) Get(id int) (*User, error) { return nil, ErrNoRows } -func (mus *NullUserCache) BulkGet(ids []int) (list []*User) { +func (c *NullUserCache) BulkGet(ids []int) (list []*User) { return make([]*User, len(ids)) } -func (mus *NullUserCache) GetUnsafe(id int) (*User, error) { +func (c *NullUserCache) GetUnsafe(id int) (*User, error) { return nil, ErrNoRows } -func (mus *NullUserCache) Set(_ *User) error { +func (c *NullUserCache) Set(_ *User) error { return nil } -func (mus *NullUserCache) Add(_ *User) error { +func (c *NullUserCache) Add(_ *User) error { return nil } -func (mus *NullUserCache) AddUnsafe(_ *User) error { +func (c *NullUserCache) AddUnsafe(_ *User) error { return nil } -func (mus *NullUserCache) Remove(id int) error { +func (c *NullUserCache) Remove(id int) error { return nil } -func (mus *NullUserCache) RemoveUnsafe(id int) error { +func (c *NullUserCache) RemoveUnsafe(id int) error { return nil } -func (mus *NullUserCache) BulkRemove(ids []int) {} -func (mus *NullUserCache) Flush() { +func (c *NullUserCache) BulkRemove(ids []int) {} +func (c *NullUserCache) Flush() { } -func (mus *NullUserCache) Length() int { +func (c *NullUserCache) Length() int { return 0 } -func (mus *NullUserCache) SetCapacity(_ int) { +func (c *NullUserCache) SetCapacity(_ int) { } -func (mus *NullUserCache) GetCapacity() int { +func (c *NullUserCache) GetCapacity() int { return 0 } diff --git a/common/pages.go b/common/pages.go index cddd3a97..c5e99e2f 100644 --- a/common/pages.go +++ b/common/pages.go @@ -32,10 +32,12 @@ type Header struct { ZoneData interface{} Path string MetaDesc string - StartedAt time.Time - Elapsed1 string - Writer http.ResponseWriter - ExtData ExtData + //OGImage string + //OGDesc string + StartedAt time.Time + Elapsed1 string + Writer http.ResponseWriter + ExtData ExtData } func (header *Header) AddScript(name string) { @@ -255,9 +257,14 @@ type PanelCustomPageEditPage struct { Page *CustomPage } -type PanelTimeGraph struct { +/*type PanelTimeGraph struct { Series []int64 // The counts on the left Labels []int64 // unixtimes for the bottom, gets converted into 1:00, 2:00, etc. with JS +}*/ +type PanelTimeGraph struct { + Series [][]int64 // The counts on the left + Labels []int64 // unixtimes for the bottom, gets converted into 1:00, 2:00, etc. with JS + Legends []string } type PanelAnalyticsItem struct { @@ -267,9 +274,9 @@ type PanelAnalyticsItem struct { type PanelAnalyticsPage struct { *BasePanelPage - PrimaryGraph PanelTimeGraph - ViewItems []PanelAnalyticsItem - TimeRange string + Graph PanelTimeGraph + ViewItems []PanelAnalyticsItem + TimeRange string } type PanelAnalyticsRoutesItem struct { @@ -297,20 +304,27 @@ type PanelAnalyticsAgentsPage struct { type PanelAnalyticsRoutePage struct { *BasePanelPage - Route string - PrimaryGraph PanelTimeGraph - ViewItems []PanelAnalyticsItem - TimeRange string + Route string + Graph PanelTimeGraph + ViewItems []PanelAnalyticsItem + TimeRange string } type PanelAnalyticsAgentPage struct { *BasePanelPage Agent string FriendlyAgent string - PrimaryGraph PanelTimeGraph + Graph PanelTimeGraph TimeRange string } +type PanelAnalyticsDuoPage struct { + *BasePanelPage + ItemList []PanelAnalyticsAgentsItem + Graph PanelTimeGraph + TimeRange string +} + type PanelThemesPage struct { *BasePanelPage PrimaryThemes []*Theme diff --git a/common/search.go b/common/search.go index 6277a746..3b393ef7 100644 --- a/common/search.go +++ b/common/search.go @@ -3,70 +3,131 @@ package common import ( "database/sql" "errors" + "strconv" "github.com/Azareal/Gosora/query_gen" ) -//var RepliesSearch Searcher +var RepliesSearch Searcher type Searcher interface { - Query(q string) ([]int, error) -} - -type ZoneSearcher interface { - QueryZone(q string, zoneID int) ([]int, error) + Query(q string, zones []int) ([]int, error) } // TODO: Implement this // Note: This is slow compared to something like ElasticSearch and very limited type SQLSearcher struct { - queryReplies *sql.Stmt - queryTopics *sql.Stmt - queryZoneReplies *sql.Stmt - queryZoneTopics *sql.Stmt + queryReplies *sql.Stmt + queryTopics *sql.Stmt + queryZone *sql.Stmt } // TODO: Support things other than MySQL +// TODO: Use LIMIT? func NewSQLSearcher(acc *qgen.Accumulator) (*SQLSearcher, error) { if acc.GetAdapter().GetName() != "mysql" { return nil, errors.New("SQLSearcher only supports MySQL at this time") } return &SQLSearcher{ - queryReplies: acc.RawPrepare("SELECT `rid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"), - queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title,content) AGAINST (? IN NATURAL LANGUAGE MODE);"), - queryZoneReplies: acc.RawPrepare("SELECT `rid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE) AND `parentID` = ?;"), - queryZoneTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title,content) AGAINST (? IN NATURAL LANGUAGE MODE) AND `parentID` = ?;"), + queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"), + queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"), + queryZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` = ?;"), }, acc.FirstError() } -func (searcher *SQLSearcher) Query(q string) ([]int, error) { - return nil, nil +func (search *SQLSearcher) queryAll(q string) ([]int, error) { + var ids []int + var run = func(stmt *sql.Stmt, q ...interface{}) error { + rows, err := stmt.Query(q...) + if err == sql.ErrNoRows { + return nil + } else if err != nil { + return err + } + defer rows.Close() - /* - rows, err := stmt.Query(q) + for rows.Next() { + var id int + err := rows.Scan(&id) + if err != nil { + return err + } + ids = append(ids, id) + } + return rows.Err() + } + + err := run(search.queryReplies, q) + if err != nil { + return nil, err + } + err = run(search.queryTopics, q, q) + if err != nil { + return nil, err + } + if len(ids) == 0 { + err = sql.ErrNoRows + } + return ids, err +} + +func (search *SQLSearcher) Query(q string, zones []int) (ids []int, err error) { + if len(zones) == 0 { + return nil, nil + } + var run = func(rows *sql.Rows, err error) error { + if err == sql.ErrNoRows { + return nil + } else if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var id int + err := rows.Scan(&id) + if err != nil { + return err + } + ids = append(ids, id) + } + return rows.Err() + } + + if len(zones) == 1 { + err = run(search.queryZone.Query(q, q, q, zones[0])) + } else { + var zList string + for _, zone := range zones { + zList += strconv.Itoa(zone) + "," + } + zList = zList[:len(zList)-1] + + acc := qgen.NewAcc() + stmt := acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` IN(" + zList + ");") + err := acc.FirstError() if err != nil { return nil, err } - defer rows.Close() - */ -} - -func (searcher *SQLSearcher) QueryZone(q string, zoneID int) ([]int, error) { - return nil, nil + err = run(stmt.Query(q, q, q)) + } + if err != nil { + return nil, err + } + if len(ids) == 0 { + err = sql.ErrNoRows + } + return ids, err } // TODO: Implement this type ElasticSearchSearcher struct { } -func NewElasticSearchSearcher() *ElasticSearchSearcher { - return &ElasticSearchSearcher{} +func NewElasticSearchSearcher() (*ElasticSearchSearcher, error) { + return &ElasticSearchSearcher{}, nil } -func (searcher *ElasticSearchSearcher) Query(q string) ([]int, error) { - return nil, nil -} - -func (searcher *ElasticSearchSearcher) QueryZone(q string, zoneID int) ([]int, error) { +func (search *ElasticSearchSearcher) Query(q string, zones []int) ([]int, error) { return nil, nil } diff --git a/common/template_init.go b/common/template_init.go index 45d19437..52eaedc3 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -359,12 +359,9 @@ func compileTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName string writeTemplate(name, tmpl) } } - /*writeTemplate("profile", profileTmpl) - writeTemplate("forums", forumsTmpl) - writeTemplate("login", loginTmpl) + /*writeTemplate("login", loginTmpl) writeTemplate("register", registerTmpl) writeTemplate("ip_search", ipSearchTmpl) - writeTemplate("account", accountTmpl) writeTemplate("error", errorTmpl)*/ return nil } diff --git a/common/topic.go b/common/topic.go index 4c539a94..73c537c3 100644 --- a/common/topic.go +++ b/common/topic.go @@ -146,6 +146,28 @@ func (row *TopicsRow) WebSockets() *WsTopicsRow { return &WsTopicsRow{row.ID, row.Link, row.Title, row.CreatedBy, row.IsClosed, row.Sticky, row.CreatedAt, row.LastReplyAt, RelativeTime(row.LastReplyAt), row.LastReplyBy, row.LastReplyID, row.ParentID, row.ViewCount, row.PostCount, row.LikeCount, row.AttachCount, row.ClassName, row.Creator.WebSockets(), row.LastUser.WebSockets(), row.ForumName, row.ForumLink} } +// TODO: Stop relying on so many struct types? +// ! Not quite safe as Topic doesn't contain all the data needed to constructs a TopicsRow +func (t *Topic) TopicsRow() *TopicsRow { + lastPage := 1 + var creator *User = nil + contentLines := 1 + var lastUser *User = nil + forumName := "" + forumLink := "" + + return &TopicsRow{t.ID, t.Link, t.Title, t.Content, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, t.LastReplyBy, t.LastReplyID, t.ParentID, t.Status, t.IPAddress, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, lastPage, t.ClassName, t.Data, creator, "", contentLines, lastUser, forumName, forumLink} +} + +// ! Not quite safe as Topic doesn't contain all the data needed to constructs a WsTopicsRow +/*func (t *Topic) WsTopicsRows() *WsTopicsRow { + var creator *User = nil + var lastUser *User = nil + forumName := "" + forumLink := "" + return &WsTopicsRow{t.ID, t.Link, t.Title, t.CreatedBy, t.IsClosed, t.Sticky, t.CreatedAt, t.LastReplyAt, RelativeTime(t.LastReplyAt), t.LastReplyBy, t.LastReplyID, t.ParentID, t.ViewCount, t.PostCount, t.LikeCount, t.AttachCount, t.ClassName, creator, lastUser, forumName, forumLink} +}*/ + type TopicStmts struct { addReplies *sql.Stmt updateLastReply *sql.Stmt diff --git a/common/topic_cache.go b/common/topic_cache.go index 3ae28990..40aadcdd 100644 --- a/common/topic_cache.go +++ b/common/topic_cache.go @@ -9,6 +9,7 @@ import ( type TopicCache interface { Get(id int) (*Topic, error) GetUnsafe(id int) (*Topic, error) + BulkGet(ids []int) (list []*Topic) Set(item *Topic) error Add(item *Topic) error AddUnsafe(item *Topic) error @@ -57,6 +58,17 @@ func (mts *MemoryTopicCache) GetUnsafe(id int) (*Topic, error) { return item, ErrNoRows } +// BulkGet fetches multiple topics by their IDs. Indices without topics will be set to nil, so make sure you check for those, we might want to change this behaviour to make it less confusing. +func (c *MemoryTopicCache) BulkGet(ids []int) (list []*Topic) { + list = make([]*Topic, len(ids)) + c.RLock() + for i, id := range ids { + list[i] = c.items[id] + } + c.RUnlock() + return list +} + // Set overwrites the value of a topic in the cache, whether it's present or not. May return a capacity overflow error. func (mts *MemoryTopicCache) Set(item *Topic) error { mts.Lock() diff --git a/common/topic_list.go b/common/topic_list.go index b1ce9513..596f47a7 100644 --- a/common/topic_list.go +++ b/common/topic_list.go @@ -143,6 +143,7 @@ func (tList *DefaultTopicList) GetListByGroup(group *Group, page int, orderby st } func (tList *DefaultTopicList) GetListByCanSee(canSee []int, page int, orderby string, filterIDs []int) (topicList []*TopicsRow, forumList []Forum, paginator Paginator, err error) { + // TODO: Optimise this by filtering canSee and then fetching the forums? // We need a list of the visible forums for Quick Topic // ? - Would it be useful, if we could post in social groups from /topics/? for _, fid := range canSee { @@ -269,27 +270,27 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter var reqUserList = make(map[int]bool) for rows.Next() { // TODO: Embed Topic structs in TopicsRow to make it easier for us to reuse this work in the topic cache - topicItem := TopicsRow{ID: 0} - err := rows.Scan(&topicItem.ID, &topicItem.Title, &topicItem.Content, &topicItem.CreatedBy, &topicItem.IsClosed, &topicItem.Sticky, &topicItem.CreatedAt, &topicItem.LastReplyAt, &topicItem.LastReplyBy, &topicItem.LastReplyID, &topicItem.ParentID, &topicItem.ViewCount, &topicItem.PostCount, &topicItem.LikeCount) + topic := TopicsRow{ID: 0} + err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.IsClosed, &topic.Sticky, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyBy, &topic.LastReplyID, &topic.ParentID, &topic.ViewCount, &topic.PostCount, &topic.LikeCount) if err != nil { return nil, Paginator{nil, 1, 1}, err } - topicItem.Link = BuildTopicURL(NameToSlug(topicItem.Title), topicItem.ID) + topic.Link = BuildTopicURL(NameToSlug(topic.Title), topic.ID) // TODO: Pass forum to something like topicItem.Forum and use that instead of these two properties? Could be more flexible. - forum := Forums.DirtyGet(topicItem.ParentID) - topicItem.ForumName = forum.Name - topicItem.ForumLink = forum.Link + forum := Forums.DirtyGet(topic.ParentID) + topic.ForumName = forum.Name + topic.ForumLink = forum.Link // TODO: Create a specialised function with a bit less overhead for getting the last page for a post count - _, _, lastPage := PageOffset(topicItem.PostCount, 1, Config.ItemsPerPage) - topicItem.LastPage = lastPage + _, _, lastPage := PageOffset(topic.PostCount, 1, Config.ItemsPerPage) + topic.LastPage = lastPage // TODO: Rename this Vhook to better reflect moving the topic list from /routes/ to /common/ - GetHookTable().Vhook("topics_topic_row_assign", &topicItem, &forum) - topicList = append(topicList, &topicItem) - reqUserList[topicItem.CreatedBy] = true - reqUserList[topicItem.LastReplyBy] = true + GetHookTable().Vhook("topics_topic_row_assign", &topic, &forum) + topicList = append(topicList, &topic) + reqUserList[topic.CreatedBy] = true + reqUserList[topic.LastReplyBy] = true } err = rows.Err() if err != nil { @@ -312,9 +313,9 @@ func (tList *DefaultTopicList) getList(page int, orderby string, argList []inter // Second pass to the add the user data // TODO: Use a pointer to TopicsRow instead of TopicsRow itself? - for _, topicItem := range topicList { - topicItem.Creator = userList[topicItem.CreatedBy] - topicItem.LastUser = userList[topicItem.LastReplyBy] + for _, topic := range topicList { + topic.Creator = userList[topic.CreatedBy] + topic.LastUser = userList[topic.LastReplyBy] } pageList := Paginate(topicCount, Config.ItemsPerPage, 5) diff --git a/common/topic_store.go b/common/topic_store.go index 03291e1d..04e86fc6 100644 --- a/common/topic_store.go +++ b/common/topic_store.go @@ -9,6 +9,7 @@ package common import ( "database/sql" "errors" + "strconv" "strings" "github.com/Azareal/Gosora/query_gen" @@ -27,6 +28,7 @@ type TopicStore interface { DirtyGet(id int) *Topic Get(id int) (*Topic, error) BypassGet(id int) (*Topic, error) + BulkGetMap(ids []int) (list map[int]*Topic, err error) Exists(id int) bool Create(fid int, topicName string, content string, uid int, ipaddress string) (tid int, err error) AddLastTopic(item *Topic, fid int) error // unimplemented @@ -57,7 +59,7 @@ func NewDefaultTopicStore(cache TopicCache) (*DefaultTopicStore, error) { } return &DefaultTopicStore{ cache: cache, - get: acc.Select("topics").Columns("title, content, createdBy, createdAt, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid = ?").Prepare(), + get: acc.Select("topics").Columns("title, content, createdBy, createdAt, lastReplyBy, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid = ?").Prepare(), exists: acc.Select("topics").Columns("tid").Where("tid = ?").Prepare(), topicCount: acc.Count("topics").Prepare(), create: acc.Insert("topics").Columns("parentID, title, content, parsed_content, createdAt, lastReplyAt, lastReplyBy, ipaddress, words, createdBy").Fields("?,?,?,?,UTC_TIMESTAMP(),UTC_TIMESTAMP(),?,?,?,?").Prepare(), @@ -71,7 +73,7 @@ func (mts *DefaultTopicStore) DirtyGet(id int) *Topic { } topic = &Topic{ID: id} - err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) + err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) if err == nil { topic.Link = BuildTopicURL(NameToSlug(topic.Title), id) _ = mts.cache.Add(topic) @@ -88,7 +90,7 @@ func (mts *DefaultTopicStore) Get(id int) (topic *Topic, err error) { } topic = &Topic{ID: id} - err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) + err = mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) if err == nil { topic.Link = BuildTopicURL(NameToSlug(topic.Title), id) _ = mts.cache.Add(topic) @@ -99,14 +101,89 @@ func (mts *DefaultTopicStore) Get(id int) (topic *Topic, err error) { // BypassGet will always bypass the cache and pull the topic directly from the database func (mts *DefaultTopicStore) BypassGet(id int) (*Topic, error) { topic := &Topic{ID: id} - err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) + err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) topic.Link = BuildTopicURL(NameToSlug(topic.Title), id) return topic, err } +// TODO: Avoid duplicating much of this logic from user_store.go +func (s *DefaultTopicStore) BulkGetMap(ids []int) (list map[int]*Topic, err error) { + var idCount = len(ids) + list = make(map[int]*Topic) + if idCount == 0 { + return list, nil + } + + var stillHere []int + sliceList := s.cache.BulkGet(ids) + if len(sliceList) > 0 { + for i, sliceItem := range sliceList { + if sliceItem != nil { + list[sliceItem.ID] = sliceItem + } else { + stillHere = append(stillHere, ids[i]) + } + } + ids = stillHere + } + + // If every user is in the cache, then return immediately + if len(ids) == 0 { + return list, nil + } else if len(ids) == 1 { + topic, err := s.Get(ids[0]) + if err != nil { + return list, err + } + list[topic.ID] = topic + return list, nil + } + + // TODO: Add a function for the qlist stuff + var qlist string + var idList []interface{} + for _, id := range ids { + idList = append(idList, strconv.Itoa(id)) + qlist += "?," + } + qlist = qlist[0 : len(qlist)-1] + + rows, err := qgen.NewAcc().Select("topics").Columns("tid, title, content, createdBy, createdAt, lastReplyBy, lastReplyAt, lastReplyID, is_closed, sticky, parentID, ipaddress, views, postCount, likeCount, attachCount, poll, data").Where("tid IN(" + qlist + ")").Query(idList...) + if err != nil { + return list, err + } + for rows.Next() { + topic := &Topic{} + err := rows.Scan(&topic.ID, &topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) + if err != nil { + return list, err + } + topic.Link = BuildTopicURL(NameToSlug(topic.Title), topic.ID) + s.cache.Set(topic) + list[topic.ID] = topic + } + + // Did we miss any topics? + if idCount > len(list) { + var sidList string + for _, id := range ids { + _, ok := list[id] + if !ok { + sidList += strconv.Itoa(id) + "," + } + } + if sidList != "" { + sidList = sidList[0 : len(sidList)-1] + err = errors.New("Unable to find topics with the following IDs: " + sidList) + } + } + + return list, err +} + func (mts *DefaultTopicStore) Reload(id int) error { topic := &Topic{ID: id} - err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) + err := mts.get.QueryRow(id).Scan(&topic.Title, &topic.Content, &topic.CreatedBy, &topic.CreatedAt, &topic.LastReplyBy, &topic.LastReplyAt, &topic.LastReplyID, &topic.IsClosed, &topic.Sticky, &topic.ParentID, &topic.IPAddress, &topic.ViewCount, &topic.PostCount, &topic.LikeCount, &topic.AttachCount, &topic.Poll, &topic.Data) if err == nil { topic.Link = BuildTopicURL(NameToSlug(topic.Title), id) _ = mts.cache.Set(topic) diff --git a/common/user_store.go b/common/user_store.go index d62b173a..f634b5a3 100644 --- a/common/user_store.go +++ b/common/user_store.go @@ -132,7 +132,6 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User if err != nil { return nil, err } - user.Init() store.cache.Set(user) users = append(users, user) @@ -176,25 +175,23 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro // TODO: Add a function for the qlist stuff var qlist string - var uidList []interface{} + var idList []interface{} for _, id := range ids { - uidList = append(uidList, strconv.Itoa(id)) + idList = append(idList, strconv.Itoa(id)) qlist += "?," } qlist = qlist[0 : len(qlist)-1] - rows, err := qgen.NewAcc().Select("users").Columns("uid, name, group, active, is_super_admin, session, email, avatar, message, url_prefix, url_name, level, score, liked, last_ip, temp_group").Where("uid IN(" + qlist + ")").Query(uidList...) + rows, err := qgen.NewAcc().Select("users").Columns("uid, name, group, active, is_super_admin, session, email, avatar, message, url_prefix, url_name, level, score, liked, last_ip, temp_group").Where("uid IN(" + qlist + ")").Query(idList...) if err != nil { return list, err } - for rows.Next() { user := &User{Loggedin: true} err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) if err != nil { return list, err } - user.Init() mus.cache.Set(user) list[user.ID] = user @@ -211,7 +208,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro } if sidList != "" { sidList = sidList[0 : len(sidList)-1] - err = errors.New("Unable to find the users with the following IDs: " + sidList) + err = errors.New("Unable to find users with the following IDs: " + sidList) } } @@ -233,7 +230,6 @@ func (mus *DefaultUserStore) Reload(id int) error { mus.cache.Remove(id) return err } - user.Init() _ = mus.cache.Set(user) TopicListThaw.Thaw() @@ -276,7 +272,6 @@ func (mus *DefaultUserStore) Create(username string, password string, email stri if err != nil { return 0, err } - lastID, err := res.LastInsertId() return int(lastID), err } diff --git a/docs/updating.md b/docs/updating.md index 8253fd14..0e635c5f 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -39,7 +39,7 @@ If this is the first time you've done an update as the `gosora` user, then you m Replace that name and email with whatever you like. This name and email only applies to the `gosora` user. If you see a zillion modified files pop-up, then that is due to you changing their permissions, don't worry about it. -If you get an access denied error, then you might need to run `chown -R gosora /home/gosora` and `chgrp -R www-data /home/gosora` to fix the ownership of the files. +If you get an access denied error, then you might need to run `chown -R gosora /home/gosora` and `chgrp -R www-data /home/gosora` (with the corresponding user you setup for your instance) to fix the ownership of the files. If you want to manually patch Gosora rather than relying on the above scripts to do it, you'll first want to save your changes with `git stash`, and then, you'll overwrite the files with the new ones with `git pull origin master`, and then, you'll re-apply your changes with `git stash apply`. diff --git a/gen_router.go b/gen_router.go index 8de9be60..1cc02f7e 100644 --- a/gen_router.go +++ b/gen_router.go @@ -487,7 +487,8 @@ var agentMapEnum = map[string]int{ "malformed": 26, "suspicious": 27, "semrush": 28, - "zgrab": 29, + "dotbot": 29, + "zgrab": 30, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -519,7 +520,8 @@ var reverseAgentMapEnum = map[int]string{ 26: "malformed", 27: "suspicious", 28: "semrush", - 29: "zgrab", + 29: "dotbot", + 30: "zgrab", } var markToAgent = map[string]string{ "OPR": "opera", @@ -546,6 +548,7 @@ var markToAgent = map[string]string{ "Twitterbot": "twitter", "Discourse": "discourse", "SemrushBot": "semrush", + "DotBot": "dotbot", "zgrab": "zgrab", } /*var agentRank = map[string]int{ @@ -641,7 +644,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) { var heads string for key, value := range req.Header { for _, vvalue := range value { - heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!!\n" + heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!\n" } } @@ -791,7 +794,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { // TODO: Test this items = items[:0] - r.SuspiciousRequest(req,"Illegal char in UA") + r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break @@ -815,7 +818,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { r.requestLogger.Print("parsed agent: ", agent) } - if common.Dev.SuperDebug { r.requestLogger.Print("os: ", os) r.requestLogger.Printf("items: %+v\n",items) @@ -861,7 +863,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { lang = strings.TrimSpace(lang) lLang := strings.Split(lang,"-") common.DebugDetail("lLang:", lLang) - counters.LangViewCounter.Bump(lLang[0]) + validCode := counters.LangViewCounter.Bump(lLang[0]) + if !validCode { + r.DumpRequest(req,"Invalid ISO Code") + } } else { counters.LangViewCounter.Bump("none") } diff --git a/go.mod b/go.mod index ab8d2cc1..3c88a96f 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,21 @@ require ( github.com/Azareal/gopsutil v0.0.0-20170716174751-0763ca4e911d github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f + github.com/fortytw2/leaktest v1.3.0 // indirect github.com/fsnotify/fsnotify v1.4.7 github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sql-driver/mysql v1.4.0 github.com/gorilla/websocket v1.4.0 github.com/lib/pq v1.0.0 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 + github.com/olivere/elastic v6.2.16+incompatible // indirect github.com/oschwald/geoip2-golang v1.2.1 github.com/oschwald/maxminddb-golang v1.3.0 // indirect github.com/pkg/errors v0.8.0 github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 google.golang.org/appengine v1.2.0 // indirect + gopkg.in/olivere/elastic.v6 v6.2.16 gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/src-d/go-git.v4 v4.7.1 ) diff --git a/go.sum b/go.sum index 3d841ed4..cca094e6 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo= github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw= @@ -45,6 +47,8 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/olivere/elastic v6.2.16+incompatible h1:+mQIHbkADkOgq9tFqnbyg7uNFVV6swGU07EoK1u0nEQ= +github.com/olivere/elastic v6.2.16+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/oschwald/geoip2-golang v1.2.1 h1:3iz+jmeJc6fuCyWeKgtXSXu7+zvkxJbHFXkMT5FVebU= github.com/oschwald/geoip2-golang v1.2.1/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE= github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE= @@ -79,6 +83,8 @@ google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH2 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/olivere/elastic.v6 v6.2.16 h1:SvZm4VE4auXSIWpuG2630o+NA1hcIFFzzcHFQpCsv/w= +gopkg.in/olivere/elastic.v6 v6.2.16/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo= diff --git a/langs/english.json b/langs/english.json index b24a5468..01155052 100644 --- a/langs/english.json +++ b/langs/english.json @@ -191,6 +191,7 @@ "lynx":"Lynx", "semrush":"SemrushBot", + "dotbot":"DotBot", "zgrab":"Zgrab Application Scanner", "suspicious":"Suspicious", "unknown":"Unknown", @@ -829,6 +830,7 @@ "panel_statistics_topic_counts_head":"Topic Counts", "panel_statistics_requests_head":"Requests", + "panel_statistics_time_range_three_months":"3 months", "panel_statistics_time_range_one_month":"1 month", "panel_statistics_time_range_one_week":"1 week", "panel_statistics_time_range_two_days":"2 days", diff --git a/main.go b/main.go index a882e222..999f71b6 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ /* * * Gosora Main File -* Copyright Azareal 2016 - 2019 +* Copyright Azareal 2016 - 2020 * */ // Package main contains the main initialisation logic for Gosora @@ -34,7 +34,6 @@ import ( ) var router *GenRouter -var logWriter = io.MultiWriter(os.Stderr) // TODO: Wrap the globals in here so we can pass pointers to them to subpackages var globs *Globs @@ -144,6 +143,10 @@ func afterDBInit() (err error) { if err != nil { return errors.WithStack(err) } + common.RepliesSearch, err = common.NewSQLSearcher(acc) + if err != nil { + return errors.WithStack(err) + } common.Subscriptions, err = common.NewDefaultSubscriptionStore() if err != nil { return errors.WithStack(err) @@ -227,8 +230,8 @@ func main() { if err != nil { log.Fatal(err) } - logWriter = io.MultiWriter(os.Stderr, f) - log.SetOutput(logWriter) + common.LogWriter = io.MultiWriter(os.Stderr, f) + log.SetOutput(common.LogWriter) log.Print("Running Gosora v" + common.SoftwareVersion.String()) fmt.Println("") diff --git a/patcher/patches.go b/patcher/patches.go index 525f1664..c7dc7ef7 100644 --- a/patcher/patches.go +++ b/patcher/patches.go @@ -25,6 +25,7 @@ func init() { addPatch(11, patch11) addPatch(12, patch12) addPatch(13, patch13) + addPatch(14, patch14) } func patch0(scanner *bufio.Scanner) (err error) { @@ -514,3 +515,20 @@ func patch13(scanner *bufio.Scanner) error { return nil } + +func patch14(scanner *bufio.Scanner) error { + err := execStmt(qgen.Builder.AddKey("topics", "title", tblKey{"title", "fulltext"})) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddKey("topics", "content", tblKey{"content", "fulltext"})) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddKey("replies", "content", tblKey{"content", "fulltext"})) + if err != nil { + return err + } + + return nil +} diff --git a/public/analytics.js b/public/analytics.js index 0fd534a2..71c1d80b 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2,7 +2,9 @@ })*/ -function buildStatsChart(rawLabels, seriesData, timeRange) { +// TODO: Fully localise this +// TODO: Load rawLabels and seriesData dynamically rather than potentially fiddling with nonces for the CSP? +function buildStatsChart(rawLabels, seriesData, timeRange, legendNames) { let labels = []; if(timeRange=="one-month") { labels = ["today","01 days"]; @@ -28,12 +30,20 @@ function buildStatsChart(rawLabels, seriesData, timeRange) { } } labels = labels.reverse() - seriesData = seriesData.reverse(); + for(let i = 0; i < seriesData.length;i++) { + seriesData[i] = seriesData[i].reverse(); + } + let config = { + height: '250px', + }; + if(legendNames.length > 0) config.plugins = [ + Chartist.plugins.legend({ + legendNames: legendNames, + }) + ]; Chartist.Line('.ct_chart', { labels: labels, - series: [seriesData], - }, { - height: '250px', - }); + series: seriesData, + }, config); } \ No newline at end of file diff --git a/public/chartist/chartist-plugin-legend.css b/public/chartist/chartist-plugin-legend.css new file mode 100644 index 00000000..b22127c1 --- /dev/null +++ b/public/chartist/chartist-plugin-legend.css @@ -0,0 +1,75 @@ +.ct-legend { + position: relative; + z-index: 10; + list-style: none; + text-align: center; + } + .ct-legend li { + position: relative; + padding-left: 23px; + margin-right: 10px; + margin-bottom: 3px; + cursor: pointer; + display: inline-block; + } + .ct-legend li:before { + width: 12px; + height: 12px; + position: absolute; + left: 0; + content: ''; + border: 3px solid transparent; + border-radius: 2px; + } + .ct-legend li.inactive:before { + background: transparent; + } + .ct-legend.ct-legend-inside { + position: absolute; + top: 0; + right: 0; + } + .ct-legend.ct-legend-inside li{ + display: block; + margin: 0; + } + .ct-legend .ct-series-0:before { + background-color: #d70206; + border-color: #d70206; + } + .ct-legend .ct-series-1:before { + background-color: #f05b4f; + border-color: #f05b4f; + } + .ct-legend .ct-series-2:before { + background-color: #f4c63d; + border-color: #f4c63d; + } + .ct-legend .ct-series-3:before { + background-color: #d17905; + border-color: #d17905; + } + .ct-legend .ct-series-4:before { + background-color: #453d3f; + border-color: #453d3f; + } + .ct-legend .ct-series-5:before { + background-color: #59922b; + border-color: #59922b; + } + .ct-legend .ct-series-6:before { + background-color: #0544d3; + border-color: #0544d3; + } + + .ct-chart-line-multipleseries .ct-legend .ct-series-0:before { + background-color: #d70206; + border-color: #d70206; + } + .ct-chart-line-multipleseries .ct-legend .ct-series-1:before { + background-color: #f4c63d; + border-color: #f4c63d; + } + .ct-chart-line-multipleseries .ct-legend li.inactive:before { + background: transparent; + } \ No newline at end of file diff --git a/public/chartist/chartist-plugin-legend.min.js b/public/chartist/chartist-plugin-legend.min.js new file mode 100644 index 00000000..17a60709 --- /dev/null +++ b/public/chartist/chartist-plugin-legend.min.js @@ -0,0 +1,2 @@ +//https://github.com/CodeYellowBV/chartist-plugin-legend +!function(e,t){"function"==typeof define&&define.amd?define(["chartist"],function(s){return e.returnExportsGlobal=t(s)}):"object"==typeof exports?module.exports=t(require("chartist")):e["Chartist.plugins.legend"]=t(e.Chartist)}(this,function(e){"use strict";var t={className:"",classNames:!1,removeAll:!1,legendNames:!1,clickable:!0,onClick:null,position:"top"};return e.plugins=e.plugins||{},e.plugins.legend=function(s){function a(e,t){return e-t}if(s&&s.position){if(!("top"===s.position||"bottom"===s.position||s.position instanceof HTMLElement))throw Error("The position you entered is not a valid position");if(s.position instanceof HTMLElement){var i=s.position;delete s.position}}return s=e.extend({},t,s),i&&(s.position=i),function(t){var i=t.container.querySelector(".ct-legend");if(i&&i.parentNode.removeChild(i),s.clickable){var n=t.data.series.map(function(s,a){return"object"!=typeof s&&(s={value:s}),s.className=s.className||t.options.classNames.series+"-"+e.alphaNumerate(a),s});t.data.series=n}var o=document.createElement("ul"),l=t instanceof e.Pie;o.className="ct-legend",t instanceof e.Pie&&o.classList.add("ct-legend-inside"),"string"==typeof s.className&&s.className.length>0&&o.classList.add(s.className),t.options.width&&(o.style.cssText="width: "+t.options.width+"px;margin: 0 auto;");var r=[],c=t.data.series.slice(0),d=t.data.series,p=l&&t.data.labels&&t.data.labels.length;if(p){var u=t.data.labels.slice(0);d=t.data.labels}d=s.legendNames||d;var f=Array.isArray(s.classNames)&&s.classNames.length===d.length;d.forEach(function(e,t){var a=document.createElement("li");a.className="ct-series-"+t,f&&(a.className+=" "+s.classNames[t]),a.setAttribute("data-legend",t),a.textContent=e.name||e,o.appendChild(a)}),t.on("created",function(e){if(s.position instanceof HTMLElement)s.position.insertBefore(o,null);else switch(s.position){case"top":t.container.insertBefore(o,t.container.childNodes[0]);break;case"bottom":t.container.insertBefore(o,null)}}),s.clickable&&o.addEventListener("click",function(e){var i=e.target;if(i.parentNode===o&&i.hasAttribute("data-legend")){e.preventDefault();var n=parseInt(i.getAttribute("data-legend")),l=r.indexOf(n);l>-1?(r.splice(l,1),i.classList.remove("inactive")):s.removeAll?(r.push(n),i.classList.add("inactive")):t.data.series.length>1?(r.push(n),i.classList.add("inactive")):(r=[],Array.prototype.slice.call(o.childNodes).forEach(function(e){e.classList.remove("inactive")}));var d=c.slice(0);if(p)var f=u.slice(0);r.sort(a).reverse(),r.forEach(function(e){d.splice(e,1),p&&f.splice(e,1)}),s.onClick&&s.onClick(t,e),t.data.series=d,p&&(t.data.labels=f),t.update()}})}},e.plugins.legend}); \ No newline at end of file diff --git a/public/global.js b/public/global.js index 1d90c360..69c752c8 100644 --- a/public/global.js +++ b/public/global.js @@ -392,8 +392,7 @@ function mainInit(){ $(".link_label").click(function(event) { event.preventDefault(); - let forSelect = $(this).attr("data-for"); - let linkSelect = $('#'+forSelect); + let linkSelect = $('#'+$(this).attr("data-for")); if(!linkSelect.hasClass("link_opened")) { event.stopPropagation(); linkSelect.addClass("link_opened"); @@ -415,9 +414,7 @@ function mainInit(){ this.outerHTML = Template_paginator({PageList: pageList, Page: page, LastPage: lastPage}); ok = true; }); - if(!ok) { - $(Template_paginator({PageList: pageList, Page: page, LastPage: lastPage})).insertAfter("#topic_list"); - } + if(!ok) $(Template_paginator({PageList: pageList, Page: page, LastPage: lastPage})).insertAfter("#topic_list"); } function rebindPaginator() { @@ -496,6 +493,41 @@ function mainInit(){ if (document.getElementById("topicsItemList")!==null) rebindPaginator(); if (document.getElementById("forumItemList")!==null) rebindPaginator(); + // TODO: Show a search button when JS is disabled? + $(".widget_search_input").keypress(function(e) { + if (e.keyCode != '13') return; + event.preventDefault(); + // TODO: Take mostviewed into account + let url = "//"+window.location.host+window.location.pathname; + let urlParams = new URLSearchParams(window.location.search); + urlParams.set("q",this.value); + let q = "?"; + for(let item of urlParams.entries()) q += item[0]+"="+item[1]+"&"; + if(q.length>1) q = q.slice(0,-1); + + // TODO: Try to de-duplicate some of these fetch calls + fetch(url+q+"&js=1", {credentials: "same-origin"}) + .then((resp) => resp.json()) + .then((data) => { + if(!"Topics" in data) throw("no Topics in data"); + let topics = data["Topics"]; + + // TODO: Fix the data race where the function hasn't been loaded yet + let out = ""; + for(let i = 0; i < topics.length;i++) out += Template_topics_topic(topics[i]); + $(".topic_list").html(out); + + let obj = {Title: document.title, Url: url+q}; + history.pushState(obj, obj.Title, obj.Url); + rebuildPaginator(data.LastPage); + rebindPaginator(); + }).catch((ex) => { + console.log("Unable to get script '"+url+q+"&js=1"+"'"); + console.log("ex: ", ex); + console.trace(); + }); + }); + $(".open_edit").click((event) => { event.preventDefault(); $('.hide_on_edit').addClass("edit_opened"); diff --git a/query_gen/builder.go b/query_gen/builder.go index 0be74b44..1e8de785 100644 --- a/query_gen/builder.go +++ b/query_gen/builder.go @@ -116,6 +116,10 @@ func (build *builder) AddIndex(table string, iname string, colname string) (stmt return build.prepare(build.adapter.AddIndex("", table, iname, colname)) } +func (build *builder) AddKey(table string, column string, key DBTableKey) (stmt *sql.Stmt, err error) { + return build.prepare(build.adapter.AddKey("", table, column, key)) +} + func (build *builder) SimpleInsert(table string, columns string, fields string) (stmt *sql.Stmt, err error) { return build.prepare(build.adapter.SimpleInsert("", table, columns, fields)) } diff --git a/query_gen/mssql.go b/query_gen/mssql.go index 2c9453f5..e9d76b55 100644 --- a/query_gen/mssql.go +++ b/query_gen/mssql.go @@ -136,7 +136,7 @@ func (adapter *MssqlAdapter) parseColumn(column DBTableColumn) (col DBTableColum // TODO: Test this, not sure if some things work // TODO: Add support for keys -func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTableColumn,key *DBTableKey) (string, error) { +func (adapter *MssqlAdapter) AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error) { if table == "" { return "", errors.New("You need a name for this table") } @@ -162,6 +162,18 @@ func (adapter *MssqlAdapter) AddIndex(name string, table string, iname string, c return "", errors.New("not implemented") } +// TODO: Implement this +// TODO: Test to make sure everything works here +func (adapter *MssqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if column == "" { + return "", errors.New("You need a name for the column") + } + return "", errors.New("not implemented") +} + func (adapter *MssqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") diff --git a/query_gen/mysql.go b/query_gen/mysql.go index 387cccf8..4bd91372 100644 --- a/query_gen/mysql.go +++ b/query_gen/mysql.go @@ -213,6 +213,22 @@ func (adapter *MysqlAdapter) AddIndex(name string, table string, iname string, c return querystr, nil } +// TODO: Test to make sure everything works here +// Only supports FULLTEXT right now +func (adapter *MysqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if key.Type != "fulltext" { + return "", errors.New("Only fulltext is supported by AddKey right now") + } + querystr := "ALTER TABLE `" + table + "` ADD FULLTEXT(`" + column + "`)" + + // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator + adapter.pushStatement(name, "add-key", querystr) + return querystr, nil +} + func (adapter *MysqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") @@ -689,11 +705,23 @@ func (adapter *MysqlAdapter) buildLimit(limit string) (querystr string) { func (adapter *MysqlAdapter) buildJoinColumns(columns string) (querystr string) { for _, column := range processColumns(columns) { + // TODO: Move the stirng and number logic to processColumns? + // TODO: Error if [0] doesn't exist + firstChar := column.Left[0] + if firstChar == '\'' { + column.Type = "string" + } else { + _, err := strconv.Atoi(string(firstChar)) + if err == nil { + column.Type = "number" + } + } + // Escape the column names, just in case we've used a reserved keyword var source = column.Left if column.Table != "" { source = "`" + column.Table + "`.`" + source + "`" - } else if column.Type != "function" { + } else if column.Type != "function" && column.Type != "number" && column.Type != "substitute" && column.Type != "string" { source = "`" + source + "`" } diff --git a/query_gen/pgsql.go b/query_gen/pgsql.go index 57c00c85..f668be03 100644 --- a/query_gen/pgsql.go +++ b/query_gen/pgsql.go @@ -113,7 +113,7 @@ func (adapter *PgsqlAdapter) CreateTable(name string, table string, charset stri } // TODO: Implement this -func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTableColumn,key *DBTableKey) (string, error) { +func (adapter *PgsqlAdapter) AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error) { if table == "" { return "", errors.New("You need a name for this table") } @@ -135,6 +135,18 @@ func (adapter *PgsqlAdapter) AddIndex(name string, table string, iname string, c return "", errors.New("not implemented") } +// TODO: Implement this +// TODO: Test to make sure everything works here +func (adapter *PgsqlAdapter) AddKey(name string, table string, column string, key DBTableKey) (string, error) { + if table == "" { + return "", errors.New("You need a name for this table") + } + if column == "" { + return "", errors.New("You need a name for the column") + } + return "", errors.New("not implemented") +} + // TODO: Test this // ! We need to get the last ID out of this somehow, maybe add returning to every query? Might require some sort of wrapper over the sql statements func (adapter *PgsqlAdapter) SimpleInsert(name string, table string, columns string, fields string) (string, error) { diff --git a/query_gen/querygen.go b/query_gen/querygen.go index b1f28ff3..5d9483ea 100644 --- a/query_gen/querygen.go +++ b/query_gen/querygen.go @@ -110,6 +110,7 @@ type Adapter interface { // TODO: Test this AddColumn(name string, table string, column DBTableColumn, key *DBTableKey) (string, error) AddIndex(name string, table string, iname string, colname string) (string, error) + AddKey(name string, table string, column string, key DBTableKey) (string, error) SimpleInsert(name string, table string, columns string, fields string) (string, error) SimpleUpdate(up *updatePrebuilder) (string, error) SimpleUpdateSelect(up *updatePrebuilder) (string, error) // ! Experimental diff --git a/query_gen/utils.go b/query_gen/utils.go index 52309279..e53af5ef 100644 --- a/query_gen/utils.go +++ b/query_gen/utils.go @@ -2,17 +2,17 @@ * * Query Generator Library * WIP Under Construction -* Copyright Azareal 2017 - 2019 +* Copyright Azareal 2017 - 2020 * */ package qgen -//import "fmt" import ( "os" "strings" ) +// TODO: Add support for numbers and strings? func processColumns(colstr string) (columns []DBColumn) { if colstr == "" { return columns diff --git a/router_gen/main.go b/router_gen/main.go index 7a997960..df2bc689 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -222,6 +222,7 @@ func main() { "malformed", "suspicious", "semrush", + "dotbot", "zgrab", } @@ -257,6 +258,7 @@ func main() { "Discourse", "SemrushBot", + "DotBot", "zgrab", } @@ -287,6 +289,7 @@ func main() { "Discourse": "discourse", "SemrushBot": "semrush", + "DotBot": "dotbot", "zgrab": "zgrab", } @@ -433,7 +436,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) { var heads string for key, value := range req.Header { for _, vvalue := range value { - heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!!\n" + heads += "Header '" + common.SanitiseSingleLine(key) + "': " + common.SanitiseSingleLine(vvalue) + "!\n" } } @@ -583,7 +586,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { // TODO: Test this items = items[:0] - r.SuspiciousRequest(req,"Illegal char in UA") + r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break @@ -607,7 +610,6 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { r.requestLogger.Print("parsed agent: ", agent) } - if common.Dev.SuperDebug { r.requestLogger.Print("os: ", os) r.requestLogger.Printf("items: %+v\n",items) @@ -653,7 +655,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { lang = strings.TrimSpace(lang) lLang := strings.Split(lang,"-") common.DebugDetail("lLang:", lLang) - counters.LangViewCounter.Bump(lLang[0]) + validCode := counters.LangViewCounter.Bump(lLang[0]) + if !validCode { + r.DumpRequest(req,"Invalid ISO Code") + } } else { counters.LangViewCounter.Bump("none") } diff --git a/routes/panel/analytics.go b/routes/panel/analytics.go index fbe5830e..73b92594 100644 --- a/routes/panel/analytics.go +++ b/routes/panel/analytics.go @@ -30,6 +30,13 @@ func analyticsTimeRange(rawTimeRange string) (timeRange AnalyticsTimeRange, err timeRange.Range = "six-hours" switch rawTimeRange { + // This might be pushing it, we might want to come up with a more efficient scheme for dealing with large timeframes like this + case "three-months": + timeRange.Quantity = 90 + timeRange.Unit = "day" + timeRange.Slices = 30 + timeRange.SliceWidth = 60 * 60 * 24 * 3 + timeRange.Range = "three-months" case "one-month": timeRange.Quantity = 30 timeRange.Unit = "day" @@ -59,7 +66,6 @@ func analyticsTimeRange(rawTimeRange string) (timeRange AnalyticsTimeRange, err timeRange.Slices = 24 timeRange.Range = "twelve-hours" case "six-hours", "": - timeRange.Range = "six-hours" default: return timeRange, errors.New("Unknown time range") } @@ -89,7 +95,6 @@ func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64 if err != nil { return viewMap, err } - var unixCreatedAt = createdAt.Unix() // TODO: Bulk log this if common.Dev.SuperDebug { @@ -97,7 +102,6 @@ func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64 log.Print("createdAt: ", createdAt) log.Print("unixCreatedAt: ", unixCreatedAt) } - for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -113,7 +117,6 @@ func PreAnalyticsDetail(w http.ResponseWriter, r *http.Request, user *common.Use if ferr != nil { return nil, ferr } - basePage.AddSheet("chartist/chartist.min.css") basePage.AddScript("chartist/chartist.min.js") basePage.AddScript("analytics.js") @@ -125,7 +128,6 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co if ferr != nil { return ferr } - timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return common.LocalError(err.Error(), w, r, user) @@ -138,7 +140,6 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -150,7 +151,7 @@ func AnalyticsViews(w http.ResponseWriter, r *http.Request, user common.User) co viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range} @@ -162,7 +163,6 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use if ferr != nil { return ferr } - timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return common.LocalError(err.Error(), w, r, user) @@ -175,7 +175,6 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -187,7 +186,7 @@ func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.Use viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) pi := common.PanelAnalyticsRoutePage{basePage, common.SanitiseSingleLine(route), graph, viewItems, timeRange.Range} @@ -199,13 +198,11 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use if ferr != nil { return ferr } - timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return common.LocalError(err.Error(), w, r, user) } revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) - // ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff agent = common.SanitiseSingleLine(agent) @@ -215,7 +212,6 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -225,7 +221,7 @@ func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user common.Use for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) friendlyAgent, ok := phrases.GetUserAgentPhrase(agent) @@ -259,7 +255,6 @@ func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user common.Use if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -269,7 +264,7 @@ func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user common.Use for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) forum, err := common.Forums.Get(fid) @@ -286,7 +281,6 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us if ferr != nil { return ferr } - timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return common.LocalError(err.Error(), w, r, user) @@ -300,7 +294,6 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -310,7 +303,7 @@ func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.Us for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) friendlySystem, ok := phrases.GetOSPhrase(system) @@ -350,7 +343,7 @@ func AnalyticsLanguageViews(w http.ResponseWriter, r *http.Request, user common. for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) friendlyLang, ok := phrases.GetHumanLangPhrase(lang) @@ -379,7 +372,6 @@ func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user common. if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -389,9 +381,8 @@ func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user common. for _, value := range revLabelList { viewList = append(viewList, viewMap[value]) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) - pi := common.PanelAnalyticsAgentPage{basePage, common.SanitiseSingleLine(domain), "", graph, timeRange.Range} return renderTemplate("panel_analytics_referrer_views", w, r, basePage.Header, &pi) } @@ -412,7 +403,6 @@ func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) c if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -424,9 +414,8 @@ func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) c viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) - pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range} return renderTemplate("panel_analytics_topics", w, r, basePage.Header, &pi) } @@ -447,7 +436,6 @@ func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user common.User) co if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) @@ -459,13 +447,39 @@ func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user common.User) co viewList = append(viewList, viewMap[value]) viewItems = append(viewItems, common.PanelAnalyticsItem{Time: value, Count: viewMap[value]}) } - graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + graph := common.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList} common.DebugLogf("graph: %+v\n", graph) - pi := common.PanelAnalyticsPage{basePage, graph, viewItems, timeRange.Range} return renderTemplate("panel_analytics_posts", w, r, basePage.Header, &pi) } +/*func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[int64]int64, error) { + defer rows.Close() + for rows.Next() { + var count int64 + var createdAt time.Time + err := rows.Scan(&count, &createdAt) + if err != nil { + return viewMap, err + } + + var unixCreatedAt = createdAt.Unix() + // TODO: Bulk log this + if common.Dev.SuperDebug { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { + if unixCreatedAt > value { + viewMap[value] += count + break + } + } + } + return viewMap, rows.Err() +}*/ + func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) { nameMap := make(map[string]int) defer rows.Close() @@ -476,7 +490,6 @@ func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) { if err != nil { return nameMap, err } - // TODO: Bulk log this if common.Dev.SuperDebug { log.Print("count: ", count) @@ -487,6 +500,46 @@ func analyticsRowsToNameMap(rows *sql.Rows) (map[string]int, error) { return nameMap, rows.Err() } +func analyticsRowsToDuoMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[string]map[int64]int64, map[string]int, error) { + vMap := make(map[string]map[int64]int64) + nameMap := make(map[string]int) + defer rows.Close() + for rows.Next() { + var count int64 + var name string + var createdAt time.Time + err := rows.Scan(&count, &name, &createdAt) + if err != nil { + return vMap, nameMap, err + } + + // TODO: Bulk log this + var unixCreatedAt = createdAt.Unix() + if common.Dev.SuperDebug { + log.Print("count: ", count) + log.Print("name: ", name) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + vvMap, ok := vMap[name] + if !ok { + vvMap = make(map[int64]int64) + for key, val := range viewMap { + vvMap[key] = val + } + vMap[name] = vvMap + } + for _, value := range labelList { + if unixCreatedAt > value { + vvMap[value] += count + break + } + } + nameMap[name] += int(count) + } + return vMap, nameMap, rows.Err() +} + func AnalyticsForums(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics") if ferr != nil { @@ -501,7 +554,6 @@ func AnalyticsForums(w http.ResponseWriter, r *http.Request, user common.User) c if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - forumMap, err := analyticsRowsToNameMap(rows) if err != nil { return common.InternalError(err, w, r) @@ -543,7 +595,6 @@ func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) c if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - routeMap, err := analyticsRowsToNameMap(rows) if err != nil { return common.InternalError(err, w, r) @@ -562,26 +613,78 @@ func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) c return renderTemplate("panel_analytics_routes", w, r, basePage.Header, &pi) } +type OVItem struct { + name string + count int + viewMap map[int64]int64 +} + +// Trialling multi-series charts func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { - basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics") + basePage, ferr := PreAnalyticsDetail(w, r, &user) if ferr != nil { return ferr } + basePage.AddScript("chartist/chartist-plugin-legend.min.js") + basePage.AddSheet("chartist/chartist-plugin-legend.css") + timeRange, err := analyticsTimeRange(r.FormValue("timeRange")) if err != nil { return common.LocalError(err.Error(), w, r, user) } + revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange) - rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, browser").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() + rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count, browser, createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - - agentMap, err := analyticsRowsToNameMap(rows) + vMap, agentMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap) if err != nil { return common.InternalError(err, w, r) } + // Order the map + var ovList []OVItem + for name, viewMap := range vMap { + var totcount int + for _, count := range viewMap { + totcount += int(count) + } + ovList = append(ovList, OVItem{name, totcount, viewMap}) + } + // Use bubble sort for now as there shouldn't be too many items + for i := 0; i < len(ovList)-1; i++ { + for j := 0; j < len(ovList)-1; j++ { + if ovList[j].count > ovList[j+1].count { + temp := ovList[j] + ovList[j] = ovList[j+1] + ovList[j+1] = temp + } + } + } + + var vList [][]int64 + var legendList []string + var i int + for _, ovitem := range ovList { + var viewList []int64 + for _, value := range revLabelList { + viewList = append(viewList, ovitem.viewMap[value]) + } + vList = append(vList, viewList) + lName, ok := phrases.GetUserAgentPhrase(ovitem.name) + if !ok { + lName = ovitem.name + } + legendList = append(legendList, lName) + if i >= 6 { + break + } + i++ + } + graph := common.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList} + common.DebugLogf("graph: %+v\n", graph) + // TODO: Sort this slice var agentItems []common.PanelAnalyticsAgentsItem for agent, count := range agentMap { @@ -596,7 +699,7 @@ func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) c }) } - pi := common.PanelAnalyticsAgentsPage{basePage, agentItems, timeRange.Range} + pi := common.PanelAnalyticsDuoPage{basePage, agentItems, graph, timeRange.Range} return renderTemplate("panel_analytics_agents", w, r, basePage.Header, &pi) } @@ -614,7 +717,6 @@ func AnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User) if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - osMap, err := analyticsRowsToNameMap(rows) if err != nil { return common.InternalError(err, w, r) @@ -652,7 +754,6 @@ func AnalyticsLanguages(w http.ResponseWriter, r *http.Request, user common.User if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - langMap, err := analyticsRowsToNameMap(rows) if err != nil { return common.InternalError(err, w, r) @@ -691,7 +792,6 @@ func AnalyticsReferrers(w http.ResponseWriter, r *http.Request, user common.User if err != nil && err != sql.ErrNoRows { return common.InternalError(err, w, r) } - refMap, err := analyticsRowsToNameMap(rows) if err != nil { return common.InternalError(err, w, r) diff --git a/routes/topic_list.go b/routes/topic_list.go index 8546f4fa..df4c783a 100644 --- a/routes/topic_list.go +++ b/routes/topic_list.go @@ -1,6 +1,7 @@ package routes import ( + "database/sql" "log" "net/http" "strconv" @@ -10,72 +11,6 @@ import ( "github.com/Azareal/Gosora/common/phrases" ) -// TODO: Implement search -func TopicList(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError { - group, err := common.Groups.Get(user.Group) - if err != nil { - log.Printf("Group #%d doesn't exist despite being used by common.User #%d", user.Group, user.ID) - return common.LocalError("Something weird happened", w, r, user) - } - - // Get the current page - page, _ := strconv.Atoi(r.FormValue("page")) - sfids := r.FormValue("fids") - var fids []int - if sfids != "" { - for _, sfid := range strings.Split(sfids, ",") { - fid, err := strconv.Atoi(sfid) - if err != nil { - return common.LocalError("Invalid fid", w, r, user) - } - fids = append(fids, fid) - } - } - - // TODO: Pass a struct back rather than passing back so many variables - var topicList []*common.TopicsRow - var forumList []common.Forum - var paginator common.Paginator - if user.IsSuperAdmin { - topicList, forumList, paginator, err = common.TopicList.GetList(page, "", fids) - } else { - topicList, forumList, paginator, err = common.TopicList.GetListByGroup(group, page, "", fids) - } - if err != nil { - return common.InternalError(err, w, r) - } - // ! Need an inline error not a page level error - if len(topicList) == 0 { - return common.NotFound(w, r, header) - } - - // TODO: Reduce the amount of boilerplate here - if r.FormValue("js") == "1" { - outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON() - if err != nil { - return common.InternalError(err, w, r) - } - w.Write(outBytes) - return nil - } - - header.Title = phrases.GetTitlePhrase("topics") - header.Zone = "topics" - header.Path = "/topics/" - header.MetaDesc = header.Settings["meta_desc"].(string) - if len(fids) == 1 { - forum, err := common.Forums.Get(fids[0]) - if err != nil { - return common.LocalError("Invalid fid forum", w, r, user) - } - header.Title = forum.Name - header.ZoneID = forum.ID - } - - pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{"lastupdated", false}, paginator} - return renderTemplate("topics", w, r, header, pi) -} - func wsTopicList(topicList []*common.TopicsRow, lastPage int) *common.WsTopicList { wsTopicList := make([]*common.WsTopicsRow, len(topicList)) for i, topicRow := range topicList { @@ -84,7 +19,16 @@ func wsTopicList(topicList []*common.TopicsRow, lastPage int) *common.WsTopicLis return &common.WsTopicList{wsTopicList, lastPage} } +func TopicList(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError { + return TopicListCommon(w, r, user, header, "lastupdated") +} + func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header) common.RouteError { + return TopicListCommon(w, r, user, header, "mostviewed") +} + +// TODO: Implement search +func TopicListCommon(w http.ResponseWriter, r *http.Request, user common.User, header *common.Header, torder string) common.RouteError { header.Title = phrases.GetTitlePhrase("topics") header.Zone = "topics" header.Path = "/topics/" @@ -118,10 +62,111 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use } } - // TODO: Pass a struct back rather than passing back so many variables + //(t *Topic) WsTopicsRows() *WsTopicsRow + // TODO: Allow multiple forums in searches + // TODO: Simplify this block after initially landing search var topicList []*common.TopicsRow var forumList []common.Forum var paginator common.Paginator + q := r.FormValue("q") + if q != "" { + var canSee []int + if user.IsSuperAdmin { + canSee, err = common.Forums.GetAllVisibleIDs() + if err != nil { + return common.InternalError(err, w, r) + } + } else { + canSee = group.CanSee + } + + var cfids []int + if len(fids) > 0 { + var inSlice = func(haystack []int, needle int) bool { + for _, item := range haystack { + if needle == item { + return true + } + } + return false + } + for _, fid := range fids { + if inSlice(canSee, fid) { + forum := common.Forums.DirtyGet(fid) + if forum.Name != "" && forum.Active && (forum.ParentType == "" || forum.ParentType == "forum") { + // TODO: Add a hook here for plugin_guilds? + cfids = append(cfids, fid) + } + } + } + } else { + cfids = canSee + } + + tids, err := common.RepliesSearch.Query(q, cfids) + if err != nil && err != sql.ErrNoRows { + return common.InternalError(err, w, r) + } + //fmt.Printf("tids %+v\n", tids) + // TODO: Handle the case where there aren't any items... + // TODO: Add a BulkGet method which returns a slice? + tMap, err := common.Topics.BulkGetMap(tids) + if err != nil { + return common.InternalError(err, w, r) + } + var reqUserList = make(map[int]bool) + for _, topic := range tMap { + reqUserList[topic.CreatedBy] = true + reqUserList[topic.LastReplyBy] = true + topicList = append(topicList, topic.TopicsRow()) + } + //fmt.Printf("reqUserList %+v\n", reqUserList) + + // Convert the user ID map to a slice, then bulk load the users + var idSlice = make([]int, len(reqUserList)) + var i int + for userID := range reqUserList { + idSlice[i] = userID + i++ + } + + // TODO: What if a user is deleted via the Control Panel? + //fmt.Printf("idSlice %+v\n", idSlice) + userList, err := common.Users.BulkGetMap(idSlice) + if err != nil { + return nil // TODO: Implement this! + } + + // TODO: De-dupe this logic in common/topic_list.go? + for _, topic := range topicList { + topic.Link = common.BuildTopicURL(common.NameToSlug(topic.Title), topic.ID) + // TODO: Pass forum to something like topic.Forum and use that instead of these two properties? Could be more flexible. + forum := common.Forums.DirtyGet(topic.ParentID) + topic.ForumName = forum.Name + topic.ForumLink = forum.Link + + // TODO: Create a specialised function with a bit less overhead for getting the last page for a post count + _, _, lastPage := common.PageOffset(topic.PostCount, 1, common.Config.ItemsPerPage) + topic.LastPage = lastPage + topic.Creator = userList[topic.CreatedBy] + topic.LastUser = userList[topic.LastReplyBy] + } + + // TODO: Reduce the amount of boilerplate here + if r.FormValue("js") == "1" { + outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON() + if err != nil { + return common.InternalError(err, w, r) + } + w.Write(outBytes) + return nil + } + + pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{torder, false}, paginator} + return renderTemplate("topics", w, r, header, pi) + } + + // TODO: Pass a struct back rather than passing back so many variables if user.IsSuperAdmin { topicList, forumList, paginator, err = common.TopicList.GetList(page, "most-viewed", fids) } else { @@ -135,7 +180,6 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use return common.NotFound(w, r, header) } - //MarshalJSON() ([]byte, error) // TODO: Reduce the amount of boilerplate here if r.FormValue("js") == "1" { outBytes, err := wsTopicList(topicList, paginator.LastPage).MarshalJSON() @@ -146,6 +190,6 @@ func TopicListMostViewed(w http.ResponseWriter, r *http.Request, user common.Use return nil } - pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{"mostviewed", false}, paginator} + pi := common.TopicListPage{header, topicList, forumList, common.Config.DefaultForum, common.TopicListSort{torder, false}, paginator} return renderTemplate("topics", w, r, header, pi) } diff --git a/schema/mssql/query_replies.sql b/schema/mssql/query_replies.sql index 96b5decc..d10dbc64 100644 --- a/schema/mssql/query_replies.sql +++ b/schema/mssql/query_replies.sql @@ -14,5 +14,6 @@ CREATE TABLE [replies] ( [words] int DEFAULT 1 not null, [actionType] nvarchar (20) DEFAULT '' not null, [poll] int DEFAULT 0 not null, - primary key([rid]) + primary key([rid]), + fulltext key([content]) ); \ No newline at end of file diff --git a/schema/mssql/query_topics.sql b/schema/mssql/query_topics.sql index fec0b0b6..019078ea 100644 --- a/schema/mssql/query_topics.sql +++ b/schema/mssql/query_topics.sql @@ -20,5 +20,6 @@ CREATE TABLE [topics] ( [css_class] nvarchar (100) DEFAULT '' not null, [poll] int DEFAULT 0 not null, [data] nvarchar (200) DEFAULT '' not null, - primary key([tid]) + primary key([tid]), + fulltext key([content]) ); \ No newline at end of file diff --git a/schema/mysql/query_replies.sql b/schema/mysql/query_replies.sql index 65dbe737..4341ade3 100644 --- a/schema/mysql/query_replies.sql +++ b/schema/mysql/query_replies.sql @@ -14,5 +14,6 @@ CREATE TABLE `replies` ( `words` int DEFAULT 1 not null, `actionType` varchar(20) DEFAULT '' not null, `poll` int DEFAULT 0 not null, - primary key(`rid`) + primary key(`rid`), + fulltext key(`content`) ) CHARSET=utf8mb4 COLLATE utf8mb4_general_ci; \ No newline at end of file diff --git a/schema/mysql/query_topics.sql b/schema/mysql/query_topics.sql index 3110996b..563ee144 100644 --- a/schema/mysql/query_topics.sql +++ b/schema/mysql/query_topics.sql @@ -20,5 +20,6 @@ CREATE TABLE `topics` ( `css_class` varchar(100) DEFAULT '' not null, `poll` int DEFAULT 0 not null, `data` varchar(200) DEFAULT '' not null, - primary key(`tid`) + primary key(`tid`), + fulltext key(`content`) ) CHARSET=utf8mb4 COLLATE utf8mb4_general_ci; \ No newline at end of file diff --git a/schema/pgsql/query_replies.sql b/schema/pgsql/query_replies.sql index d15948e3..3d36452f 100644 --- a/schema/pgsql/query_replies.sql +++ b/schema/pgsql/query_replies.sql @@ -14,5 +14,6 @@ CREATE TABLE "replies" ( `words` int DEFAULT 1 not null, `actionType` varchar (20) DEFAULT '' not null, `poll` int DEFAULT 0 not null, - primary key(`rid`) + primary key(`rid`), + fulltext key(`content`) ); \ No newline at end of file diff --git a/schema/pgsql/query_topics.sql b/schema/pgsql/query_topics.sql index 2b82c054..9fff0b9e 100644 --- a/schema/pgsql/query_topics.sql +++ b/schema/pgsql/query_topics.sql @@ -20,5 +20,6 @@ CREATE TABLE "topics" ( `css_class` varchar (100) DEFAULT '' not null, `poll` int DEFAULT 0 not null, `data` varchar (200) DEFAULT '' not null, - primary key(`tid`) + primary key(`tid`), + fulltext key(`content`) ); \ No newline at end of file diff --git a/templates/panel_analytics_agent_views.html b/templates/panel_analytics_agent_views.html index b1c33117..1751d531 100644 --- a/templates/panel_analytics_agent_views.html +++ b/templates/panel_analytics_agent_views.html @@ -15,13 +15,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_agents.html b/templates/panel_analytics_agents.html index 0975f416..22fad81a 100644 --- a/templates/panel_analytics_agents.html +++ b/templates/panel_analytics_agents.html @@ -10,6 +10,9 @@ +
+
+
{{range .ItemList}}
@@ -20,4 +23,5 @@
+{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_forum_views.html b/templates/panel_analytics_forum_views.html index f262849b..ec11212a 100644 --- a/templates/panel_analytics_forum_views.html +++ b/templates/panel_analytics_forum_views.html @@ -15,13 +15,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_lang_views.html b/templates/panel_analytics_lang_views.html index ea98e48d..06086ee1 100644 --- a/templates/panel_analytics_lang_views.html +++ b/templates/panel_analytics_lang_views.html @@ -16,12 +16,5 @@ +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_posts.html b/templates/panel_analytics_posts.html index dbc4bd19..ae874572 100644 --- a/templates/panel_analytics_posts.html +++ b/templates/panel_analytics_posts.html @@ -26,13 +26,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_referrer_views.html b/templates/panel_analytics_referrer_views.html index a74c3f22..941b3820 100644 --- a/templates/panel_analytics_referrer_views.html +++ b/templates/panel_analytics_referrer_views.html @@ -15,13 +15,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_route_views.html b/templates/panel_analytics_route_views.html index 4387629b..549df9ee 100644 --- a/templates/panel_analytics_route_views.html +++ b/templates/panel_analytics_route_views.html @@ -26,13 +26,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_script.html b/templates/panel_analytics_script.html new file mode 100644 index 00000000..47243110 --- /dev/null +++ b/templates/panel_analytics_script.html @@ -0,0 +1,14 @@ + + \ No newline at end of file diff --git a/templates/panel_analytics_system_views.html b/templates/panel_analytics_system_views.html index b7dfc6f7..ecdcb33c 100644 --- a/templates/panel_analytics_system_views.html +++ b/templates/panel_analytics_system_views.html @@ -15,13 +15,5 @@ - +{{template "panel_analytics_script.html" . }} {{template "footer.html" . }} diff --git a/templates/panel_analytics_time_range.html b/templates/panel_analytics_time_range.html index 170494f2..e688d705 100644 --- a/templates/panel_analytics_time_range.html +++ b/templates/panel_analytics_time_range.html @@ -1,4 +1,5 @@ +
- +
diff --git a/templates/widget_search_and_filter.html b/templates/widget_search_and_filter.html index f61551e9..9e9c7a7e 100644 --- a/templates/widget_search_and_filter.html +++ b/templates/widget_search_and_filter.html @@ -1,5 +1,5 @@
{{range .Forums}} diff --git a/themes/nox/public/panel.css b/themes/nox/public/panel.css index c8c76f12..54c8357c 100644 --- a/themes/nox/public/panel.css +++ b/themes/nox/public/panel.css @@ -125,6 +125,7 @@ button, .formbutton, .panel_right_button:not(.has_inner_button), #panel_users .p padding: 16px; padding-bottom: 0px; padding-left: 0px; + margin-bottom: 10px; } .colstack_graph_holder .ct-label { color: rgb(195,195,195); @@ -299,6 +300,10 @@ button, .formbutton, .panel_right_button:not(.has_inner_button), #panel_users .p .wtype_about .w_about, .wtype_simple .w_simple, .wtype_wol .w_wol, .wtype_default .w_default { display: block; } +.wtext, .rwtext { + width: 100%; + height: 80px; +} #panel_debug .grid_stat:not(.grid_stat_head) { margin-bottom: 5px;