From 24cef434399c2a2e57d097387945d6418dc5af80 Mon Sep 17 00:00:00 2001 From: Azareal Date: Fri, 21 Feb 2020 21:42:21 +1000 Subject: [PATCH] fix search not working properly fix missing title index properly add search tests roll back accidental duplicate indices You will need to run the updater / patcher for this commit. --- cmd/query_gen/main.go | 7 +++-- cmd/query_gen/tables.go | 1 + common/search.go | 46 ++++++++++++++++++++++++------- misc_test.go | 34 +++++++++++++++++++++++ patcher/patches.go | 33 ++++++++++++++++++++-- templates/topic_c_poll_input.html | 2 +- 6 files changed, 107 insertions(+), 16 deletions(-) diff --git a/cmd/query_gen/main.go b/cmd/query_gen/main.go index 0d43a1b7..5c66fd90 100644 --- a/cmd/query_gen/main.go +++ b/cmd/query_gen/main.go @@ -81,9 +81,10 @@ func seedTables(a qgen.Adapter) error { qgen.Install.AddIndex("attachments", "path", "path") qgen.Install.AddIndex("activity_stream_matches", "watcher", "watcher") // TODO: Remove these keys to save space when Elasticsearch is active? - qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false}) - qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false}) - qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false}) + //qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false}) + //qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false}) + //qgen.Install.AddKey("topics", "title,content", tK{"title,content", "fulltext", "", false}) + //qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false}) qgen.Install.SimpleInsert("sync", "last_update", "UTC_TIMESTAMP()") qgen.Install.SimpleInsert("settings", "name, content, type, constraints", "'activation_type','1','list','1-3'") diff --git a/cmd/query_gen/tables.go b/cmd/query_gen/tables.go index b4027652..7127198f 100644 --- a/cmd/query_gen/tables.go +++ b/cmd/query_gen/tables.go @@ -287,6 +287,7 @@ func createTables(adapter qgen.Adapter) (err error) { }, []tblKey{ tblKey{"tid", "primary", "", false}, + tblKey{"title", "fulltext", "", false}, tblKey{"content", "fulltext", "", false}, }, ) diff --git a/common/search.go b/common/search.go index 2abd8722..069f64c3 100644 --- a/common/search.go +++ b/common/search.go @@ -5,7 +5,7 @@ import ( "errors" "strconv" - "github.com/Azareal/Gosora/query_gen" + qgen "github.com/Azareal/Gosora/query_gen" ) var RepliesSearch Searcher @@ -19,7 +19,10 @@ type Searcher interface { type SQLSearcher struct { queryReplies *sql.Stmt queryTopics *sql.Stmt - queryZone *sql.Stmt + queryRepliesZone *sql.Stmt + queryTopicsZone *sql.Stmt + //queryZone *sql.Stmt + fuzzyZone *sql.Stmt } // TODO: Support things other than MySQL @@ -29,9 +32,12 @@ func NewSQLSearcher(acc *qgen.Accumulator) (*SQLSearcher, error) { return nil, errors.New("SQLSearcher only supports MySQL at this time") } return &SQLSearcher{ - queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"), - queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"), - queryZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` = ?;"), + queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE)"), + queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)"), + queryRepliesZone: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE) AND tid=?"), + queryTopicsZone: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE (MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)) AND parentID=?"), + //queryZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title`=? OR (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE)) OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID`=?"), + fuzzyZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title` LIKE ? OR `topics`.`content` LIKE ? OR `replies`.`content` LIKE ?) AND `topics`.`parentID`=?"), }, acc.FirstError() } @@ -76,9 +82,9 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) { return nil, nil } run := func(rows *sql.Rows, err error) error { - if err == sql.ErrNoRows { + /*if err == sql.ErrNoRows { return nil - } else if err != nil { + } else */if err != nil { return err } defer rows.Close() @@ -95,7 +101,12 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) { } if len(zones) == 1 { - err = run(s.queryZone.Query(q, q, q, zones[0])) + //err = run(s.queryZone.Query(q, q, q, q, q,q, zones[0])) + err = run(s.queryRepliesZone.Query(q, zones[0])) + if err != nil { + return nil, err + } + err = run(s.queryTopicsZone.Query(q, q,zones[0])) } else { var zList string for _, zone := range zones { @@ -104,12 +115,27 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) { zList = zList[:len(zList)-1] acc := qgen.NewAcc() - stmt := acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` IN(" + zList + ");") + /*stmt := acc.RawPrepare("SELECT topics.tid FROM `topics` INNER JOIN `replies` ON topics.tid = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) OR `topics`.`title`=? OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID` IN(" + zList + ")") + err = acc.FirstError() + if err != nil { + return nil, err + }*/ + stmt := acc.RawPrepare("SELECT tid FROM topics WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE)) AND parentID IN(" + zList + ")") err = acc.FirstError() if err != nil { return nil, err } - err = run(stmt.Query(q, q, q)) + err = run(stmt.Query(q, q)) + if err != nil { + return nil, err + } + stmt = acc.RawPrepare("SELECT tid FROM replies WHERE MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) AND tid IN(" + zList + ")") + err = acc.FirstError() + if err != nil { + return nil, err + } + err = run(stmt.Query(q)) + //err = run(stmt.Query(q, q, q, q, q, q)) } if err != nil { return nil, err diff --git a/misc_test.go b/misc_test.go index 21dee227..897f14c4 100644 --- a/misc_test.go +++ b/misc_test.go @@ -1298,6 +1298,40 @@ func TestPolls(t *testing.T) { recordMustNotExist(t, err, "poll 1 should no longer exist") } +func TestSearch(t *testing.T) { + miscinit(t) + if !c.PluginsInited { + c.InitPlugins() + } + + title := "search" + body := "bab bab bab bab" + q := "search" + tid, err := c.Topics.Create(2, title, body, 1, "") + expectNilErr(t, err) + + tids, err := c.RepliesSearch.Query(q, []int{2}) + fmt.Printf("tids: %+v\n", tids) + expectNilErr(t, err) + expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids))) + + topic, err := c.Topics.Get(tids[0]) + expectNilErr(t, err) + expect(t, topic.ID == tid, fmt.Sprintf("topic.ID should be %d not %d", tid, topic.ID)) + expect(t, topic.Title == title, fmt.Sprintf("topic.Title should be %s not %s", title, topic.Title)) + + tids, err = c.RepliesSearch.Query(q, []int{1,2}) + fmt.Printf("tids: %+v\n", tids) + expectNilErr(t, err) + expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids))) + + q = "bab" + tids, err = c.RepliesSearch.Query(q, []int{1,2}) + fmt.Printf("tids: %+v\n", tids) + expectNilErr(t, err) + expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids))) +} + func TestProfileReplyStore(t *testing.T) { miscinit(t) if !c.PluginsInited { diff --git a/patcher/patches.go b/patcher/patches.go index 6cc1b53a..e806f525 100644 --- a/patcher/patches.go +++ b/patcher/patches.go @@ -48,6 +48,7 @@ func init() { addPatch(28, patch28) addPatch(29, patch29) addPatch(30, patch30) + addPatch(31, patch31) } func patch0(scanner *bufio.Scanner) (err error) { @@ -528,7 +529,7 @@ func patch13(scanner *bufio.Scanner) error { } func patch14(scanner *bufio.Scanner) error { - err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false})) + /*err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false})) if err != nil { return err } @@ -539,7 +540,7 @@ func patch14(scanner *bufio.Scanner) error { err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false})) if err != nil { return err - } + }*/ return nil } @@ -863,3 +864,31 @@ func patch30(scanner *bufio.Scanner) error { } return execStmt(qgen.Builder.SetDefaultColumn("users", "last_ip", "varchar", "")) } + +func patch31(scanner *bufio.Scanner) error { + err := execStmt(qgen.Builder.RemoveIndex("topics", "title")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.RemoveIndex("topics", "content")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.RemoveIndex("replies", "content")) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false})) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddKey("topics", "content", tK{"content", "fulltext", "", false})) + if err != nil { + return err + } + err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false})) + if err != nil { + return err + } + return nil +} diff --git a/templates/topic_c_poll_input.html b/templates/topic_c_poll_input.html index 4d1f781f..e3444a2e 100644 --- a/templates/topic_c_poll_input.html +++ b/templates/topic_c_poll_input.html @@ -1,5 +1,5 @@
- +
\ No newline at end of file