fix search not working properly

fix missing title index properly
add search tests
roll back accidental duplicate indices

You will need to run the updater / patcher for this commit.

cmd/query_gen/main.go

@@ -81,9 +81,10 @@ func seedTables(a qgen.Adapter) error {
 	qgen.Install.AddIndex("attachments", "path", "path")
 	qgen.Install.AddIndex("activity_stream_matches", "watcher", "watcher")
 	// TODO: Remove these keys to save space when Elasticsearch is active?
-	qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false})
+	//qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false})
-	qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false})
+	//qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false})
-	qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false})
+	//qgen.Install.AddKey("topics", "title,content", tK{"title,content", "fulltext", "", false})
+	//qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false})
 
 	qgen.Install.SimpleInsert("sync", "last_update", "UTC_TIMESTAMP()")
 	qgen.Install.SimpleInsert("settings", "name, content, type, constraints", "'activation_type','1','list','1-3'")

cmd/query_gen/tables.go

@@ -287,6 +287,7 @@ func createTables(adapter qgen.Adapter) (err error) {
 		},
 		[]tblKey{
 			tblKey{"tid", "primary", "", false},
+			tblKey{"title", "fulltext", "", false},
 			tblKey{"content", "fulltext", "", false},
 		},
 	)

common/search.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"strconv"
 
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
 )
 
 var RepliesSearch Searcher
@@ -19,7 +19,10 @@ type Searcher interface {
 type SQLSearcher struct {
 	queryReplies *sql.Stmt
 	queryTopics  *sql.Stmt
-	queryZone    *sql.Stmt
+	queryRepliesZone *sql.Stmt
+	queryTopicsZone *sql.Stmt
+	//queryZone    *sql.Stmt
+	fuzzyZone *sql.Stmt
 }
 
 // TODO: Support things other than MySQL
@@ -29,9 +32,12 @@ func NewSQLSearcher(acc *qgen.Accumulator) (*SQLSearcher, error) {
 		return nil, errors.New("SQLSearcher only supports MySQL at this time")
 	}
 	return &SQLSearcher{
-		queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
+		queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE)"),
-		queryTopics:  acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
+		queryTopics:  acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)"),
-		queryZone:    acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` = ?;"),
+		queryRepliesZone: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE) AND tid=?"),
+		queryTopicsZone:  acc.RawPrepare("SELECT `tid` FROM `topics` WHERE (MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)) AND parentID=?"),
+		//queryZone:    acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title`=? OR (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE)) OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID`=?"),
+		fuzzyZone:    acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title` LIKE ? OR `topics`.`content` LIKE ? OR `replies`.`content` LIKE ?) AND `topics`.`parentID`=?"),
 	}, acc.FirstError()
 }
 
@@ -76,9 +82,9 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
 		return nil, nil
 	}
 	run := func(rows *sql.Rows, err error) error {
-		if err == sql.ErrNoRows {
+		/*if err == sql.ErrNoRows {
 			return nil
-		} else if err != nil {
+		} else */if err != nil {
 			return err
 		}
 		defer rows.Close()
@@ -95,7 +101,12 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
 	}
 
 	if len(zones) == 1 {
-		err = run(s.queryZone.Query(q, q, q, zones[0]))
+		//err = run(s.queryZone.Query(q, q, q, q, q,q, zones[0]))
+		err = run(s.queryRepliesZone.Query(q, zones[0]))
+		if err != nil {
+			return nil, err
+		}
+		err = run(s.queryTopicsZone.Query(q, q,zones[0]))
 	} else {
 		var zList string
 		for _, zone := range zones {
@@ -104,12 +115,27 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
 		zList = zList[:len(zList)-1]
 
 		acc := qgen.NewAcc()
-		stmt := acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` IN(" + zList + ");")
+		/*stmt := acc.RawPrepare("SELECT topics.tid FROM `topics` INNER JOIN `replies` ON topics.tid = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) OR `topics`.`title`=? OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID` IN(" + zList + ")")
+		err = acc.FirstError()
+		if err != nil {
+			return nil, err
+		}*/
+		stmt := acc.RawPrepare("SELECT tid FROM topics WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE)) AND parentID IN(" + zList + ")")
 		err = acc.FirstError()
 		if err != nil {
 			return nil, err
 		}
-		err = run(stmt.Query(q, q, q))
+		err = run(stmt.Query(q, q))
+		if err != nil {
+			return nil, err
+		}
+		stmt = acc.RawPrepare("SELECT tid FROM replies WHERE MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) AND tid IN(" + zList + ")")
+		err = acc.FirstError()
+		if err != nil {
+			return nil, err
+		}
+		err = run(stmt.Query(q))
+		//err = run(stmt.Query(q, q, q, q, q, q))
 	}
 	if err != nil {
 		return nil, err

misc_test.go

@@ -1298,6 +1298,40 @@ func TestPolls(t *testing.T) {
 	recordMustNotExist(t, err, "poll 1 should no longer exist")
 }
 
+func TestSearch(t *testing.T) {
+	miscinit(t)
+	if !c.PluginsInited {
+		c.InitPlugins()
+	}
+
+	title := "search"
+	body := "bab bab bab bab"
+	q := "search"
+	tid, err := c.Topics.Create(2, title, body, 1, "")
+	expectNilErr(t, err)
+
+	tids, err := c.RepliesSearch.Query(q, []int{2})
+	fmt.Printf("tids: %+v\n", tids)
+	expectNilErr(t, err)
+	expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
+
+	topic, err := c.Topics.Get(tids[0])
+	expectNilErr(t, err)
+	expect(t, topic.ID == tid, fmt.Sprintf("topic.ID should be %d not %d", tid, topic.ID))
+	expect(t, topic.Title == title, fmt.Sprintf("topic.Title should be %s not %s", title, topic.Title))
+
+	tids, err = c.RepliesSearch.Query(q, []int{1,2})
+	fmt.Printf("tids: %+v\n", tids)
+	expectNilErr(t, err)
+	expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
+
+	q = "bab"
+	tids, err = c.RepliesSearch.Query(q, []int{1,2})
+	fmt.Printf("tids: %+v\n", tids)
+	expectNilErr(t, err)
+	expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
+}
+
 func TestProfileReplyStore(t *testing.T) {
 	miscinit(t)
 	if !c.PluginsInited {

patcher/patches.go

@@ -48,6 +48,7 @@ func init() {
 	addPatch(28, patch28)
 	addPatch(29, patch29)
 	addPatch(30, patch30)
+	addPatch(31, patch31)
 }
 
 func patch0(scanner *bufio.Scanner) (err error) {
@@ -528,7 +529,7 @@ func patch13(scanner *bufio.Scanner) error {
 }
 
 func patch14(scanner *bufio.Scanner) error {
-	err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
+	/*err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
 	if err != nil {
 		return err
 	}
@@ -539,7 +540,7 @@ func patch14(scanner *bufio.Scanner) error {
 	err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false}))
 	if err != nil {
 		return err
-	}
+	}*/
 
 	return nil
 }
@@ -863,3 +864,31 @@ func patch30(scanner *bufio.Scanner) error {
 	}
 	return execStmt(qgen.Builder.SetDefaultColumn("users", "last_ip", "varchar", ""))
 }
+
+func patch31(scanner *bufio.Scanner) error {
+	err := execStmt(qgen.Builder.RemoveIndex("topics", "title"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.RemoveIndex("topics", "content"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.RemoveIndex("replies", "content"))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddKey("topics", "content", tK{"content", "fulltext", "", false}))
+	if err != nil {
+		return err
+	}
+	err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false}))
+	if err != nil {
+		return err
+	}
+	return nil
+}