a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix search not working properly 

fix missing title index properly
add search tests
roll back accidental duplicate indices

You will need to run the updater / patcher for this commit.
This commit is contained in:
Azareal 2020-02-21 21:42:21 +10:00
parent 3d1fd807d3
commit 24cef43439
6 changed files with 107 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/query_gen/main.go
View File

@ -81,9 +81,10 @@ func seedTables(a qgen.Adapter) error {
qgen.Install.AddIndex("attachments", "path", "path")
qgen.Install.AddIndex("activity_stream_matches", "watcher", "watcher")
// TODO: Remove these keys to save space when Elasticsearch is active?
qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false})
qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false})
qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false})
//qgen.Install.AddKey("topics", "title", tK{"title", "fulltext", "", false})
//qgen.Install.AddKey("topics", "content", tK{"content", "fulltext", "", false})
//qgen.Install.AddKey("topics", "title,content", tK{"title,content", "fulltext", "", false})
//qgen.Install.AddKey("replies", "content", tK{"content", "fulltext", "", false})
qgen.Install.SimpleInsert("sync", "last_update", "UTC_TIMESTAMP()")
qgen.Install.SimpleInsert("settings", "name, content, type, constraints", "'activation_type','1','list','1-3'")

1
cmd/query_gen/tables.go
View File

@ -287,6 +287,7 @@ func createTables(adapter qgen.Adapter) (err error) {
},
[]tblKey{
tblKey{"tid", "primary", "", false},
tblKey{"title", "fulltext", "", false},
tblKey{"content", "fulltext", "", false},
},
)

46
common/search.go
View File

@ -5,7 +5,7 @@ import (
"errors"
"strconv"
"github.com/Azareal/Gosora/query_gen"
qgen "github.com/Azareal/Gosora/query_gen"
)
var RepliesSearch Searcher
@ -19,7 +19,10 @@ type Searcher interface {
type SQLSearcher struct {
queryReplies *sql.Stmt
queryTopics *sql.Stmt
queryZone *sql.Stmt
queryRepliesZone *sql.Stmt
queryTopicsZone *sql.Stmt
//queryZone *sql.Stmt
fuzzyZone *sql.Stmt
}
// TODO: Support things other than MySQL
@ -29,9 +32,12 @@ func NewSQLSearcher(acc *qgen.Accumulator) (*SQLSearcher, error) {
return nil, errors.New("SQLSearcher only supports MySQL at this time")
}
return &SQLSearcher{
queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(content) AGAINST (? IN NATURAL LANGUAGE MODE);"),
queryZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` = ?;"),
queryReplies: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE)"),
queryTopics: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)"),
queryRepliesZone: acc.RawPrepare("SELECT `tid` FROM `replies` WHERE MATCH(content) AGAINST (? IN BOOLEAN MODE) AND tid=?"),
queryTopicsZone: acc.RawPrepare("SELECT `tid` FROM `topics` WHERE (MATCH(title) AGAINST (? IN BOOLEAN MODE) OR MATCH(content) AGAINST (? IN BOOLEAN MODE)) AND parentID=?"),
//queryZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title`=? OR (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE)) OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID`=?"),
fuzzyZone: acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (`topics`.`title` LIKE ? OR `topics`.`content` LIKE ? OR `replies`.`content` LIKE ?) AND `topics`.`parentID`=?"),
}, acc.FirstError()
}
@ -76,9 +82,9 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
return nil, nil
}
run := func(rows *sql.Rows, err error) error {
if err == sql.ErrNoRows {
/*if err == sql.ErrNoRows {
return nil
} else if err != nil {
} else */if err != nil {
return err
}
defer rows.Close()
@ -95,7 +101,12 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
}
if len(zones) == 1 {
err = run(s.queryZone.Query(q, q, q, zones[0]))
//err = run(s.queryZone.Query(q, q, q, q, q,q, zones[0]))
err = run(s.queryRepliesZone.Query(q, zones[0]))
if err != nil {
return nil, err
}
err = run(s.queryTopicsZone.Query(q, q,zones[0]))
} else {
var zList string
for _, zone := range zones {
@ -104,12 +115,27 @@ func (s *SQLSearcher) Query(q string, zones []int) (ids []int, err error) {
zList = zList[:len(zList)-1]
acc := qgen.NewAcc()
stmt := acc.RawPrepare("SELECT `topics`.`tid` FROM `topics` INNER JOIN `replies` ON `topics`.`tid` = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`topics`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE) OR MATCH(`replies`.`content`) AGAINST (? IN NATURAL LANGUAGE MODE)) AND `topics`.`parentID` IN(" + zList + ");")
/*stmt := acc.RawPrepare("SELECT topics.tid FROM `topics` INNER JOIN `replies` ON topics.tid = `replies`.`tid` WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) OR `topics`.`title`=? OR `topics`.`content`=? OR `replies`.`content`=?) AND `topics`.`parentID` IN(" + zList + ")")
err = acc.FirstError()
if err != nil {
return nil, err
}*/
stmt := acc.RawPrepare("SELECT tid FROM topics WHERE (MATCH(`topics`.`title`) AGAINST (? IN BOOLEAN MODE) OR MATCH(`topics`.`content`) AGAINST (? IN BOOLEAN MODE)) AND parentID IN(" + zList + ")")
err = acc.FirstError()
if err != nil {
return nil, err
}
err = run(stmt.Query(q, q, q))
err = run(stmt.Query(q, q))
if err != nil {
return nil, err
}
stmt = acc.RawPrepare("SELECT tid FROM replies WHERE MATCH(`replies`.`content`) AGAINST (? IN BOOLEAN MODE) AND tid IN(" + zList + ")")
err = acc.FirstError()
if err != nil {
return nil, err
}
err = run(stmt.Query(q))
//err = run(stmt.Query(q, q, q, q, q, q))
}
if err != nil {
return nil, err

34
misc_test.go
View File

@ -1298,6 +1298,40 @@ func TestPolls(t *testing.T) {
recordMustNotExist(t, err, "poll 1 should no longer exist")
}
func TestSearch(t *testing.T) {
miscinit(t)
if !c.PluginsInited {
c.InitPlugins()
}
title := "search"
body := "bab bab bab bab"
q := "search"
tid, err := c.Topics.Create(2, title, body, 1, "")
expectNilErr(t, err)
tids, err := c.RepliesSearch.Query(q, []int{2})
fmt.Printf("tids: %+v\n", tids)
expectNilErr(t, err)
expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
topic, err := c.Topics.Get(tids[0])
expectNilErr(t, err)
expect(t, topic.ID == tid, fmt.Sprintf("topic.ID should be %d not %d", tid, topic.ID))
expect(t, topic.Title == title, fmt.Sprintf("topic.Title should be %s not %s", title, topic.Title))
tids, err = c.RepliesSearch.Query(q, []int{1,2})
fmt.Printf("tids: %+v\n", tids)
expectNilErr(t, err)
expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
q = "bab"
tids, err = c.RepliesSearch.Query(q, []int{1,2})
fmt.Printf("tids: %+v\n", tids)
expectNilErr(t, err)
expect(t, len(tids) == 1, fmt.Sprintf("len(tids) should be 1 not %d", len(tids)))
}
func TestProfileReplyStore(t *testing.T) {
miscinit(t)
if !c.PluginsInited {

33
patcher/patches.go
View File

@ -48,6 +48,7 @@ func init() {
addPatch(28, patch28)
addPatch(29, patch29)
addPatch(30, patch30)
addPatch(31, patch31)
}
func patch0(scanner *bufio.Scanner) (err error) {
@ -528,7 +529,7 @@ func patch13(scanner *bufio.Scanner) error {
}
func patch14(scanner *bufio.Scanner) error {
err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
/*err := execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
if err != nil {
return err
}
@ -539,7 +540,7 @@ func patch14(scanner *bufio.Scanner) error {
err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false}))
if err != nil {
return err
}
}*/
return nil
}
@ -863,3 +864,31 @@ func patch30(scanner *bufio.Scanner) error {
}
return execStmt(qgen.Builder.SetDefaultColumn("users", "last_ip", "varchar", ""))
}
func patch31(scanner *bufio.Scanner) error {
err := execStmt(qgen.Builder.RemoveIndex("topics", "title"))
if err != nil {
return err
}
err = execStmt(qgen.Builder.RemoveIndex("topics", "content"))
if err != nil {
return err
}
err = execStmt(qgen.Builder.RemoveIndex("replies", "content"))
if err != nil {
return err
}
err = execStmt(qgen.Builder.AddKey("topics", "title", tK{"title", "fulltext", "", false}))
if err != nil {
return err
}
err = execStmt(qgen.Builder.AddKey("topics", "content", tK{"content", "fulltext", "", false}))
if err != nil {
return err
}
err = execStmt(qgen.Builder.AddKey("replies", "content", tK{"content", "fulltext", "", false}))
if err != nil {
return err
}
return nil
}