From f789a8498e2542e4d907798e38efcf58cf2dfb8b Mon Sep 17 00:00:00 2001 From: Azareal Date: Thu, 2 Jan 2020 15:28:36 +1000 Subject: [PATCH] Add PollIPCutoff configuration setting. Add tests for qgen processWhere and qgen mysql buildWhere. Shorten some things. --- common/site.go | 4 +++ docs/configuration.md | 6 ++-- query_gen/mssql.go | 16 +++++----- query_gen/mysql.go | 16 +++++----- query_gen/utils_test.go | 65 +++++++++++++++++++++++++++++++++++++++++ tickloop.go | 24 ++++++++++----- 6 files changed, 104 insertions(+), 27 deletions(-) create mode 100644 query_gen/utils_test.go diff --git a/common/site.go b/common/site.go index fc7ff49f..6bf723ef 100644 --- a/common/site.go +++ b/common/site.go @@ -93,6 +93,7 @@ type config struct { ServerCount int LastIPCutoff int // Currently just -1, non--1, but will accept the number of months a user's last IP should be retained for in the future before being purged. Please note that the other two cutoffs below operate off the numbers of days instead. PostIPCutoff int + PollIPCutoff int LogPruneCutoff int DisableLiveTopicList bool @@ -228,6 +229,9 @@ func ProcessConfig() (err error) { if Config.LastIPCutoff > 12 { Config.LastIPCutoff = 12 } + if Config.PollIPCutoff == 0 { + Config.PollIPCutoff = -1 // Default cutoff + } if Config.NoEmbed { DefaultParseSettings.NoEmbed = true } diff --git a/docs/configuration.md b/docs/configuration.md index c747af9e..ce2ba3a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -86,9 +86,11 @@ ServerCount - The number of instances you're running. This setting is currently LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature. -PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. Default: 0 +PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. -LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0 +PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently -1 and -1 disables this feature. + +LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false diff --git a/query_gen/mssql.go b/query_gen/mssql.go index 0b2ce577..c6f41b1d 100644 --- a/query_gen/mssql.go +++ b/query_gen/mssql.go @@ -236,7 +236,7 @@ func (a *MssqlAdapter) SimpleInsert(name, table, cols, fields string) (string, e } // ! DEPRECATED -func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string, fields string) (string, error) { +func (a *MssqlAdapter) SimpleReplace(name, table, columns, fields string) (string, error) { log.Print("In SimpleReplace") key, ok := a.keys[table] if !ok { @@ -271,7 +271,7 @@ func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string, return a.SimpleUpsert(name, table, columns, fields, "key = "+keyValue) } -func (a *MssqlAdapter) SimpleUpsert(name string, table string, columns string, fields string, where string) (string, error) { +func (a *MssqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") } @@ -975,14 +975,12 @@ func (a *MssqlAdapter) simpleJoin(name string, ins DBInsert, sel DBJoin, joinTyp } q += source + alias + "," } - // Remove the trailing comma q = q[0 : len(q)-1] q += " FROM [" + sel.Table1 + "] " + joinType + " JOIN [" + sel.Table2 + "] ON " for _, j := range processJoiner(sel.Joiners) { q += "[" + j.LeftTable + "].[" + j.LeftColumn + "] " + j.Operator + " [" + j.RightTable + "].[" + j.RightColumn + "] AND " } - // Remove the trailing AND q = q[0 : len(q)-4] // Add support for BETWEEN x.x @@ -1166,17 +1164,17 @@ func _gen_mssql() (err error) { } // Internal methods, not exposed in the interface -func (a *MssqlAdapter) pushStatement(name string, stype string, querystr string) { +func (a *MssqlAdapter) pushStatement(name, stype, q string) { if name == "" { return } - a.Buffer[name] = DBStmt{querystr, stype} + a.Buffer[name] = DBStmt{q, stype} a.BufferOrder = append(a.BufferOrder, name) } -func (a *MssqlAdapter) stringyType(ctype string) bool { - ctype = strings.ToLower(ctype) - return ctype == "char" || ctype == "varchar" || ctype == "datetime" || ctype == "text" || ctype == "nvarchar" +func (a *MssqlAdapter) stringyType(ct string) bool { + ct = strings.ToLower(ct) + return ct == "char" || ct == "varchar" || ct == "datetime" || ct == "text" || ct == "nvarchar" } type SetPrimaryKeys interface { diff --git a/query_gen/mysql.go b/query_gen/mysql.go index 69622c4f..443f27ca 100644 --- a/query_gen/mysql.go +++ b/query_gen/mysql.go @@ -338,11 +338,11 @@ func (a *MysqlAdapter) SimpleReplace(name, table, columns, fields string) (strin for _, field := range processFields(fields) { q += field.Name + "," } - q = q[0 : len(q)-1] + q = q[0 : len(q)-1] + ")" // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator - a.pushStatement(name, "replace", q+")") - return q + ")", nil + a.pushStatement(name, "replace", q) + return q, nil } func (a *MysqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) { @@ -968,15 +968,15 @@ func _gen_mysql() (err error) { } // Internal methods, not exposed in the interface -func (a *MysqlAdapter) pushStatement(name string, stype string, querystr string) { +func (a *MysqlAdapter) pushStatement(name, stype, q string) { if name == "" { return } - a.Buffer[name] = DBStmt{querystr, stype} + a.Buffer[name] = DBStmt{q, stype} a.BufferOrder = append(a.BufferOrder, name) } -func (a *MysqlAdapter) stringyType(ctype string) bool { - ctype = strings.ToLower(ctype) - return ctype == "varchar" || ctype == "tinytext" || ctype == "text" || ctype == "mediumtext" || ctype == "longtext" || ctype == "char" || ctype == "datetime" || ctype == "timestamp" || ctype == "time" || ctype == "date" +func (a *MysqlAdapter) stringyType(ct string) bool { + ct = strings.ToLower(ct) + return ct == "varchar" || ct == "tinytext" || ct == "text" || ct == "mediumtext" || ct == "longtext" || ct == "char" || ct == "datetime" || ct == "timestamp" || ct == "time" || ct == "date" } diff --git a/query_gen/utils_test.go b/query_gen/utils_test.go new file mode 100644 index 00000000..b844592f --- /dev/null +++ b/query_gen/utils_test.go @@ -0,0 +1,65 @@ +package qgen + +import ( + "strings" + "testing" +) + +type MT struct { + Type int + Contents string +} + +func expectTokens(t *testing.T, whs []DBWhere, tokens ...MT) { + i := 0 + for _, wh := range whs { + for _, expr := range wh.Expr { + if expr.Type != tokens[i].Type || expr.Contents != tokens[i].Contents { + t.Fatalf("token mismatch: %+v - %+v\n", expr, tokens[i]) + } + i++ + } + } +} + +func TestProcessWhere(t *testing.T) { + whs := processWhere("uid = ?") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"}) + whs = processWhere("uid = 1") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"}) + whs = processWhere("uid = 0") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"}) + whs = processWhere("uid = '1'") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"}) + whs = processWhere("uid = ''") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""}) + whs = processWhere("uid = '") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""}) + + whs = processWhere("uid=?") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"}) + whs = processWhere("uid=1") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"}) + whs = processWhere("uid=0") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"}) + whs = processWhere("uid='1'") + expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"}) + + whs = processWhere("uid") + expectTokens(t, whs, MT{TokenColumn, "uid"}) +} + +func TestMySQLBuildWhere(t *testing.T) { + a := &MysqlAdapter{Name: "mysql", Buffer: make(map[string]DBStmt)} + reap := func(wh, ex string) { + sb := &strings.Builder{} + a.buildWhere(wh, sb) + res := sb.String() + if res != ex { + t.Fatalf("build where mismatch: '%+v' - '%+v'\n", ex, res) + } + } + reap("uid = 0", " WHERE `uid`= 0 ") + reap("uid = '0'", " WHERE `uid`= '0'") + reap("uid=0", " WHERE `uid`= 0 ") +} diff --git a/tickloop.go b/tickloop.go index b8205308..f8188f4c 100644 --- a/tickloop.go +++ b/tickloop.go @@ -1,15 +1,15 @@ package main import ( + "database/sql" "errors" "log" - "time" "strconv" "sync/atomic" - "database/sql" + "time" c "github.com/Azareal/Gosora/common" - "github.com/Azareal/Gosora/query_gen" + qgen "github.com/Azareal/Gosora/query_gen" ) // TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there? @@ -59,7 +59,7 @@ func tickLoop(thumbChan chan bool) { if lastDaily < low { dailies() } - + // TODO: Write tests for these // Run this goroutine once every half second halfSecondTicker := time.NewTicker(time.Second / 2) @@ -172,7 +172,7 @@ func dailies() { if c.Config.LogPruneCutoff > -1 { f := func(tbl string) { - _, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run() + _, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt", c.Config.LogPruneCutoff, "day").Run() if err != nil { c.LogError(err) } @@ -184,7 +184,7 @@ func dailies() { if c.Config.PostIPCutoff > -1 { // TODO: Use unixtime to remove this MySQLesque logic? f := func(tbl string) { - _, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress!='0'").Exec() + _, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt", c.Config.PostIPCutoff, "day").Where("ipaddress!='0'").Exec() if err != nil { c.LogError(err) } @@ -192,6 +192,14 @@ func dailies() { f("topics") f("replies") f("users_replies") + } + + if c.Config.PollIPCutoff > -1 { + // TODO: Use unixtime to remove this MySQLesque logic? + _, err := qgen.NewAcc().Update("polls_votes").Set("ipaddress='0'").DateOlderThan("castAt", c.Config.PollIPCutoff, "day").Where("ipaddress!='0'").Exec() + if err != nil { + c.LogError(err) + } // TODO: Find some way of purging the ip data in polls_votes without breaking any anti-cheat measures which might be running... maybe hash it instead? } @@ -208,7 +216,7 @@ func dailies() { c.LogError(err) }*/ mon := time.Now().Month() - _, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '"+strconv.Itoa(int(mon))+"-%'").Exec() + _, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '" + strconv.Itoa(int(mon)) + "-%'").Exec() if err != nil { c.LogError(err) } @@ -220,4 +228,4 @@ func dailies() { c.LogError(err) } } -} \ No newline at end of file +}