Add PollIPCutoff configuration setting.

Add tests for qgen processWhere and qgen mysql buildWhere.

Shorten some things.

common/site.go

@@ -93,6 +93,7 @@ type config struct {
 	ServerCount    int
 	LastIPCutoff   int // Currently just -1, non--1, but will accept the number of months a user's last IP should be retained for in the future before being purged. Please note that the other two cutoffs below operate off the numbers of days instead.
 	PostIPCutoff   int
+	PollIPCutoff int
 	LogPruneCutoff int
 
 	DisableLiveTopicList bool
@@ -228,6 +229,9 @@ func ProcessConfig() (err error) {
 	if Config.LastIPCutoff > 12 {
 		Config.LastIPCutoff = 12
 	}
+	if Config.PollIPCutoff == 0 {
+		Config.PollIPCutoff = -1 // Default cutoff
+	}
 	if Config.NoEmbed {
 		DefaultParseSettings.NoEmbed = true
 	}

docs/configuration.md

@@ -86,9 +86,11 @@ ServerCount - The number of instances you're running. This setting is currently
 
 LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature.
 
-PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. Default: 0
+PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature.
 
-LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0
+PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently -1 and -1 disables this feature.
+
+LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature.
 
 DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false
 

query_gen/mssql.go

@@ -236,7 +236,7 @@ func (a *MssqlAdapter) SimpleInsert(name, table, cols, fields string) (string, e
 }
 
 // ! DEPRECATED
-func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string, fields string) (string, error) {
+func (a *MssqlAdapter) SimpleReplace(name, table, columns, fields string) (string, error) {
 	log.Print("In SimpleReplace")
 	key, ok := a.keys[table]
 	if !ok {
@@ -271,7 +271,7 @@ func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string,
 	return a.SimpleUpsert(name, table, columns, fields, "key = "+keyValue)
 }
 
-func (a *MssqlAdapter) SimpleUpsert(name string, table string, columns string, fields string, where string) (string, error) {
+func (a *MssqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
 	if table == "" {
 		return "", errors.New("You need a name for this table")
 	}
@@ -975,14 +975,12 @@ func (a *MssqlAdapter) simpleJoin(name string, ins DBInsert, sel DBJoin, joinTyp
 		}
 		q += source + alias + ","
 	}
-	// Remove the trailing comma
 	q = q[0 : len(q)-1]
 
 	q += " FROM [" + sel.Table1 + "] " + joinType + " JOIN [" + sel.Table2 + "] ON "
 	for _, j := range processJoiner(sel.Joiners) {
 		q += "[" + j.LeftTable + "].[" + j.LeftColumn + "] " + j.Operator + " [" + j.RightTable + "].[" + j.RightColumn + "] AND "
 	}
-	// Remove the trailing AND
 	q = q[0 : len(q)-4]
 
 	// Add support for BETWEEN x.x
@@ -1166,17 +1164,17 @@ func _gen_mssql() (err error) {
 }
 
 // Internal methods, not exposed in the interface
-func (a *MssqlAdapter) pushStatement(name string, stype string, querystr string) {
+func (a *MssqlAdapter) pushStatement(name, stype, q string) {
 	if name == "" {
 		return
 	}
-	a.Buffer[name] = DBStmt{querystr, stype}
+	a.Buffer[name] = DBStmt{q, stype}
 	a.BufferOrder = append(a.BufferOrder, name)
 }
 
-func (a *MssqlAdapter) stringyType(ctype string) bool {
+func (a *MssqlAdapter) stringyType(ct string) bool {
-	ctype = strings.ToLower(ctype)
+	ct = strings.ToLower(ct)
-	return ctype == "char" || ctype == "varchar" || ctype == "datetime" || ctype == "text" || ctype == "nvarchar"
+	return ct == "char" || ct == "varchar" || ct == "datetime" || ct == "text" || ct == "nvarchar"
 }
 
 type SetPrimaryKeys interface {

query_gen/mysql.go

@@ -338,11 +338,11 @@ func (a *MysqlAdapter) SimpleReplace(name, table, columns, fields string) (strin
 	for _, field := range processFields(fields) {
 		q += field.Name + ","
 	}
-	q = q[0 : len(q)-1]
+	q = q[0 : len(q)-1] + ")"
 
 	// TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator
-	a.pushStatement(name, "replace", q+")")
+	a.pushStatement(name, "replace", q)
-	return q + ")", nil
+	return q, nil
 }
 
 func (a *MysqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
@@ -968,15 +968,15 @@ func _gen_mysql() (err error) {
 }
 
 // Internal methods, not exposed in the interface
-func (a *MysqlAdapter) pushStatement(name string, stype string, querystr string) {
+func (a *MysqlAdapter) pushStatement(name, stype, q string) {
 	if name == "" {
 		return
 	}
-	a.Buffer[name] = DBStmt{querystr, stype}
+	a.Buffer[name] = DBStmt{q, stype}
 	a.BufferOrder = append(a.BufferOrder, name)
 }
 
-func (a *MysqlAdapter) stringyType(ctype string) bool {
+func (a *MysqlAdapter) stringyType(ct string) bool {
-	ctype = strings.ToLower(ctype)
+	ct = strings.ToLower(ct)
-	return ctype == "varchar" || ctype == "tinytext" || ctype == "text" || ctype == "mediumtext" || ctype == "longtext" || ctype == "char" || ctype == "datetime" || ctype == "timestamp" || ctype == "time" || ctype == "date"
+	return ct == "varchar" || ct == "tinytext" || ct == "text" || ct == "mediumtext" || ct == "longtext" || ct == "char" || ct == "datetime" || ct == "timestamp" || ct == "time" || ct == "date"
 }

query_gen/utils_test.go

@@ -0,0 +1,65 @@
+package qgen
+
+import (
+	"strings"
+	"testing"
+)
+
+type MT struct {
+	Type     int
+	Contents string
+}
+
+func expectTokens(t *testing.T, whs []DBWhere, tokens ...MT) {
+	i := 0
+	for _, wh := range whs {
+		for _, expr := range wh.Expr {
+			if expr.Type != tokens[i].Type || expr.Contents != tokens[i].Contents {
+				t.Fatalf("token mismatch: %+v - %+v\n", expr, tokens[i])
+			}
+			i++
+		}
+	}
+}
+
+func TestProcessWhere(t *testing.T) {
+	whs := processWhere("uid = ?")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
+	whs = processWhere("uid = 1")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
+	whs = processWhere("uid = 0")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
+	whs = processWhere("uid = '1'")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
+	whs = processWhere("uid = ''")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
+	whs = processWhere("uid = '")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
+
+	whs = processWhere("uid=?")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
+	whs = processWhere("uid=1")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
+	whs = processWhere("uid=0")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
+	whs = processWhere("uid='1'")
+	expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
+
+	whs = processWhere("uid")
+	expectTokens(t, whs, MT{TokenColumn, "uid"})
+}
+
+func TestMySQLBuildWhere(t *testing.T) {
+	a := &MysqlAdapter{Name: "mysql", Buffer: make(map[string]DBStmt)}
+	reap := func(wh, ex string) {
+		sb := &strings.Builder{}
+		a.buildWhere(wh, sb)
+		res := sb.String()
+		if res != ex {
+			t.Fatalf("build where mismatch: '%+v' - '%+v'\n", ex, res)
+		}
+	}
+	reap("uid = 0", " WHERE `uid`= 0 ")
+	reap("uid = '0'", " WHERE `uid`= '0'")
+	reap("uid=0", " WHERE `uid`= 0 ")
+}

tickloop.go

@@ -1,15 +1,15 @@
 package main
 
 import (
+	"database/sql"
 	"errors"
 	"log"
-	"time"
 	"strconv"
 	"sync/atomic"
-	"database/sql"
+	"time"
 
 	c "github.com/Azareal/Gosora/common"
-	"github.com/Azareal/Gosora/query_gen"
+	qgen "github.com/Azareal/Gosora/query_gen"
 )
 
 // TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there?
@@ -172,7 +172,7 @@ func dailies() {
 
 	if c.Config.LogPruneCutoff > -1 {
 		f := func(tbl string) {
-			_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run()
+			_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt", c.Config.LogPruneCutoff, "day").Run()
 			if err != nil {
 				c.LogError(err)
 			}
@@ -184,7 +184,7 @@ func dailies() {
 	if c.Config.PostIPCutoff > -1 {
 		// TODO: Use unixtime to remove this MySQLesque logic?
 		f := func(tbl string) {
-			_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress!='0'").Exec()
+			_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt", c.Config.PostIPCutoff, "day").Where("ipaddress!='0'").Exec()
 			if err != nil {
 				c.LogError(err)
 			}
@@ -192,6 +192,14 @@ func dailies() {
 		f("topics")
 		f("replies")
 		f("users_replies")
+	}
+
+	if c.Config.PollIPCutoff > -1 {
+		// TODO: Use unixtime to remove this MySQLesque logic?
+		_, err := qgen.NewAcc().Update("polls_votes").Set("ipaddress='0'").DateOlderThan("castAt", c.Config.PollIPCutoff, "day").Where("ipaddress!='0'").Exec()
+		if err != nil {
+			c.LogError(err)
+		}
 
 		// TODO: Find some way of purging the ip data in polls_votes without breaking any anti-cheat measures which might be running... maybe hash it instead?
 	}
@@ -208,7 +216,7 @@ func dailies() {
 			c.LogError(err)
 		}*/
 		mon := time.Now().Month()
-		_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '"+strconv.Itoa(int(mon))+"-%'").Exec()
+		_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '" + strconv.Itoa(int(mon)) + "-%'").Exec()
 		if err != nil {
 			c.LogError(err)
 		}