Add PollIPCutoff configuration setting.
Add tests for qgen processWhere and qgen mysql buildWhere. Shorten some things.
This commit is contained in:
Azareal
parent
53212f3022
commit
f789a8498e
|
|
@ -93,6 +93,7 @@ type config struct {
|
|||
ServerCount int
|
||||
LastIPCutoff int // Currently just -1, non--1, but will accept the number of months a user's last IP should be retained for in the future before being purged. Please note that the other two cutoffs below operate off the numbers of days instead.
|
||||
PostIPCutoff int
|
||||
PollIPCutoff int
|
||||
LogPruneCutoff int
|
||||
|
||||
DisableLiveTopicList bool
|
||||
|
|
@ -228,6 +229,9 @@ func ProcessConfig() (err error) {
|
|||
if Config.LastIPCutoff > 12 {
|
||||
Config.LastIPCutoff = 12
|
||||
}
|
||||
if Config.PollIPCutoff == 0 {
|
||||
Config.PollIPCutoff = -1 // Default cutoff
|
||||
}
|
||||
if Config.NoEmbed {
|
||||
DefaultParseSettings.NoEmbed = true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,9 +86,11 @@ ServerCount - The number of instances you're running. This setting is currently
|
|||
|
||||
LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature.
|
||||
|
||||
PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. Default: 0
|
||||
PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature.
|
||||
|
||||
LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0
|
||||
PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently -1 and -1 disables this feature.
|
||||
|
||||
LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature.
|
||||
|
||||
DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false
|
||||
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ func (a *MssqlAdapter) SimpleInsert(name, table, cols, fields string) (string, e
|
|||
}
|
||||
|
||||
// ! DEPRECATED
|
||||
func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string, fields string) (string, error) {
|
||||
func (a *MssqlAdapter) SimpleReplace(name, table, columns, fields string) (string, error) {
|
||||
log.Print("In SimpleReplace")
|
||||
key, ok := a.keys[table]
|
||||
if !ok {
|
||||
|
|
@ -271,7 +271,7 @@ func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string,
|
|||
return a.SimpleUpsert(name, table, columns, fields, "key = "+keyValue)
|
||||
}
|
||||
|
||||
func (a *MssqlAdapter) SimpleUpsert(name string, table string, columns string, fields string, where string) (string, error) {
|
||||
func (a *MssqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
|
||||
if table == "" {
|
||||
return "", errors.New("You need a name for this table")
|
||||
}
|
||||
|
|
@ -975,14 +975,12 @@ func (a *MssqlAdapter) simpleJoin(name string, ins DBInsert, sel DBJoin, joinTyp
|
|||
}
|
||||
q += source + alias + ","
|
||||
}
|
||||
// Remove the trailing comma
|
||||
q = q[0 : len(q)-1]
|
||||
|
||||
q += " FROM [" + sel.Table1 + "] " + joinType + " JOIN [" + sel.Table2 + "] ON "
|
||||
for _, j := range processJoiner(sel.Joiners) {
|
||||
q += "[" + j.LeftTable + "].[" + j.LeftColumn + "] " + j.Operator + " [" + j.RightTable + "].[" + j.RightColumn + "] AND "
|
||||
}
|
||||
// Remove the trailing AND
|
||||
q = q[0 : len(q)-4]
|
||||
|
||||
// Add support for BETWEEN x.x
|
||||
|
|
@ -1166,17 +1164,17 @@ func _gen_mssql() (err error) {
|
|||
}
|
||||
|
||||
// Internal methods, not exposed in the interface
|
||||
func (a *MssqlAdapter) pushStatement(name string, stype string, querystr string) {
|
||||
func (a *MssqlAdapter) pushStatement(name, stype, q string) {
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
a.Buffer[name] = DBStmt{querystr, stype}
|
||||
a.Buffer[name] = DBStmt{q, stype}
|
||||
a.BufferOrder = append(a.BufferOrder, name)
|
||||
}
|
||||
|
||||
func (a *MssqlAdapter) stringyType(ctype string) bool {
|
||||
ctype = strings.ToLower(ctype)
|
||||
return ctype == "char" || ctype == "varchar" || ctype == "datetime" || ctype == "text" || ctype == "nvarchar"
|
||||
func (a *MssqlAdapter) stringyType(ct string) bool {
|
||||
ct = strings.ToLower(ct)
|
||||
return ct == "char" || ct == "varchar" || ct == "datetime" || ct == "text" || ct == "nvarchar"
|
||||
}
|
||||
|
||||
type SetPrimaryKeys interface {
|
||||
|
|
|
|||
|
|
@ -338,11 +338,11 @@ func (a *MysqlAdapter) SimpleReplace(name, table, columns, fields string) (strin
|
|||
for _, field := range processFields(fields) {
|
||||
q += field.Name + ","
|
||||
}
|
||||
q = q[0 : len(q)-1]
|
||||
q = q[0 : len(q)-1] + ")"
|
||||
|
||||
// TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator
|
||||
a.pushStatement(name, "replace", q+")")
|
||||
return q + ")", nil
|
||||
a.pushStatement(name, "replace", q)
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (a *MysqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
|
||||
|
|
@ -968,15 +968,15 @@ func _gen_mysql() (err error) {
|
|||
}
|
||||
|
||||
// Internal methods, not exposed in the interface
|
||||
func (a *MysqlAdapter) pushStatement(name string, stype string, querystr string) {
|
||||
func (a *MysqlAdapter) pushStatement(name, stype, q string) {
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
a.Buffer[name] = DBStmt{querystr, stype}
|
||||
a.Buffer[name] = DBStmt{q, stype}
|
||||
a.BufferOrder = append(a.BufferOrder, name)
|
||||
}
|
||||
|
||||
func (a *MysqlAdapter) stringyType(ctype string) bool {
|
||||
ctype = strings.ToLower(ctype)
|
||||
return ctype == "varchar" || ctype == "tinytext" || ctype == "text" || ctype == "mediumtext" || ctype == "longtext" || ctype == "char" || ctype == "datetime" || ctype == "timestamp" || ctype == "time" || ctype == "date"
|
||||
func (a *MysqlAdapter) stringyType(ct string) bool {
|
||||
ct = strings.ToLower(ct)
|
||||
return ct == "varchar" || ct == "tinytext" || ct == "text" || ct == "mediumtext" || ct == "longtext" || ct == "char" || ct == "datetime" || ct == "timestamp" || ct == "time" || ct == "date"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
package qgen
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type MT struct {
|
||||
Type int
|
||||
Contents string
|
||||
}
|
||||
|
||||
func expectTokens(t *testing.T, whs []DBWhere, tokens ...MT) {
|
||||
i := 0
|
||||
for _, wh := range whs {
|
||||
for _, expr := range wh.Expr {
|
||||
if expr.Type != tokens[i].Type || expr.Contents != tokens[i].Contents {
|
||||
t.Fatalf("token mismatch: %+v - %+v\n", expr, tokens[i])
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessWhere(t *testing.T) {
|
||||
whs := processWhere("uid = ?")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
|
||||
whs = processWhere("uid = 1")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
|
||||
whs = processWhere("uid = 0")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
|
||||
whs = processWhere("uid = '1'")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
|
||||
whs = processWhere("uid = ''")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
|
||||
whs = processWhere("uid = '")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
|
||||
|
||||
whs = processWhere("uid=?")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
|
||||
whs = processWhere("uid=1")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
|
||||
whs = processWhere("uid=0")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
|
||||
whs = processWhere("uid='1'")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
|
||||
|
||||
whs = processWhere("uid")
|
||||
expectTokens(t, whs, MT{TokenColumn, "uid"})
|
||||
}
|
||||
|
||||
func TestMySQLBuildWhere(t *testing.T) {
|
||||
a := &MysqlAdapter{Name: "mysql", Buffer: make(map[string]DBStmt)}
|
||||
reap := func(wh, ex string) {
|
||||
sb := &strings.Builder{}
|
||||
a.buildWhere(wh, sb)
|
||||
res := sb.String()
|
||||
if res != ex {
|
||||
t.Fatalf("build where mismatch: '%+v' - '%+v'\n", ex, res)
|
||||
}
|
||||
}
|
||||
reap("uid = 0", " WHERE `uid`= 0 ")
|
||||
reap("uid = '0'", " WHERE `uid`= '0'")
|
||||
reap("uid=0", " WHERE `uid`= 0 ")
|
||||
}
|
||||
20
tickloop.go
20
tickloop.go
|
|
@ -1,15 +1,15 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
c "github.com/Azareal/Gosora/common"
|
||||
"github.com/Azareal/Gosora/query_gen"
|
||||
qgen "github.com/Azareal/Gosora/query_gen"
|
||||
)
|
||||
|
||||
// TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there?
|
||||
|
|
@ -172,7 +172,7 @@ func dailies() {
|
|||
|
||||
if c.Config.LogPruneCutoff > -1 {
|
||||
f := func(tbl string) {
|
||||
_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run()
|
||||
_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt", c.Config.LogPruneCutoff, "day").Run()
|
||||
if err != nil {
|
||||
c.LogError(err)
|
||||
}
|
||||
|
|
@ -184,7 +184,7 @@ func dailies() {
|
|||
if c.Config.PostIPCutoff > -1 {
|
||||
// TODO: Use unixtime to remove this MySQLesque logic?
|
||||
f := func(tbl string) {
|
||||
_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress!='0'").Exec()
|
||||
_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt", c.Config.PostIPCutoff, "day").Where("ipaddress!='0'").Exec()
|
||||
if err != nil {
|
||||
c.LogError(err)
|
||||
}
|
||||
|
|
@ -192,6 +192,14 @@ func dailies() {
|
|||
f("topics")
|
||||
f("replies")
|
||||
f("users_replies")
|
||||
}
|
||||
|
||||
if c.Config.PollIPCutoff > -1 {
|
||||
// TODO: Use unixtime to remove this MySQLesque logic?
|
||||
_, err := qgen.NewAcc().Update("polls_votes").Set("ipaddress='0'").DateOlderThan("castAt", c.Config.PollIPCutoff, "day").Where("ipaddress!='0'").Exec()
|
||||
if err != nil {
|
||||
c.LogError(err)
|
||||
}
|
||||
|
||||
// TODO: Find some way of purging the ip data in polls_votes without breaking any anti-cheat measures which might be running... maybe hash it instead?
|
||||
}
|
||||
|
|
@ -208,7 +216,7 @@ func dailies() {
|
|||
c.LogError(err)
|
||||
}*/
|
||||
mon := time.Now().Month()
|
||||
_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '"+strconv.Itoa(int(mon))+"-%'").Exec()
|
||||
_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '" + strconv.Itoa(int(mon)) + "-%'").Exec()
|
||||
if err != nil {
|
||||
c.LogError(err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue