a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add PollIPCutoff configuration setting. 

Add tests for qgen processWhere and qgen mysql buildWhere.

Shorten some things.
This commit is contained in:
Azareal 2020-01-02 15:28:36 +10:00
parent 53212f3022
commit f789a8498e
6 changed files with 104 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/site.go
View File

@ -93,6 +93,7 @@ type config struct {
ServerCount int
LastIPCutoff int // Currently just -1, non--1, but will accept the number of months a user's last IP should be retained for in the future before being purged. Please note that the other two cutoffs below operate off the numbers of days instead.
PostIPCutoff int
PollIPCutoff int
LogPruneCutoff int
DisableLiveTopicList bool
@ -228,6 +229,9 @@ func ProcessConfig() (err error) {
if Config.LastIPCutoff > 12 {
Config.LastIPCutoff = 12
}
if Config.PollIPCutoff == 0 {
Config.PollIPCutoff = -1 // Default cutoff
}
if Config.NoEmbed {
DefaultParseSettings.NoEmbed = true
}

6
docs/configuration.md
View File

@ -86,9 +86,11 @@ ServerCount - The number of instances you're running. This setting is currently
LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature.
PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. Default: 0
PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature.
LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0
PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently -1 and -1 disables this feature.
LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature.
DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false

16
query_gen/mssql.go
View File

@ -236,7 +236,7 @@ func (a *MssqlAdapter) SimpleInsert(name, table, cols, fields string) (string, e
}
// ! DEPRECATED
func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string, fields string) (string, error) {
func (a *MssqlAdapter) SimpleReplace(name, table, columns, fields string) (string, error) {
log.Print("In SimpleReplace")
key, ok := a.keys[table]
if !ok {
@ -271,7 +271,7 @@ func (a *MssqlAdapter) SimpleReplace(name string, table string, columns string,
return a.SimpleUpsert(name, table, columns, fields, "key = "+keyValue)
}
func (a *MssqlAdapter) SimpleUpsert(name string, table string, columns string, fields string, where string) (string, error) {
func (a *MssqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
if table == "" {
return "", errors.New("You need a name for this table")
}
@ -975,14 +975,12 @@ func (a *MssqlAdapter) simpleJoin(name string, ins DBInsert, sel DBJoin, joinTyp
}
q += source + alias + ","
}
// Remove the trailing comma
q = q[0 : len(q)-1]
q += " FROM [" + sel.Table1 + "] " + joinType + " JOIN [" + sel.Table2 + "] ON "
for _, j := range processJoiner(sel.Joiners) {
q += "[" + j.LeftTable + "].[" + j.LeftColumn + "] " + j.Operator + " [" + j.RightTable + "].[" + j.RightColumn + "] AND "
}
// Remove the trailing AND
q = q[0 : len(q)-4]
// Add support for BETWEEN x.x
@ -1166,17 +1164,17 @@ func _gen_mssql() (err error) {
}
// Internal methods, not exposed in the interface
func (a *MssqlAdapter) pushStatement(name string, stype string, querystr string) {
func (a *MssqlAdapter) pushStatement(name, stype, q string) {
if name == "" {
return
}
a.Buffer[name] = DBStmt{querystr, stype}
a.Buffer[name] = DBStmt{q, stype}
a.BufferOrder = append(a.BufferOrder, name)
}
func (a *MssqlAdapter) stringyType(ctype string) bool {
ctype = strings.ToLower(ctype)
return ctype == "char" || ctype == "varchar" || ctype == "datetime" || ctype == "text" || ctype == "nvarchar"
func (a *MssqlAdapter) stringyType(ct string) bool {
ct = strings.ToLower(ct)
return ct == "char" || ct == "varchar" || ct == "datetime" || ct == "text" || ct == "nvarchar"
}
type SetPrimaryKeys interface {

16
query_gen/mysql.go
View File

@ -338,11 +338,11 @@ func (a *MysqlAdapter) SimpleReplace(name, table, columns, fields string) (strin
for _, field := range processFields(fields) {
q += field.Name + ","
}
q = q[0 : len(q)-1]
q = q[0 : len(q)-1] + ")"
// TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator
a.pushStatement(name, "replace", q+")")
return q + ")", nil
a.pushStatement(name, "replace", q)
return q, nil
}
func (a *MysqlAdapter) SimpleUpsert(name, table, columns, fields, where string) (string, error) {
@ -968,15 +968,15 @@ func _gen_mysql() (err error) {
}
// Internal methods, not exposed in the interface
func (a *MysqlAdapter) pushStatement(name string, stype string, querystr string) {
func (a *MysqlAdapter) pushStatement(name, stype, q string) {
if name == "" {
return
}
a.Buffer[name] = DBStmt{querystr, stype}
a.Buffer[name] = DBStmt{q, stype}
a.BufferOrder = append(a.BufferOrder, name)
}
func (a *MysqlAdapter) stringyType(ctype string) bool {
ctype = strings.ToLower(ctype)
return ctype == "varchar" || ctype == "tinytext" || ctype == "text" || ctype == "mediumtext" || ctype == "longtext" || ctype == "char" || ctype == "datetime" || ctype == "timestamp" || ctype == "time" || ctype == "date"
func (a *MysqlAdapter) stringyType(ct string) bool {
ct = strings.ToLower(ct)
return ct == "varchar" || ct == "tinytext" || ct == "text" || ct == "mediumtext" || ct == "longtext" || ct == "char" || ct == "datetime" || ct == "timestamp" || ct == "time" || ct == "date"
}

65
query_gen/utils_test.go Normal file
View File

@ -0,0 +1,65 @@
package qgen
import (
"strings"
"testing"
)
type MT struct {
Type int
Contents string
}
func expectTokens(t *testing.T, whs []DBWhere, tokens ...MT) {
i := 0
for _, wh := range whs {
for _, expr := range wh.Expr {
if expr.Type != tokens[i].Type || expr.Contents != tokens[i].Contents {
t.Fatalf("token mismatch: %+v - %+v\n", expr, tokens[i])
}
i++
}
}
}
func TestProcessWhere(t *testing.T) {
whs := processWhere("uid = ?")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
whs = processWhere("uid = 1")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
whs = processWhere("uid = 0")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
whs = processWhere("uid = '1'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
whs = processWhere("uid = ''")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
whs = processWhere("uid = '")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
whs = processWhere("uid=?")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenSub, "?"})
whs = processWhere("uid=1")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
whs = processWhere("uid=0")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
whs = processWhere("uid='1'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
whs = processWhere("uid")
expectTokens(t, whs, MT{TokenColumn, "uid"})
}
func TestMySQLBuildWhere(t *testing.T) {
a := &MysqlAdapter{Name: "mysql", Buffer: make(map[string]DBStmt)}
reap := func(wh, ex string) {
sb := &strings.Builder{}
a.buildWhere(wh, sb)
res := sb.String()
if res != ex {
t.Fatalf("build where mismatch: '%+v' - '%+v'\n", ex, res)
}
}
reap("uid = 0", " WHERE `uid`= 0 ")
reap("uid = '0'", " WHERE `uid`= '0'")
reap("uid=0", " WHERE `uid`= 0 ")
}

24
tickloop.go
View File

@ -1,15 +1,15 @@
package main
import (
"database/sql"
"errors"
"log"
"time"
"strconv"
"sync/atomic"
"database/sql"
"time"
c "github.com/Azareal/Gosora/common"
"github.com/Azareal/Gosora/query_gen"
qgen "github.com/Azareal/Gosora/query_gen"
)
// TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there?
@ -59,7 +59,7 @@ func tickLoop(thumbChan chan bool) {
if lastDaily < low {
dailies()
}
// TODO: Write tests for these
// Run this goroutine once every half second
halfSecondTicker := time.NewTicker(time.Second / 2)
@ -172,7 +172,7 @@ func dailies() {
if c.Config.LogPruneCutoff > -1 {
f := func(tbl string) {
_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run()
_, err := qgen.NewAcc().Delete(tbl).DateOlderThan("doneAt", c.Config.LogPruneCutoff, "day").Run()
if err != nil {
c.LogError(err)
}
@ -184,7 +184,7 @@ func dailies() {
if c.Config.PostIPCutoff > -1 {
// TODO: Use unixtime to remove this MySQLesque logic?
f := func(tbl string) {
_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress!='0'").Exec()
_, err := qgen.NewAcc().Update(tbl).Set("ipaddress='0'").DateOlderThan("createdAt", c.Config.PostIPCutoff, "day").Where("ipaddress!='0'").Exec()
if err != nil {
c.LogError(err)
}
@ -192,6 +192,14 @@ func dailies() {
f("topics")
f("replies")
f("users_replies")
}
if c.Config.PollIPCutoff > -1 {
// TODO: Use unixtime to remove this MySQLesque logic?
_, err := qgen.NewAcc().Update("polls_votes").Set("ipaddress='0'").DateOlderThan("castAt", c.Config.PollIPCutoff, "day").Where("ipaddress!='0'").Exec()
if err != nil {
c.LogError(err)
}
// TODO: Find some way of purging the ip data in polls_votes without breaking any anti-cheat measures which might be running... maybe hash it instead?
}
@ -208,7 +216,7 @@ func dailies() {
c.LogError(err)
}*/
mon := time.Now().Month()
_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '"+strconv.Itoa(int(mon))+"-%'").Exec()
_, err := qgen.NewAcc().Update("users").Set("last_ip=0").Where("last_ip!=0 AND last_ip NOT LIKE '" + strconv.Itoa(int(mon)) + "-%'").Exec()
if err != nil {
c.LogError(err)
}
@ -220,4 +228,4 @@ func dailies() {
c.LogError(err)
}
}
}
}