From e76223224a9341668f5a6ba793bfca247961ffdd Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 8 May 2019 19:50:10 +1000 Subject: [PATCH] Implemented periodic post IP purging for replies. Update for MySQL now supports DateCutoff and DateOlderThan. Added the DateOlderThan method to accUpdateBuilder. Added the PostIPCutoff config setting. --- common/site.go | 5 ++++ docs/configuration.md | 2 ++ query_gen/acc_builders.go | 50 ++++++++++++++++++++++--------------- query_gen/micro_builders.go | 4 ++- query_gen/mysql.go | 8 ++++-- tickloop.go | 8 ++++++ 6 files changed, 54 insertions(+), 23 deletions(-) diff --git a/common/site.go b/common/site.go index 6431a37b..2d93f334 100644 --- a/common/site.go +++ b/common/site.go @@ -88,6 +88,7 @@ type config struct { PrimaryServer bool ServerCount int + PostIPCutoff int DisableLiveTopicList bool DisableJSAntispam bool @@ -197,6 +198,10 @@ func ProcessConfig() (err error) { } Site.MaxRequestSize = Config.MaxRequestSize + if Config.PostIPCutoff == 0 { + Config.PostIPCutoff = 180 // Default cutoff + } + // ? Find a way of making these unlimited if zero? It might rule out some optimisations, waste memory, and break layouts if Config.MaxTopicTitleLength == 0 { Config.MaxTopicTitleLength = 100 diff --git a/docs/configuration.md b/docs/configuration.md index 32cec701..9a598c32 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -80,6 +80,8 @@ BuildSlugs - Whether you want the title appear in the URL. For instance: `/topic ServerCount - The number of instances you're running. This setting is currently experimental. +PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0 + DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false DisableJSAntispam - This switch lets you disable the JS anti-spam feature. It may be useful if you primarily get users who for one reason or another have decided to disable JavaScript. Default: false diff --git a/query_gen/acc_builders.go b/query_gen/acc_builders.go index 71e60758..565828f4 100644 --- a/query_gen/acc_builders.go +++ b/query_gen/acc_builders.go @@ -44,37 +44,47 @@ type accUpdateBuilder struct { build *Accumulator } -func (update *accUpdateBuilder) Set(set string) *accUpdateBuilder { - update.up.set = set - return update +func (u *accUpdateBuilder) Set(set string) *accUpdateBuilder { + u.up.set = set + return u } -func (update *accUpdateBuilder) Where(where string) *accUpdateBuilder { - if update.up.where != "" { - update.up.where += " AND " +func (u *accUpdateBuilder) Where(where string) *accUpdateBuilder { + if u.up.where != "" { + u.up.where += " AND " } - update.up.where += where - return update + u.up.where += where + return u } -func (update *accUpdateBuilder) WhereQ(sel *selectPrebuilder) *accUpdateBuilder { - update.up.whereSubQuery = sel - return update +func (u *accUpdateBuilder) DateCutoff(column string, quantity int, unit string) *accUpdateBuilder { + u.up.dateCutoff = &dateCutoff{column, quantity, unit, 0} + return u } -func (builder *accUpdateBuilder) Prepare() *sql.Stmt { - if builder.up.whereSubQuery != nil { - return builder.build.prepare(builder.build.adapter.SimpleUpdateSelect(builder.up)) +func (u *accUpdateBuilder) DateOlderThan(column string, quantity int, unit string) *accUpdateBuilder { + u.up.dateCutoff = &dateCutoff{column, quantity, unit, 1} + return u +} + +func (u *accUpdateBuilder) WhereQ(sel *selectPrebuilder) *accUpdateBuilder { + u.up.whereSubQuery = sel + return u +} + +func (u *accUpdateBuilder) Prepare() *sql.Stmt { + if u.up.whereSubQuery != nil { + return u.build.prepare(u.build.adapter.SimpleUpdateSelect(u.up)) } - return builder.build.prepare(builder.build.adapter.SimpleUpdate(builder.up)) + return u.build.prepare(u.build.adapter.SimpleUpdate(u.up)) } -func (builder *accUpdateBuilder) Exec(args ...interface{}) (res sql.Result, err error) { - query, err := builder.build.adapter.SimpleUpdate(builder.up) +func (u *accUpdateBuilder) Exec(args ...interface{}) (res sql.Result, err error) { + query, err := u.build.adapter.SimpleUpdate(u.up) if err != nil { return res, err } - return builder.build.exec(query, args...) + return u.build.exec(query, args...) } type AccSelectBuilder struct { @@ -134,7 +144,7 @@ func (selectItem *AccSelectBuilder) InQ(column string, subBuilder *AccSelectBuil } func (builder *AccSelectBuilder) DateCutoff(column string, quantity int, unit string) *AccSelectBuilder { - builder.dateCutoff = &dateCutoff{column, quantity, unit} + builder.dateCutoff = &dateCutoff{column, quantity, unit, 0} return builder } @@ -309,7 +319,7 @@ func (b *accCountBuilder) Limit(limit string) *accCountBuilder { } func (b *accCountBuilder) DateCutoff(column string, quantity int, unit string) *accCountBuilder { - b.dateCutoff = &dateCutoff{column, quantity, unit} + b.dateCutoff = &dateCutoff{column, quantity, unit, 0} return b } diff --git a/query_gen/micro_builders.go b/query_gen/micro_builders.go index 23a73364..796c99cc 100644 --- a/query_gen/micro_builders.go +++ b/query_gen/micro_builders.go @@ -4,6 +4,7 @@ type dateCutoff struct { Column string Quantity int Unit string + Type int } type prebuilder struct { @@ -27,7 +28,7 @@ func (build *prebuilder) Insert(nlist ...string) *insertPrebuilder { func (build *prebuilder) Update(nlist ...string) *updatePrebuilder { name := optString(nlist, "") - return &updatePrebuilder{name, "", "", "", nil, build.adapter} + return &updatePrebuilder{name, "", "", "", nil, nil, build.adapter} } func (build *prebuilder) Delete(nlist ...string) *deletePrebuilder { @@ -69,6 +70,7 @@ type updatePrebuilder struct { table string set string where string + dateCutoff *dateCutoff // We might want to do this in a slightly less hacky way whereSubQuery *selectPrebuilder build Adapter diff --git a/query_gen/mysql.go b/query_gen/mysql.go index d56d2a4b..30c0b964 100644 --- a/query_gen/mysql.go +++ b/query_gen/mysql.go @@ -402,7 +402,7 @@ func (adapter *MysqlAdapter) SimpleUpdate(up *updatePrebuilder) (string, error) } querystr = querystr[0 : len(querystr)-1] - whereStr, err := adapter.buildWhere(up.where) + whereStr, err := adapter.buildFlexiWhere(up.where,up.dateCutoff) if err != nil { return querystr, err } @@ -485,7 +485,11 @@ func (adapter *MysqlAdapter) buildFlexiWhere(where string, dateCutoff *dateCutof } querystr = " WHERE" if dateCutoff != nil { - querystr += " " + dateCutoff.Column + " BETWEEN (UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + ") AND UTC_TIMESTAMP() AND" + if dateCutoff.Type == 0 { + querystr += " " + dateCutoff.Column + " BETWEEN (UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + ") AND UTC_TIMESTAMP() AND" + } else { + querystr += " " + dateCutoff.Column + " < UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + " AND" + } } if len(where) != 0 { for _, loc := range processWhere(where) { diff --git a/tickloop.go b/tickloop.go index cfc05a09..bc42c36d 100644 --- a/tickloop.go +++ b/tickloop.go @@ -142,6 +142,14 @@ func tickLoop(thumbChan chan bool) { if err != nil && err != sql.ErrNoRows { c.LogError(err) } + + if c.Config.PostIPCutoff > -1 { + // TODO: Use unixtime to remove this MySQLesque logic? + _, err := qgen.NewAcc().Update("replies").Set("ipaddress = '0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress != '0'").Exec() + if err != nil { + c.LogError(err) + } + } } // TODO: Handle the daily clean-up.