From ac0dd6d2cb9c1d6a4b0498eb5c721f902ee9168e Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 5 Jun 2019 14:57:10 +1000 Subject: [PATCH] The login and registration logs are pruned after a year now. Add filter exclusions for the user agents analytics pane. Shorten some of the variable names in qgen. Add ComplexDelete to the query adapter. Add DateCutoff and DateOlderThan to accDeleteBuilder. --- common/site.go | 4 + docs/configuration.md | 2 + mysql.go | 2 +- query_gen/acc_builders.go | 199 +++++++++++++++++-------------- query_gen/accumulator.go | 2 +- query_gen/micro_builders.go | 167 +++++++++++++------------- query_gen/mssql.go | 6 +- query_gen/mysql.go | 226 +++++++++++++++++++----------------- query_gen/pgsql.go | 11 ++ query_gen/querygen.go | 5 +- routes/panel/analytics.go | 30 ++++- tickloop.go | 11 ++ 12 files changed, 382 insertions(+), 283 deletions(-) diff --git a/common/site.go b/common/site.go index 5a9a528b..a90d604e 100644 --- a/common/site.go +++ b/common/site.go @@ -91,6 +91,7 @@ type config struct { PrimaryServer bool ServerCount int PostIPCutoff int + LogPruneCutoff int DisableLiveTopicList bool DisableJSAntispam bool @@ -204,6 +205,9 @@ func ProcessConfig() (err error) { if Config.PostIPCutoff == 0 { Config.PostIPCutoff = 180 // Default cutoff } + if Config.LogPruneCutoff == 0 { + Config.LogPruneCutoff = 365 // Default cutoff + } // ? Find a way of making these unlimited if zero? It might rule out some optimisations, waste memory, and break layouts if Config.MaxTopicTitleLength == 0 { diff --git a/docs/configuration.md b/docs/configuration.md index 9c55ed30..a2610608 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -86,6 +86,8 @@ ServerCount - The number of instances you're running. This setting is currently PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 180 and -1 disables this feature. Default: 0 +LogPruneCutoff - The number of days which need to pass before the login and registration logs are pruned. 0 defaults to whatever the current default is, currently 365 and -1 disables this feature. Default: 0 + DisableLiveTopicList - This switch allows you to disable the live topic list. Default: false DisableJSAntispam - This switch lets you disable the JS anti-spam feature. It may be useful if you primarily get users who for one reason or another have decided to disable JavaScript. Default: false diff --git a/mysql.go b/mysql.go index 505a22d4..e1a7277a 100644 --- a/mysql.go +++ b/mysql.go @@ -3,7 +3,7 @@ /* * * Gosora MySQL Interface -* Copyright Azareal 2016 - 2019 +* Copyright Azareal 2016 - 2020 * */ package main diff --git a/query_gen/acc_builders.go b/query_gen/acc_builders.go index 565828f4..b70e8e91 100644 --- a/query_gen/acc_builders.go +++ b/query_gen/acc_builders.go @@ -8,26 +8,47 @@ import ( type accDeleteBuilder struct { table string where string + dateCutoff *dateCutoff // We might want to do this in a slightly less hacky way build *Accumulator } -func (builder *accDeleteBuilder) Where(where string) *accDeleteBuilder { - if builder.where != "" { - builder.where += " AND " +func (b *accDeleteBuilder) Where(where string) *accDeleteBuilder { + if b.where != "" { + b.where += " AND " } - builder.where += where - return builder + b.where += where + return b } -func (builder *accDeleteBuilder) Prepare() *sql.Stmt { - return builder.build.SimpleDelete(builder.table, builder.where) +func (b *accDeleteBuilder) DateCutoff(column string, quantity int, unit string) *accDeleteBuilder { + b.dateCutoff = &dateCutoff{column, quantity, unit, 0} + return b } -func (builder *accDeleteBuilder) Run(args ...interface{}) (int, error) { - stmt := builder.Prepare() +func (b *accDeleteBuilder) DateOlderThan(column string, quantity int, unit string) *accDeleteBuilder { + b.dateCutoff = &dateCutoff{column, quantity, unit, 1} + return b +} + +/*func (b *accDeleteBuilder) Prepare() *sql.Stmt { + return b.build.SimpleDelete(b.table, b.where) +}*/ + +// TODO: Fix this nasty hack +func (b *accDeleteBuilder) Prepare() *sql.Stmt { + // TODO: Phase out the procedural API and use the adapter's OO API? The OO API might need a bit more work before we do that and it needs to be rolled out to MSSQL. + if b.dateCutoff != nil { + dBuilder := b.build.GetAdapter().Builder().Delete().FromAcc(b) + return b.build.prepare(b.build.GetAdapter().ComplexDelete(dBuilder)) + } + return b.build.SimpleDelete(b.table, b.where) +} + +func (b *accDeleteBuilder) Run(args ...interface{}) (int, error) { + stmt := b.Prepare() if stmt == nil { - return 0, builder.build.FirstError() + return 0, b.build.FirstError() } res, err := stmt.Exec(args...) @@ -57,26 +78,26 @@ func (u *accUpdateBuilder) Where(where string) *accUpdateBuilder { return u } -func (u *accUpdateBuilder) DateCutoff(column string, quantity int, unit string) *accUpdateBuilder { - u.up.dateCutoff = &dateCutoff{column, quantity, unit, 0} - return u +func (b *accUpdateBuilder) DateCutoff(column string, quantity int, unit string) *accUpdateBuilder { + b.up.dateCutoff = &dateCutoff{column, quantity, unit, 0} + return b } -func (u *accUpdateBuilder) DateOlderThan(column string, quantity int, unit string) *accUpdateBuilder { - u.up.dateCutoff = &dateCutoff{column, quantity, unit, 1} - return u +func (b *accUpdateBuilder) DateOlderThan(column string, quantity int, unit string) *accUpdateBuilder { + b.up.dateCutoff = &dateCutoff{column, quantity, unit, 1} + return b } -func (u *accUpdateBuilder) WhereQ(sel *selectPrebuilder) *accUpdateBuilder { - u.up.whereSubQuery = sel - return u +func (b *accUpdateBuilder) WhereQ(sel *selectPrebuilder) *accUpdateBuilder { + b.up.whereSubQuery = sel + return b } -func (u *accUpdateBuilder) Prepare() *sql.Stmt { - if u.up.whereSubQuery != nil { - return u.build.prepare(u.build.adapter.SimpleUpdateSelect(u.up)) +func (b *accUpdateBuilder) Prepare() *sql.Stmt { + if b.up.whereSubQuery != nil { + return b.build.prepare(b.build.adapter.SimpleUpdateSelect(b.up)) } - return u.build.prepare(u.build.adapter.SimpleUpdate(u.up)) + return b.build.prepare(b.build.adapter.SimpleUpdate(b.up)) } func (u *accUpdateBuilder) Exec(args ...interface{}) (res sql.Result, err error) { @@ -100,28 +121,28 @@ type AccSelectBuilder struct { build *Accumulator } -func (builder *AccSelectBuilder) Columns(columns string) *AccSelectBuilder { - builder.columns = columns - return builder +func (b *AccSelectBuilder) Columns(columns string) *AccSelectBuilder { + b.columns = columns + return b } -func (builder *AccSelectBuilder) Cols(columns string) *AccSelectBuilder { - builder.columns = columns - return builder +func (b *AccSelectBuilder) Cols(columns string) *AccSelectBuilder { + b.columns = columns + return b } -func (builder *AccSelectBuilder) Where(where string) *AccSelectBuilder { - if builder.where != "" { - builder.where += " AND " +func (b *AccSelectBuilder) Where(where string) *AccSelectBuilder { + if b.where != "" { + b.where += " AND " } - builder.where += where - return builder + b.where += where + return b } // TODO: Don't implement the SQL at the accumulator level but the adapter level -func (selectItem *AccSelectBuilder) In(column string, inList []int) *AccSelectBuilder { +func (b *AccSelectBuilder) In(column string, inList []int) *AccSelectBuilder { if len(inList) == 0 { - return selectItem + return b } var where = column + " IN(" @@ -129,59 +150,59 @@ func (selectItem *AccSelectBuilder) In(column string, inList []int) *AccSelectBu where += strconv.Itoa(item) + "," } where = where[:len(where)-1] + ")" - if selectItem.where != "" { - where += " AND " + selectItem.where + if b.where != "" { + where += " AND " + b.where } - selectItem.where = where - return selectItem + b.where = where + return b } -func (selectItem *AccSelectBuilder) InQ(column string, subBuilder *AccSelectBuilder) *AccSelectBuilder { - selectItem.inChain = subBuilder - selectItem.inColumn = column - return selectItem +func (b *AccSelectBuilder) InQ(column string, subBuilder *AccSelectBuilder) *AccSelectBuilder { + b.inChain = subBuilder + b.inColumn = column + return b } -func (builder *AccSelectBuilder) DateCutoff(column string, quantity int, unit string) *AccSelectBuilder { - builder.dateCutoff = &dateCutoff{column, quantity, unit, 0} - return builder +func (b *AccSelectBuilder) DateCutoff(column string, quantity int, unit string) *AccSelectBuilder { + b.dateCutoff = &dateCutoff{column, quantity, unit, 0} + return b } -func (builder *AccSelectBuilder) Orderby(orderby string) *AccSelectBuilder { - builder.orderby = orderby - return builder +func (b *AccSelectBuilder) Orderby(orderby string) *AccSelectBuilder { + b.orderby = orderby + return b } -func (builder *AccSelectBuilder) Limit(limit string) *AccSelectBuilder { - builder.limit = limit - return builder +func (b *AccSelectBuilder) Limit(limit string) *AccSelectBuilder { + b.limit = limit + return b } -func (builder *AccSelectBuilder) Prepare() *sql.Stmt { +func (b *AccSelectBuilder) Prepare() *sql.Stmt { // TODO: Phase out the procedural API and use the adapter's OO API? The OO API might need a bit more work before we do that and it needs to be rolled out to MSSQL. - if builder.dateCutoff != nil || builder.inChain != nil { - selectBuilder := builder.build.GetAdapter().Builder().Select().FromAcc(builder) - return builder.build.prepare(builder.build.GetAdapter().ComplexSelect(selectBuilder)) + if b.dateCutoff != nil || b.inChain != nil { + selectBuilder := b.build.GetAdapter().Builder().Select().FromAcc(b) + return b.build.prepare(b.build.GetAdapter().ComplexSelect(selectBuilder)) } - return builder.build.SimpleSelect(builder.table, builder.columns, builder.where, builder.orderby, builder.limit) + return b.build.SimpleSelect(b.table, b.columns, b.where, b.orderby, b.limit) } -func (builder *AccSelectBuilder) query() (string, error) { +func (b *AccSelectBuilder) query() (string, error) { // TODO: Phase out the procedural API and use the adapter's OO API? The OO API might need a bit more work before we do that and it needs to be rolled out to MSSQL. - if builder.dateCutoff != nil || builder.inChain != nil { - selectBuilder := builder.build.GetAdapter().Builder().Select().FromAcc(builder) - return builder.build.GetAdapter().ComplexSelect(selectBuilder) + if b.dateCutoff != nil || b.inChain != nil { + selectBuilder := b.build.GetAdapter().Builder().Select().FromAcc(b) + return b.build.GetAdapter().ComplexSelect(selectBuilder) } - return builder.build.adapter.SimpleSelect("", builder.table, builder.columns, builder.where, builder.orderby, builder.limit) + return b.build.adapter.SimpleSelect("", b.table, b.columns, b.where, b.orderby, b.limit) } -func (builder *AccSelectBuilder) Query(args ...interface{}) (*sql.Rows, error) { - stmt := builder.Prepare() +func (b *AccSelectBuilder) Query(args ...interface{}) (*sql.Rows, error) { + stmt := b.Prepare() if stmt != nil { return stmt.Query(args...) } - return nil, builder.build.FirstError() + return nil, b.build.FirstError() } type AccRowWrap struct { @@ -197,21 +218,21 @@ func (wrap *AccRowWrap) Scan(dest ...interface{}) error { } // TODO: Test to make sure the errors are passed up properly -func (builder *AccSelectBuilder) QueryRow(args ...interface{}) *AccRowWrap { - stmt := builder.Prepare() +func (b *AccSelectBuilder) QueryRow(args ...interface{}) *AccRowWrap { + stmt := b.Prepare() if stmt != nil { return &AccRowWrap{stmt.QueryRow(args...), nil} } - return &AccRowWrap{nil, builder.build.FirstError()} + return &AccRowWrap{nil, b.build.FirstError()} } // Experimental, reduces lines -func (builder *AccSelectBuilder) Each(handle func(*sql.Rows) error) error { - query, err := builder.query() +func (b *AccSelectBuilder) Each(handle func(*sql.Rows) error) error { + query, err := b.query() if err != nil { return err } - rows, err := builder.build.query(query) + rows, err := b.build.query(query) if err != nil { return err } @@ -225,12 +246,12 @@ func (builder *AccSelectBuilder) Each(handle func(*sql.Rows) error) error { } return rows.Err() } -func (builder *AccSelectBuilder) EachInt(handle func(int) error) error { - query, err := builder.query() +func (b *AccSelectBuilder) EachInt(handle func(int) error) error { + query, err := b.query() if err != nil { return err } - rows, err := builder.build.query(query) + rows, err := b.build.query(query) if err != nil { return err } @@ -258,34 +279,34 @@ type accInsertBuilder struct { build *Accumulator } -func (insert *accInsertBuilder) Columns(columns string) *accInsertBuilder { - insert.columns = columns - return insert +func (b *accInsertBuilder) Columns(columns string) *accInsertBuilder { + b.columns = columns + return b } -func (insert *accInsertBuilder) Fields(fields string) *accInsertBuilder { - insert.fields = fields - return insert +func (b *accInsertBuilder) Fields(fields string) *accInsertBuilder { + b.fields = fields + return b } -func (insert *accInsertBuilder) Prepare() *sql.Stmt { - return insert.build.SimpleInsert(insert.table, insert.columns, insert.fields) +func (b *accInsertBuilder) Prepare() *sql.Stmt { + return b.build.SimpleInsert(b.table, b.columns, b.fields) } -func (builder *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err error) { - query, err := builder.build.adapter.SimpleInsert("", builder.table, builder.columns, builder.fields) +func (b *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err error) { + query, err := b.build.adapter.SimpleInsert("", b.table, b.columns, b.fields) if err != nil { return res, err } - return builder.build.exec(query, args...) + return b.build.exec(query, args...) } -func (builder *accInsertBuilder) Run(args ...interface{}) (int, error) { - query, err := builder.build.adapter.SimpleInsert("", builder.table, builder.columns, builder.fields) +func (b *accInsertBuilder) Run(args ...interface{}) (int, error) { + query, err := b.build.adapter.SimpleInsert("", b.table, b.columns, b.fields) if err != nil { return 0, err } - res, err := builder.build.exec(query, args...) + res, err := b.build.exec(query, args...) if err != nil { return 0, err } diff --git a/query_gen/accumulator.go b/query_gen/accumulator.go index f2673deb..9d2999c3 100644 --- a/query_gen/accumulator.go +++ b/query_gen/accumulator.go @@ -224,7 +224,7 @@ func (build *Accumulator) PurgeTx(tx *sql.Tx, table string) (stmt *sql.Stmt) { } func (build *Accumulator) Delete(table string) *accDeleteBuilder { - return &accDeleteBuilder{table, "", build} + return &accDeleteBuilder{table, "", nil, build} } func (build *Accumulator) Update(table string) *accUpdateBuilder { diff --git a/query_gen/micro_builders.go b/query_gen/micro_builders.go index 796c99cc..48a96908 100644 --- a/query_gen/micro_builders.go +++ b/query_gen/micro_builders.go @@ -4,7 +4,7 @@ type dateCutoff struct { Column string Quantity int Unit string - Type int + Type int } type prebuilder struct { @@ -33,36 +33,45 @@ func (build *prebuilder) Update(nlist ...string) *updatePrebuilder { func (build *prebuilder) Delete(nlist ...string) *deletePrebuilder { name := optString(nlist, "") - return &deletePrebuilder{name, "", "", build.adapter} + return &deletePrebuilder{name, "", "", nil, build.adapter} } type deletePrebuilder struct { - name string - table string - where string + name string + table string + where string + dateCutoff *dateCutoff build Adapter } -func (delete *deletePrebuilder) Table(table string) *deletePrebuilder { - delete.table = table - return delete +func (b *deletePrebuilder) Table(table string) *deletePrebuilder { + b.table = table + return b } -func (delete *deletePrebuilder) Where(where string) *deletePrebuilder { - if delete.where != "" { - delete.where += " AND " +func (b *deletePrebuilder) Where(where string) *deletePrebuilder { + if b.where != "" { + b.where += " AND " } - delete.where += where - return delete + b.where += where + return b } -func (delete *deletePrebuilder) Text() (string, error) { - return delete.build.SimpleDelete(delete.name, delete.table, delete.where) +// TODO: We probably want to avoid the double allocation of two builders somehow +func (b *deletePrebuilder) FromAcc(acc *accDeleteBuilder) *deletePrebuilder { + b.table = acc.table + b.where = acc.where + b.dateCutoff = acc.dateCutoff + return b } -func (delete *deletePrebuilder) Parse() { - delete.build.SimpleDelete(delete.name, delete.table, delete.where) +func (b *deletePrebuilder) Text() (string, error) { + return b.build.SimpleDelete(b.name, b.table, b.where) +} + +func (b *deletePrebuilder) Parse() { + b.build.SimpleDelete(b.name, b.table, b.where) } type updatePrebuilder struct { @@ -70,7 +79,7 @@ type updatePrebuilder struct { table string set string where string - dateCutoff *dateCutoff // We might want to do this in a slightly less hacky way + dateCutoff *dateCutoff // We might want to do this in a slightly less hacky way whereSubQuery *selectPrebuilder build Adapter @@ -80,35 +89,35 @@ func qUpdate(table string, set string, where string) *updatePrebuilder { return &updatePrebuilder{table: table, set: set, where: where} } -func (update *updatePrebuilder) Table(table string) *updatePrebuilder { - update.table = table - return update +func (b *updatePrebuilder) Table(table string) *updatePrebuilder { + b.table = table + return b } -func (update *updatePrebuilder) Set(set string) *updatePrebuilder { - update.set = set - return update +func (b *updatePrebuilder) Set(set string) *updatePrebuilder { + b.set = set + return b } -func (update *updatePrebuilder) Where(where string) *updatePrebuilder { - if update.where != "" { - update.where += " AND " +func (b *updatePrebuilder) Where(where string) *updatePrebuilder { + if b.where != "" { + b.where += " AND " } - update.where += where - return update + b.where += where + return b } -func (update *updatePrebuilder) WhereQ(sel *selectPrebuilder) *updatePrebuilder { - update.whereSubQuery = sel - return update +func (b *updatePrebuilder) WhereQ(sel *selectPrebuilder) *updatePrebuilder { + b.whereSubQuery = sel + return b } -func (update *updatePrebuilder) Text() (string, error) { - return update.build.SimpleUpdate(update) +func (b *updatePrebuilder) Text() (string, error) { + return b.build.SimpleUpdate(b) } -func (update *updatePrebuilder) Parse() { - update.build.SimpleUpdate(update) +func (b *updatePrebuilder) Parse() { + b.build.SimpleUpdate(b) } type selectPrebuilder struct { @@ -125,22 +134,22 @@ type selectPrebuilder struct { build Adapter } -func (selectItem *selectPrebuilder) Table(table string) *selectPrebuilder { - selectItem.table = table - return selectItem +func (b *selectPrebuilder) Table(table string) *selectPrebuilder { + b.table = table + return b } -func (selectItem *selectPrebuilder) Columns(columns string) *selectPrebuilder { - selectItem.columns = columns - return selectItem +func (b *selectPrebuilder) Columns(columns string) *selectPrebuilder { + b.columns = columns + return b } -func (selectItem *selectPrebuilder) Where(where string) *selectPrebuilder { - if selectItem.where != "" { - selectItem.where += " AND " +func (b *selectPrebuilder) Where(where string) *selectPrebuilder { + if b.where != "" { + b.where += " AND " } - selectItem.where += where - return selectItem + b.where += where + return b } func (b *selectPrebuilder) InQ(subBuilder *selectPrebuilder) *selectPrebuilder { @@ -190,13 +199,13 @@ func (b *selectPrebuilder) FromCountAcc(acc *accCountBuilder) *selectPrebuilder } // TODO: Add support for dateCutoff -func (selectItem *selectPrebuilder) Text() (string, error) { - return selectItem.build.SimpleSelect(selectItem.name, selectItem.table, selectItem.columns, selectItem.where, selectItem.orderby, selectItem.limit) +func (b *selectPrebuilder) Text() (string, error) { + return b.build.SimpleSelect(b.name, b.table, b.columns, b.where, b.orderby, b.limit) } // TODO: Add support for dateCutoff -func (selectItem *selectPrebuilder) Parse() { - selectItem.build.SimpleSelect(selectItem.name, selectItem.table, selectItem.columns, selectItem.where, selectItem.orderby, selectItem.limit) +func (b *selectPrebuilder) Parse() { + b.build.SimpleSelect(b.name, b.table, b.columns, b.where, b.orderby, b.limit) } type insertPrebuilder struct { @@ -208,27 +217,27 @@ type insertPrebuilder struct { build Adapter } -func (insert *insertPrebuilder) Table(table string) *insertPrebuilder { - insert.table = table - return insert +func (b *insertPrebuilder) Table(table string) *insertPrebuilder { + b.table = table + return b } -func (insert *insertPrebuilder) Columns(columns string) *insertPrebuilder { - insert.columns = columns - return insert +func (b *insertPrebuilder) Columns(columns string) *insertPrebuilder { + b.columns = columns + return b } -func (insert *insertPrebuilder) Fields(fields string) *insertPrebuilder { - insert.fields = fields - return insert +func (b *insertPrebuilder) Fields(fields string) *insertPrebuilder { + b.fields = fields + return b } -func (insert *insertPrebuilder) Text() (string, error) { - return insert.build.SimpleInsert(insert.name, insert.table, insert.columns, insert.fields) +func (b *insertPrebuilder) Text() (string, error) { + return b.build.SimpleInsert(b.name, b.table, b.columns, b.fields) } -func (insert *insertPrebuilder) Parse() { - insert.build.SimpleInsert(insert.name, insert.table, insert.columns, insert.fields) +func (b *insertPrebuilder) Parse() { + b.build.SimpleInsert(b.name, b.table, b.columns, b.fields) } /*type countPrebuilder struct { @@ -240,30 +249,30 @@ func (insert *insertPrebuilder) Parse() { build Adapter } -func (count *countPrebuilder) Table(table string) *countPrebuilder { - count.table = table - return count +func (b *countPrebuilder) Table(table string) *countPrebuilder { + b.table = table + return b } -func (count *countPrebuilder) Where(where string) *countPrebuilder { - if count.where != "" { - count.where += " AND " +func b *countPrebuilder) Where(where string) *countPrebuilder { + if b.where != "" { + b.where += " AND " } - count.where += where - return count + b.where += where + return b } -func (count *countPrebuilder) Limit(limit string) *countPrebuilder { - count.limit = limit - return count +func (b *countPrebuilder) Limit(limit string) *countPrebuilder { + b.limit = limit + return b } -func (count *countPrebuilder) Text() (string, error) { - return count.build.SimpleCount(count.name, count.table, count.where, count.limit) +func (b *countPrebuilder) Text() (string, error) { + return b.build.SimpleCount(b.name, b.table, b.where, b.limit) } -func (count *countPrebuilder) Parse() { - count.build.SimpleCount(count.name, count.table, count.where, count.limit) +func (b *countPrebuilder) Parse() { + b.build.SimpleCount(b.name, b.table, b.where, b.limit) }*/ func optString(nlist []string, defaultStr string) string { diff --git a/query_gen/mssql.go b/query_gen/mssql.go index 12438c2b..9d07dc99 100644 --- a/query_gen/mssql.go +++ b/query_gen/mssql.go @@ -428,7 +428,7 @@ func (adapter *MssqlAdapter) SimpleUpdate(up *updatePrebuilder) (string, error) return querystr, nil } -func (adapter *MssqlAdapter) SimpleUpdateSelect(up *updatePrebuilder) (string, error) { +func (adapter *MssqlAdapter) SimpleUpdateSelect(b *updatePrebuilder) (string, error) { return "", errors.New("not implemented") } @@ -470,6 +470,10 @@ func (adapter *MssqlAdapter) SimpleDelete(name string, table string, where strin return querystr, nil } +func (adapter *MssqlAdapter) ComplexDelete(b *deletePrebuilder) (string, error) { + return "", errors.New("not implemented") +} + // We don't want to accidentally wipe tables, so we'll have a separate method for purging tables instead func (adapter *MssqlAdapter) Purge(name string, table string) (string, error) { if table == "" { diff --git a/query_gen/mysql.go b/query_gen/mysql.go index 30c0b964..0ff2847a 100644 --- a/query_gen/mysql.go +++ b/query_gen/mysql.go @@ -345,7 +345,7 @@ func (adapter *MysqlAdapter) SimpleUpsert(name string, table string, columns str if where == "" { return "", errors.New("You need a where for this upsert") } - + var querystr = "INSERT INTO `" + table + "`(" var parsedFields = processFields(fields) @@ -385,32 +385,32 @@ func (adapter *MysqlAdapter) SimpleUpdate(up *updatePrebuilder) (string, error) return "", errors.New("You need to set data in this update statement") } - var querystr = "UPDATE `" + up.table + "` SET " + var q = "UPDATE `" + up.table + "` SET " for _, item := range processSet(up.set) { - querystr += "`" + item.Column + "` =" + q += "`" + item.Column + "` =" for _, token := range item.Expr { switch token.Type { case "function", "operator", "number", "substitute", "or": - querystr += " " + token.Contents + q += " " + token.Contents case "column": - querystr += " `" + token.Contents + "`" + q += " `" + token.Contents + "`" case "string": - querystr += " '" + token.Contents + "'" + q += " '" + token.Contents + "'" } } - querystr += "," + q += "," } - querystr = querystr[0 : len(querystr)-1] + q = q[0 : len(q)-1] whereStr, err := adapter.buildFlexiWhere(up.where,up.dateCutoff) if err != nil { - return querystr, err + return q, err } - querystr += whereStr + q += whereStr // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator - adapter.pushStatement(up.name, "update", querystr) - return querystr, nil + adapter.pushStatement(up.name, "update", q) + return q, nil } func (adapter *MysqlAdapter) SimpleDelete(name string, table string, where string) (string, error) { @@ -421,29 +421,49 @@ func (adapter *MysqlAdapter) SimpleDelete(name string, table string, where strin return "", errors.New("You need to specify what data you want to delete") } - var querystr = "DELETE FROM `" + table + "` WHERE" + var q = "DELETE FROM `" + table + "` WHERE" // Add support for BETWEEN x.x for _, loc := range processWhere(where) { for _, token := range loc.Expr { switch token.Type { case "function", "operator", "number", "substitute", "or": - querystr += " " + token.Contents + q += " " + token.Contents case "column": - querystr += " `" + token.Contents + "`" + q += " `" + token.Contents + "`" case "string": - querystr += " '" + token.Contents + "'" + q += " '" + token.Contents + "'" default: panic("This token doesn't exist o_o") } } - querystr += " AND" + q += " AND" } - querystr = strings.TrimSpace(querystr[0 : len(querystr)-4]) + q = strings.TrimSpace(q[0 : len(q)-4]) // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator - adapter.pushStatement(name, "delete", querystr) - return querystr, nil + adapter.pushStatement(name, "delete", q) + return q, nil +} + +func (adapter *MysqlAdapter) ComplexDelete(b *deletePrebuilder) (string, error) { + if b.table == "" { + return "", errors.New("You need a name for this table") + } + if b.where == "" && b.dateCutoff == nil { + return "", errors.New("You need to specify what data you want to delete") + } + var q = "DELETE FROM `" + b.table + "`" + + whereStr, err := adapter.buildFlexiWhere(b.where, b.dateCutoff) + if err != nil { + return q, err + } + q += whereStr + + // TODO: Shunt the table name logic and associated stmt list up to the a higher layer to reduce the amount of unnecessary overhead in the builder / accumulator + adapter.pushStatement(b.name, "delete", q) + return q, nil } // We don't want to accidentally wipe tables, so we'll have a separate method for purging tables instead @@ -451,76 +471,80 @@ func (adapter *MysqlAdapter) Purge(name string, table string) (string, error) { if table == "" { return "", errors.New("You need a name for this table") } - adapter.pushStatement(name, "purge", "DELETE FROM `"+table+"`") - return "DELETE FROM `" + table + "`", nil + q := "DELETE FROM `"+table+"`" + adapter.pushStatement(name, "purge", q) + return q, nil } -func (adapter *MysqlAdapter) buildWhere(where string) (querystr string, err error) { +func (adapter *MysqlAdapter) buildWhere(where string) (q string, err error) { if len(where) == 0 { return "", nil } - querystr = " WHERE" + q = " WHERE" for _, loc := range processWhere(where) { for _, token := range loc.Expr { switch token.Type { case "function", "operator", "number", "substitute", "or": - querystr += " " + token.Contents + q += " " + token.Contents case "column": - querystr += " `" + token.Contents + "`" + q += " `" + token.Contents + "`" case "string": - querystr += " '" + token.Contents + "'" + q += " '" + token.Contents + "'" default: - return querystr, errors.New("This token doesn't exist o_o") + return q, errors.New("This token doesn't exist o_o") } } - querystr += " AND" + q += " AND" } - return querystr[0 : len(querystr)-4], nil + return q[0 : len(q)-4], nil } // The new version of buildWhere() currently only used in ComplexSelect for complex OO builder queries -func (adapter *MysqlAdapter) buildFlexiWhere(where string, dateCutoff *dateCutoff) (querystr string, err error) { +func (adapter *MysqlAdapter) buildFlexiWhere(where string, dateCutoff *dateCutoff) (q string, err error) { if len(where) == 0 && dateCutoff == nil { return "", nil } - querystr = " WHERE" + + q = " WHERE" if dateCutoff != nil { if dateCutoff.Type == 0 { - querystr += " " + dateCutoff.Column + " BETWEEN (UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + ") AND UTC_TIMESTAMP() AND" + q += " " + dateCutoff.Column + " BETWEEN (UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + ") AND UTC_TIMESTAMP() AND" } else { - querystr += " " + dateCutoff.Column + " < UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + " AND" + q += " " + dateCutoff.Column + " < UTC_TIMESTAMP() - interval " + strconv.Itoa(dateCutoff.Quantity) + " " + dateCutoff.Unit + " AND" } } + if len(where) != 0 { for _, loc := range processWhere(where) { for _, token := range loc.Expr { switch token.Type { case "function", "operator", "number", "substitute", "or": - querystr += " " + token.Contents + q += " " + token.Contents case "column": - querystr += " `" + token.Contents + "`" + q += " `" + token.Contents + "`" case "string": - querystr += " '" + token.Contents + "'" + q += " '" + token.Contents + "'" default: - return querystr, errors.New("This token doesn't exist o_o") + return q, errors.New("This token doesn't exist o_o") } } - querystr += " AND" + q += " AND" } } - return querystr[0 : len(querystr)-4], nil + + return q[0 : len(q)-4], nil } -func (adapter *MysqlAdapter) buildOrderby(orderby string) (querystr string) { +func (adapter *MysqlAdapter) buildOrderby(orderby string) (q string) { if len(orderby) != 0 { - querystr = " ORDER BY " + q = " ORDER BY " for _, column := range processOrderby(orderby) { // TODO: We might want to escape this column - querystr += "`" + strings.Replace(column.Column, ".", "`.`", -1) + "` " + strings.ToUpper(column.Order) + "," + q += "`" + strings.Replace(column.Column, ".", "`.`", -1) + "` " + strings.ToUpper(column.Order) + "," } - querystr = querystr[0 : len(querystr)-1] + q = q[0 : len(q)-1] } - return querystr + return q } func (adapter *MysqlAdapter) SimpleSelect(name string, table string, columns string, where string, orderby string, limit string) (string, error) { @@ -530,25 +554,23 @@ func (adapter *MysqlAdapter) SimpleSelect(name string, table string, columns str if len(columns) == 0 { return "", errors.New("No columns found for SimpleSelect") } - - var querystr = "SELECT " + var q = "SELECT " // Slice up the user friendly strings into something easier to process for _, column := range strings.Split(strings.TrimSpace(columns), ",") { - querystr += "`" + strings.TrimSpace(column) + "`," + q += "`" + strings.TrimSpace(column) + "`," } - querystr = querystr[0 : len(querystr)-1] + q = q[0 : len(q)-1] whereStr, err := adapter.buildWhere(where) if err != nil { - return querystr, err + return q, err } + q += " FROM `" + table + "`" + whereStr + adapter.buildOrderby(orderby) + adapter.buildLimit(limit) - querystr += " FROM `" + table + "`" + whereStr + adapter.buildOrderby(orderby) + adapter.buildLimit(limit) - - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(name, "select", querystr) - return querystr, nil + q = strings.TrimSpace(q) + adapter.pushStatement(name, "select", q) + return q, nil } func (adapter *MysqlAdapter) ComplexSelect(preBuilder *selectPrebuilder) (out string, err error) { @@ -558,35 +580,28 @@ func (adapter *MysqlAdapter) ComplexSelect(preBuilder *selectPrebuilder) (out st if len(preBuilder.columns) == 0 { return "", errors.New("No columns found for ComplexSelect") } - - var querystr = "SELECT " + adapter.buildJoinColumns(preBuilder.columns) - - // Slice up the user friendly strings into something easier to process - /*for _, column := range strings.Split(strings.TrimSpace(preBuilder.columns), ",") { - querystr += "`" + strings.TrimSpace(column) + "`," - } - querystr = querystr[0 : len(querystr)-1]*/ + var q = "SELECT " + adapter.buildJoinColumns(preBuilder.columns) var whereStr string // TODO: Let callers have a Where() and a InQ() if preBuilder.inChain != nil { whereStr, err = adapter.ComplexSelect(preBuilder.inChain) if err != nil { - return querystr, err + return q, err } whereStr = " WHERE `" + preBuilder.inColumn + "` IN(" + whereStr + ")" } else { whereStr, err = adapter.buildFlexiWhere(preBuilder.where, preBuilder.dateCutoff) if err != nil { - return querystr, err + return q, err } } - querystr += " FROM `" + preBuilder.table + "`" + whereStr + adapter.buildOrderby(preBuilder.orderby) + adapter.buildLimit(preBuilder.limit) + q += " FROM `" + preBuilder.table + "`" + whereStr + adapter.buildOrderby(preBuilder.orderby) + adapter.buildLimit(preBuilder.limit) - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(preBuilder.name, "select", querystr) - return querystr, nil + q = strings.TrimSpace(q) + adapter.pushStatement(preBuilder.name, "select", q) + return q, nil } func (adapter *MysqlAdapter) SimpleLeftJoin(name string, table1 string, table2 string, columns string, joiners string, where string, orderby string, limit string) (string, error) { @@ -678,11 +693,11 @@ func (adapter *MysqlAdapter) SimpleInsertSelect(name string, ins DBInsert, sel D return "", err } - var querystr = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table + "`" + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) + var q = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table + "`" + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(name, "insert", querystr) - return querystr, nil + q = strings.TrimSpace(q) + adapter.pushStatement(name, "insert", q) + return q, nil } func (adapter *MysqlAdapter) SimpleInsertLeftJoin(name string, ins DBInsert, sel DBJoin) (string, error) { @@ -691,59 +706,59 @@ func (adapter *MysqlAdapter) SimpleInsertLeftJoin(name string, ins DBInsert, sel return "", err } - var querystr = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table1 + "` LEFT JOIN `" + sel.Table2 + "` ON " + adapter.buildJoiners(sel.Joiners) + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) + var q = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table1 + "` LEFT JOIN `" + sel.Table2 + "` ON " + adapter.buildJoiners(sel.Joiners) + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(name, "insert", querystr) - return querystr, nil + q = strings.TrimSpace(q) + adapter.pushStatement(name, "insert", q) + return q, nil } // TODO: Make this more consistent with the other build* methods? -func (adapter *MysqlAdapter) buildJoiners(joiners string) (querystr string) { +func (adapter *MysqlAdapter) buildJoiners(joiners string) (q string) { for _, joiner := range processJoiner(joiners) { - querystr += "`" + joiner.LeftTable + "`.`" + joiner.LeftColumn + "` " + joiner.Operator + " `" + joiner.RightTable + "`.`" + joiner.RightColumn + "` AND " + q += "`" + joiner.LeftTable + "`.`" + joiner.LeftColumn + "` " + joiner.Operator + " `" + joiner.RightTable + "`.`" + joiner.RightColumn + "` AND " } // Remove the trailing AND - return querystr[0 : len(querystr)-4] + return q[0 : len(q)-4] } // Add support for BETWEEN x.x -func (adapter *MysqlAdapter) buildJoinWhere(where string) (querystr string, err error) { +func (adapter *MysqlAdapter) buildJoinWhere(where string) (q string, err error) { if len(where) != 0 { - querystr = " WHERE" + q = " WHERE" for _, loc := range processWhere(where) { for _, token := range loc.Expr { switch token.Type { case "function", "operator", "number", "substitute", "or": - querystr += " " + token.Contents + q += " " + token.Contents case "column": halves := strings.Split(token.Contents, ".") if len(halves) == 2 { - querystr += " `" + halves[0] + "`.`" + halves[1] + "`" + q += " `" + halves[0] + "`.`" + halves[1] + "`" } else { - querystr += " `" + token.Contents + "`" + q += " `" + token.Contents + "`" } case "string": - querystr += " '" + token.Contents + "'" + q += " '" + token.Contents + "'" default: - return querystr, errors.New("This token doesn't exist o_o") + return q, errors.New("This token doesn't exist o_o") } } - querystr += " AND" + q += " AND" } - querystr = querystr[0 : len(querystr)-4] + q = q[0 : len(q)-4] } - return querystr, nil + return q, nil } -func (adapter *MysqlAdapter) buildLimit(limit string) (querystr string) { +func (adapter *MysqlAdapter) buildLimit(limit string) (q string) { if limit != "" { - querystr = " LIMIT " + limit + q = " LIMIT " + limit } - return querystr + return q } -func (adapter *MysqlAdapter) buildJoinColumns(columns string) (querystr string) { +func (adapter *MysqlAdapter) buildJoinColumns(columns string) (q string) { for _, column := range processColumns(columns) { // TODO: Move the stirng and number logic to processColumns? // TODO: Error if [0] doesn't exist @@ -769,9 +784,9 @@ func (adapter *MysqlAdapter) buildJoinColumns(columns string) (querystr string) if column.Alias != "" { alias = " AS `" + column.Alias + "`" } - querystr += " " + source + alias + "," + q += " " + source + alias + "," } - return querystr[0 : len(querystr)-1] + return q[0 : len(q)-1] } func (adapter *MysqlAdapter) SimpleInsertInnerJoin(name string, ins DBInsert, sel DBJoin) (string, error) { @@ -780,27 +795,26 @@ func (adapter *MysqlAdapter) SimpleInsertInnerJoin(name string, ins DBInsert, se return "", err } - var querystr = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table1 + "` INNER JOIN `" + sel.Table2 + "` ON " + adapter.buildJoiners(sel.Joiners) + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) + var q = "INSERT INTO `" + ins.Table + "`(" + adapter.buildColumns(ins.Columns) + ") SELECT" + adapter.buildJoinColumns(sel.Columns) + " FROM `" + sel.Table1 + "` INNER JOIN `" + sel.Table2 + "` ON " + adapter.buildJoiners(sel.Joiners) + whereStr + adapter.buildOrderby(sel.Orderby) + adapter.buildLimit(sel.Limit) - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(name, "insert", querystr) - return querystr, nil + q = strings.TrimSpace(q) + adapter.pushStatement(name, "insert", q) + return q, nil } -func (adapter *MysqlAdapter) SimpleCount(name string, table string, where string, limit string) (querystr string, err error) { +func (adapter *MysqlAdapter) SimpleCount(name string, table string, where string, limit string) (q string, err error) { if table == "" { return "", errors.New("You need a name for this table") } - whereStr, err := adapter.buildWhere(where) if err != nil { return "", err } - querystr = "SELECT COUNT(*) AS `count` FROM `" + table + "`" + whereStr + adapter.buildLimit(limit) - querystr = strings.TrimSpace(querystr) - adapter.pushStatement(name, "select", querystr) - return querystr, nil + q = "SELECT COUNT(*) AS `count` FROM `" + table + "`" + whereStr + adapter.buildLimit(limit) + q = strings.TrimSpace(q) + adapter.pushStatement(name, "select", q) + return q, nil } func (adapter *MysqlAdapter) Builder() *prebuilder { diff --git a/query_gen/pgsql.go b/query_gen/pgsql.go index 54a87aaf..b8ff7a86 100644 --- a/query_gen/pgsql.go +++ b/query_gen/pgsql.go @@ -319,6 +319,17 @@ func (adapter *PgsqlAdapter) SimpleDelete(name string, table string, where strin return "", nil } +// TODO: Implement this +func (adapter *PgsqlAdapter) ComplexDelete(b *deletePrebuilder) (string, error) { + if b.table == "" { + return "", errors.New("You need a name for this table") + } + if b.where == "" { + return "", errors.New("You need to specify what data you want to delete") + } + return "", nil +} + // TODO: Implement this // We don't want to accidentally wipe tables, so we'll have a separate method for purging tables instead func (adapter *PgsqlAdapter) Purge(name string, table string) (string, error) { diff --git a/query_gen/querygen.go b/query_gen/querygen.go index f0c50ec0..11052ae9 100644 --- a/query_gen/querygen.go +++ b/query_gen/querygen.go @@ -117,11 +117,12 @@ type Adapter interface { AddKey(name string, table string, column string, key DBTableKey) (string, error) AddForeignKey(name string, table string, column string, ftable string, fcolumn string, cascade bool) (out string, e error) SimpleInsert(name string, table string, columns string, fields string) (string, error) - SimpleUpdate(up *updatePrebuilder) (string, error) - SimpleUpdateSelect(up *updatePrebuilder) (string, error) // ! Experimental + SimpleUpdate(b *updatePrebuilder) (string, error) + SimpleUpdateSelect(b *updatePrebuilder) (string, error) // ! Experimental SimpleDelete(name string, table string, where string) (string, error) Purge(name string, table string) (string, error) SimpleSelect(name string, table string, columns string, where string, orderby string, limit string) (string, error) + ComplexDelete(b *deletePrebuilder) (string, error) SimpleLeftJoin(name string, table1 string, table2 string, columns string, joiners string, where string, orderby string, limit string) (string, error) SimpleInnerJoin(string, string, string, string, string, string, string, string) (string, error) SimpleInsertSelect(string, DBInsert, DBSelect) (string, error) diff --git a/routes/panel/analytics.go b/routes/panel/analytics.go index 67abf364..0323b992 100644 --- a/routes/panel/analytics.go +++ b/routes/panel/analytics.go @@ -899,19 +899,35 @@ func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user c.User) c.Rout } ovList := analyticsVMapToOVList(vMap) + ex := strings.Split(r.FormValue("ex"), ",") + var inEx = func(name string) bool { + for _, e := range ex { + if e == name { + return true + } + } + return false + } + var vList [][]int64 var legendList []string var i int for _, ovitem := range ovList { + if inEx(ovitem.name) { + continue + } + lName, ok := phrases.GetUserAgentPhrase(ovitem.name) + if !ok { + lName = ovitem.name + } + if inEx(lName) { + continue + } var viewList []int64 for _, value := range revLabelList { viewList = append(viewList, ovitem.viewMap[value]) } vList = append(vList, viewList) - lName, ok := phrases.GetUserAgentPhrase(ovitem.name) - if !ok { - lName = ovitem.name - } legendList = append(legendList, lName) if i >= 6 { break @@ -924,10 +940,16 @@ func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user c.User) c.Rout // TODO: Sort this slice var agentItems []c.PanelAnalyticsAgentsItem for agent, count := range agentMap { + if inEx(agent) { + continue + } aAgent, ok := phrases.GetUserAgentPhrase(agent) if !ok { aAgent = agent } + if inEx(aAgent) { + continue + } agentItems = append(agentItems, c.PanelAnalyticsAgentsItem{ Agent: agent, FriendlyAgent: aAgent, diff --git a/tickloop.go b/tickloop.go index 4572143b..c3a963bf 100644 --- a/tickloop.go +++ b/tickloop.go @@ -164,6 +164,17 @@ func dailies() { c.LogError(err) } + if c.Config.LogPruneCutoff > -1 { + _, err := qgen.NewAcc().Delete("login_logs").DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run() + if err != nil { + c.LogError(err) + } + _, err = qgen.NewAcc().Delete("registration_logs").DateOlderThan("doneAt",c.Config.LogPruneCutoff,"day").Run() + if err != nil { + c.LogError(err) + } + } + if c.Config.PostIPCutoff > -1 { // TODO: Use unixtime to remove this MySQLesque logic? _, err := qgen.NewAcc().Update("topics").Set("ipaddress = '0'").DateOlderThan("createdAt",c.Config.PostIPCutoff,"day").Where("ipaddress != '0'").Exec()