Add Dev.HourDBTimeout unlisted config setting.

Shorter duration during conn drop period.
Improve TickWatch.DumpElapsed formatting.
This commit is contained in:
Azareal 2021-06-14 13:32:07 +10:00
parent 9ba6e269c6
commit 7c5d55e0b9
5 changed files with 35 additions and 7 deletions

View File

@ -161,6 +161,8 @@ type devConfig struct {
LogLongTick bool // unlisted setting LogLongTick bool // unlisted setting
LogNewLongRoute bool // unlisted setting LogNewLongRoute bool // unlisted setting
Log4thLongRoute bool // unlisted setting Log4thLongRoute bool // unlisted setting
HourDBTimeout bool // unlisted setting
} }
// configHolder is purely for having a big struct to unmarshal data into // configHolder is purely for having a big struct to unmarshal data into

View File

@ -67,13 +67,21 @@ func (l *TickLoop) Loop() {
var ErrDBDown = errors.New("The database is down") var ErrDBDown = errors.New("The database is down")
func DBTimeout() time.Duration {
if Dev.HourDBTimeout {
return time.Hour
}
//db.SetConnMaxLifetime(time.Second * 60 * 5) // Make this infinite as the temporary lifetime change will purge the stale connections?
return -1
}
func StartTick() (abort bool) { func StartTick() (abort bool) {
db := qgen.Builder.GetConn() db := qgen.Builder.GetConn()
isDBDown := atomic.LoadInt32(&IsDBDown) isDBDown := atomic.LoadInt32(&IsDBDown)
if e := db.Ping(); e != nil { if e := db.Ping(); e != nil {
// TODO: There's a bit of a race here, but it doesn't matter if this error appears multiple times in the logs as it's capped at three times, we just want to cut it down 99% of the time // TODO: There's a bit of a race here, but it doesn't matter if this error appears multiple times in the logs as it's capped at three times, we just want to cut it down 99% of the time
if isDBDown == 0 { if isDBDown == 0 {
db.SetConnMaxLifetime(time.Second / 2) // Drop all the connections and start over db.SetConnMaxLifetime(time.Second / 4) // Drop all the connections and start over
LogWarning(e, ErrDBDown.Error()) LogWarning(e, ErrDBDown.Error())
} }
atomic.StoreInt32(&IsDBDown, 1) atomic.StoreInt32(&IsDBDown, 1)
@ -82,8 +90,7 @@ func StartTick() (abort bool) {
if isDBDown == 1 { if isDBDown == 1 {
log.Print("The database is back") log.Print("The database is back")
} }
//db.SetConnMaxLifetime(time.Second * 60 * 5) // Make this infinite as the temporary lifetime change will purge the stale connections? db.SetConnMaxLifetime(DBTimeout())
db.SetConnMaxLifetime(-1)
atomic.StoreInt32(&IsDBDown, 0) atomic.StoreInt32(&IsDBDown, 0)
return false return false
} }
@ -250,6 +257,7 @@ func (w *TickWatch) DumpElapsed() {
elapse := func(name string, bef, v int64) { elapse := func(name string, bef, v int64) {
if bef == 0 || v == 0 { if bef == 0 || v == 0 {
ff("%s: %d\n", v) ff("%s: %d\n", v)
return
} }
dur := time.Duration(v - bef) dur := time.Duration(v - bef)
milli := dur.Milliseconds() milli := dur.Milliseconds()
@ -267,8 +275,20 @@ func (w *TickWatch) DumpElapsed() {
f("Name: " + w.Name + "\n") f("Name: " + w.Name + "\n")
ff("Start: %d\n", w.Start) ff("Start: %d\n", w.Start)
elapse("DBCheck", w.Start, w.DBCheck) elapse("DBCheck", w.Start, w.DBCheck)
if w.DBCheck == 0 {
Log(sb.String())
return
}
elapse("StartHook", w.DBCheck, w.StartHook) elapse("StartHook", w.DBCheck, w.StartHook)
if w.StartHook == 0 {
Log(sb.String())
return
}
elapse("Tasks", w.StartHook, w.Tasks) elapse("Tasks", w.StartHook, w.Tasks)
if w.Tasks == 0 {
Log(sb.String())
return
}
elapse("EndHook", w.Tasks, w.EndHook) elapse("EndHook", w.Tasks, w.EndHook)
Log(sb.String()) Log(sb.String())
@ -287,11 +307,13 @@ func (w *TickWatch) Run() {
select { select {
case <-w.Ticker.C: case <-w.Ticker.C:
// Less noisy logs // Less noisy logs
if n > 2 && n%2 != 0 { if n > 20 && n%5 == 0 {
Logf("%d seconds elapsed since tick %s started", 5*n, w.Name)
} else if n > 2 && n%2 != 0 {
Logf("%d seconds elapsed since tick %s started", 5*n, w.Name) Logf("%d seconds elapsed since tick %s started", 5*n, w.Name)
} }
if !downOnce && w.DBCheck == 0 { if !downOnce && w.DBCheck == 0 {
qgen.Builder.GetConn().SetConnMaxLifetime(time.Second / 2) // Drop all the connections and start over qgen.Builder.GetConn().SetConnMaxLifetime(time.Second / 4) // Drop all the connections and start over
LogWarning(ErrDBDown) LogWarning(ErrDBDown)
atomic.StoreInt32(&IsDBDown, 1) atomic.StoreInt32(&IsDBDown, 1)
downOnce = true downOnce = true

View File

@ -13,7 +13,8 @@ import (
"net/url" "net/url"
"github.com/Azareal/Gosora/common" "github.com/Azareal/Gosora/common"
"github.com/Azareal/Gosora/query_gen" c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen"
_ "github.com/denisenkom/go-mssqldb" _ "github.com/denisenkom/go-mssqldb"
) )
@ -52,6 +53,7 @@ func initMSSQL() (err error) {
// Only hold connections open for five seconds to avoid accumulating a large number of stale connections // Only hold connections open for five seconds to avoid accumulating a large number of stale connections
//db.SetConnMaxLifetime(5 * time.Second) //db.SetConnMaxLifetime(5 * time.Second)
db.SetConnMaxLifetime(c.DBTimeout())
// Build the generated prepared statements, we are going to slowly move the queries over to the query generator rather than writing them all by hand, this'll make it easier for us to implement database adapters for other databases like PostgreSQL, MSSQL, SQlite, etc. // Build the generated prepared statements, we are going to slowly move the queries over to the query generator rather than writing them all by hand, this'll make it easier for us to implement database adapters for other databases like PostgreSQL, MSSQL, SQlite, etc.
err = _gen_mssql() err = _gen_mssql()

View File

@ -45,6 +45,7 @@ func initMySQL() (err error) {
// Only hold connections open for five seconds to avoid accumulating a large number of stale connections // Only hold connections open for five seconds to avoid accumulating a large number of stale connections
//db.SetConnMaxLifetime(5 * time.Second) //db.SetConnMaxLifetime(5 * time.Second)
db.SetConnMaxLifetime(c.DBTimeout())
// Build the generated prepared statements, we are going to slowly move the queries over to the query generator rather than writing them all by hand, this'll make it easier for us to implement database adapters for other databases like PostgreSQL, MSSQL, SQlite, etc. // Build the generated prepared statements, we are going to slowly move the queries over to the query generator rather than writing them all by hand, this'll make it easier for us to implement database adapters for other databases like PostgreSQL, MSSQL, SQlite, etc.
err = _gen_mysql() err = _gen_mysql()

View File

@ -9,7 +9,7 @@ import (
"strings" "strings"
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
"github.com/Azareal/Gosora/query_gen" qgen "github.com/Azareal/Gosora/query_gen"
_ "github.com/lib/pq" _ "github.com/lib/pq"
) )
@ -45,6 +45,7 @@ func initPgsql() (err error) {
// Only hold connections open for five seconds to avoid accumulating a large number of stale connections // Only hold connections open for five seconds to avoid accumulating a large number of stale connections
//db.SetConnMaxLifetime(5 * time.Second) //db.SetConnMaxLifetime(5 * time.Second)
db.SetConnMaxLifetime(c.DBTimeout())
err = _gen_pgsql() err = _gen_pgsql()
if err != nil { if err != nil {