Database connection drops are handled better now.

common/common.go

@@ -28,6 +28,7 @@ var TmplPtrMap = make(map[string]interface{})
 var JSTokenBox atomic.Value              // TODO: Move this and some of these other globals somewhere else
 var SessionSigningKeyBox atomic.Value    // For MFA to avoid hitting the database unneccesarily
 var OldSessionSigningKeyBox atomic.Value // Just in case we've signed with a key that's about to go stale so we don't annoy the user too much
+var IsDBDown int32 = 0                   // 0 = false, 1 = true. this is value which should be manipulated with package atomic for representing whether the database is down so we don't spam the log with lots of redundant errors
 
 // ErrNoRows is an alias of sql.ErrNoRows, just in case we end up with non-database/sql datastores
 var ErrNoRows = sql.ErrNoRows

common/errors.go

@@ -124,6 +124,13 @@ func InternalErrorJS(err error, w http.ResponseWriter, r *http.Request) RouteErr
 	return HandledRouteError()
 }
 
+// When the task system detects if the database is down, some database errors might lip by this
+func DatabaseError(w http.ResponseWriter, r *http.Request) RouteError {
+	pi := ErrorPage{errorHeader(w, GuestUser, "Internal Server Error"), "A problem has occurred in the system."}
+	handleErrorTemplate(w, r, pi)
+	return HandledRouteError()
+}
+
 var xmlInternalError = []byte(`<?xml version="1.0" encoding="UTF-8"?>
 <error>A problem has occured</error>`)
 

gen_router.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"errors"
 	"os"
 	"net/http"
@@ -701,6 +702,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		routes.StaticFile(w, req)
 		return
 	}
+	if atomic.LoadInt32(&common.IsDBDown) == 1 {
+		common.DatabaseError(w, req)
+		return
+	}
 	if common.Dev.SuperDebug {
 		router.requestLogger.Print("before PreRoute")
 	}

main.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -346,6 +347,7 @@ func main() {
 	}
 
 	log.Print("Initialising the task system")
+	// TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there?
 	var runTasks = func(tasks []func() error) {
 		for _, task := range tasks {
 			if task() != nil {
@@ -361,6 +363,24 @@ func main() {
 	fifteenMinuteTicker := time.NewTicker(15 * time.Minute)
 	hourTicker := time.NewTicker(time.Hour)
 	go func() {
+		var startTick = func() (abort bool) {
+			var isDBDown = atomic.LoadInt32(&common.IsDBDown)
+			err := db.Ping()
+			if err != nil {
+				// TODO: There's a bit of a race here, but it doesn't matter if this error appears multiple times in the logs as it's capped at three times, we just want to cut it down 99% of the time
+				if isDBDown == 0 {
+					common.LogWarning(err)
+					common.LogWarning(errors.New("The database is down"))
+				}
+				atomic.StoreInt32(&common.IsDBDown, 1)
+				return true
+			}
+			if isDBDown == 1 {
+				log.Print("The database is back")
+			}
+			atomic.StoreInt32(&common.IsDBDown, 0)
+			return false
+		}
 		var runHook = func(name string) {
 			err := common.RunTaskHook(name)
 			if err != nil {
@@ -370,10 +390,16 @@ func main() {
 		for {
 			select {
 			case <-halfSecondTicker.C:
+				if startTick() {
+					continue
+				}
 				runHook("before_half_second_tick")
 				runTasks(common.ScheduledHalfSecondTasks)
 				runHook("after_half_second_tick")
 			case <-secondTicker.C:
+				if startTick() {
+					continue
+				}
 				runHook("before_second_tick")
 				runTasks(common.ScheduledSecondTasks)
 
@@ -397,6 +423,9 @@ func main() {
 				// TODO: Rescan the static files for changes
 				runHook("after_second_tick")
 			case <-fifteenMinuteTicker.C:
+				if startTick() {
+					continue
+				}
 				runHook("before_fifteen_minute_tick")
 				runTasks(common.ScheduledFifteenMinuteTasks)
 
@@ -404,6 +433,9 @@ func main() {
 				// TODO: Publish scheduled posts.
 				runHook("after_fifteen_minute_tick")
 			case <-hourTicker.C:
+				if startTick() {
+					continue
+				}
 				runHook("before_hour_tick")
 
 				jsToken, err := common.GenerateSafeString(80)

router_gen/main.go

@@ -225,6 +225,7 @@ import (
 	"strings"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"errors"
 	"os"
 	"net/http"
@@ -481,6 +482,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		routes.StaticFile(w, req)
 		return
 	}
+	if atomic.LoadInt32(&common.IsDBDown) == 1 {
+		common.DatabaseError(w, req)
+		return
+	}
 	if common.Dev.SuperDebug {
 		router.requestLogger.Print("before PreRoute")
 	}