From dbc6fdb27c7a27482164cc317b269b1abacd844c Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 5 May 2021 17:24:16 +1000 Subject: [PATCH] try to log panics better fix LogNewLongRoute --- common/alerts.go | 27 +++++++++++++++------------ common/common.go | 16 ++++++++++++---- common/counters/memory.go | 1 + common/counters/performance.go | 2 +- common/thumbnailer.go | 3 ++- common/tickloop.go | 6 ++++-- common/topic_list.go | 8 ++++---- common/websockets.go | 5 ++++- common/ws_hub.go | 1 + 9 files changed, 44 insertions(+), 25 deletions(-) diff --git a/common/alerts.go b/common/alerts.go index 4135745d..bb967a70 100644 --- a/common/alerts.go +++ b/common/alerts.go @@ -325,6 +325,7 @@ func AddActivityAndNotifyTarget(a Alert) error { // Live alerts, if the target is online and WebSockets is enabled if EnableWebsockets { go func() { + defer EatPanics() _ = WsHub.pushAlert(a.TargetUserID, a) //fmt.Println("err:",err) }() @@ -341,15 +342,18 @@ func NotifyWatchers(asid int) error { // Alert the subscribers about this without blocking us from doing something else if EnableWebsockets { - go notifyWatchers(asid) + go func() { + defer EatPanics() + notifyWatchers(asid) + }() } return nil } func notifyWatchers(asid int) { - rows, err := alertStmts.getWatchers.Query(asid) - if err != nil && err != ErrNoRows { - LogError(err) + rows, e := alertStmts.getWatchers.Query(asid) + if e != nil && e != ErrNoRows { + LogError(e) return } defer rows.Close() @@ -357,21 +361,20 @@ func notifyWatchers(asid int) { var uid int var uids []int for rows.Next() { - err := rows.Scan(&uid) - if err != nil { - LogError(err) + if e := rows.Scan(&uid); e != nil { + LogError(e) return } uids = append(uids, uid) } - if err = rows.Err(); err != nil { - LogError(err) + if e = rows.Err(); e != nil { + LogError(e) return } - alert, err := Activity.Get(asid) - if err != nil && err != ErrNoRows { - LogError(err) + alert, e := Activity.Get(asid) + if e != nil && e != ErrNoRows { + LogError(e) return } _ = WsHub.pushAlerts(uids, alert) diff --git a/common/common.go b/common/common.go index 950c1be4..366c2cf1 100644 --- a/common/common.go +++ b/common/common.go @@ -13,6 +13,7 @@ import ( "net" "net/http" "os" + "runtime/debug" "strconv" "strings" "sync/atomic" @@ -200,13 +201,12 @@ func DebugLogf(str string, args ...interface{}) { func Log(args ...interface{}) { log.Print(args...) } -func Err(args ...interface{}) { - ErrLogger.Print(args...) -} - func Logf(str string, args ...interface{}) { log.Printf(str, args...) } +func Err(args ...interface{}) { + ErrLogger.Print(args...) +} func Count(stmt *sql.Stmt) (count int) { e := stmt.QueryRow().Scan(&count) @@ -327,3 +327,11 @@ func (cw *ConnWatcher) StateChange(conn net.Conn, state http.ConnState) { func (cw *ConnWatcher) Count() int { return int(atomic.LoadInt64(&cw.n)) } + +func EatPanics() { + if r := recover(); r != nil { + log.Print(r) + debug.PrintStack() + log.Fatal("Fatal error.") + } +} diff --git a/common/counters/memory.go b/common/counters/memory.go index efe850a7..c947a90e 100644 --- a/common/counters/memory.go +++ b/common/counters/memory.go @@ -35,6 +35,7 @@ func NewMemoryCounter(acc *qgen.Accumulator) (*DefaultMemoryCounter, error) { c.Tasks.Shutdown.Add(co.Tick) ticker := time.NewTicker(time.Minute) go func() { + defer c.EatPanics() for { select { case <-ticker.C: diff --git a/common/counters/performance.go b/common/counters/performance.go index 4d6a8ceb..43357780 100644 --- a/common/counters/performance.go +++ b/common/counters/performance.go @@ -75,7 +75,7 @@ func (co *DefaultPerfCounter) insertChunk(low, high, avg int64) error { return nil } c.DebugLogf("Inserting a pchunk with low %d, high %d, avg %d", low, high, avg) - if c.Dev.LogNewLongRoute && (high*1000*1000) > 5 { + if c.Dev.LogNewLongRoute && high > (5*1000*1000) { c.Logf("pchunk high %d", high) } _, e := co.insert.Exec(low, high, avg) diff --git a/common/thumbnailer.go b/common/thumbnailer.go index 813fc062..a97659c0 100644 --- a/common/thumbnailer.go +++ b/common/thumbnailer.go @@ -15,13 +15,14 @@ import ( ) func ThumbTask(thumbChan chan bool) { + defer EatPanics() + acc := qgen.NewAcc() for { // Put this goroutine to sleep until we have work to do <-thumbChan // TODO: Use a real queue // TODO: Transactions? Self-repairing? - acc := qgen.NewAcc() err := acc.Select("users_avatar_queue").Columns("uid").Limit("0,5").EachInt(func(uid int) error { // TODO: Do a bulk user fetch instead? u, err := Users.Get(uid) diff --git a/common/tickloop.go b/common/tickloop.go index 279c462c..6af419da 100644 --- a/common/tickloop.go +++ b/common/tickloop.go @@ -10,6 +10,8 @@ import ( "github.com/pkg/errors" ) +var CTickLoop *TickLoop + type TickLoop struct { HalfSec *time.Ticker Sec *time.Ticker @@ -42,6 +44,7 @@ func (l *TickLoop) Loop() { LogError(e) } } + defer EatPanics() for { select { case <-l.HalfSec.C: @@ -199,8 +202,7 @@ func Dailies() (e error) { if e = Tasks.Day.Run(); e != nil { return e } - e = ForumActionStore.DailyTick() - if e != nil { + if e = ForumActionStore.DailyTick(); e != nil { return e } diff --git a/common/topic_list.go b/common/topic_list.go index c3310ffb..0645780c 100644 --- a/common/topic_list.go +++ b/common/topic_list.go @@ -77,11 +77,11 @@ func NewDefaultTopicList(acc *qgen.Accumulator) (*DefaultTopicList, error) { getTopicsByForum: acc.Select("topics").Columns("tid,title,content,createdBy,is_closed,sticky,createdAt,lastReplyAt,lastReplyBy,lastReplyID,views,postCount,likeCount").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(), //getTidsByForum: acc.Select("topics").Columns("tid").Where("parentID=?").Orderby("sticky DESC,lastReplyAt DESC,createdBy DESC").Limit("?,?").Prepare(), } - if err := acc.FirstError(); err != nil { - return nil, err + if e := acc.FirstError(); e != nil { + return nil, e } - if err := tList.Tick(); err != nil { - return nil, err + if e := tList.Tick(); e != nil { + return nil, e } Tasks.HalfSec.Add(tList.Tick) diff --git a/common/websockets.go b/common/websockets.go index 3fb66939..8dd598e8 100644 --- a/common/websockets.go +++ b/common/websockets.go @@ -196,7 +196,10 @@ func wsPageResponses(wsUser *WSUser, conn *websocket.Conn, page string) { watchers := len(adminStatsWatchers) adminStatsWatchers[conn] = wsUser if watchers == 0 { - go adminStatsTicker() + go func() { + defer EatPanics() + adminStatsTicker() + }() } adminStatsMutex.Unlock() default: diff --git a/common/ws_hub.go b/common/ws_hub.go index 4dd48d16..306ffd24 100644 --- a/common/ws_hub.go +++ b/common/ws_hub.go @@ -46,6 +46,7 @@ func (h *WsHubImpl) Start() { }() go func() { + defer EatPanics() for { item := func(l *sync.RWMutex, userMap map[int]*WSUser) { l.RLock()