gosora/main.go

/*
*
*    Gosora Main File
*    Copyright Azareal 2016 - 2020
*
 */
// Package main contains the main initialisation logic for Gosora
package main // import "git.tuxpa.in/a/gosora"

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"syscall"
	"time"

	c "git.tuxpa.in/a/gosora/common"
	co "git.tuxpa.in/a/gosora/common/counters"
	meta "git.tuxpa.in/a/gosora/common/meta"
	p "git.tuxpa.in/a/gosora/common/phrases"
	_ "git.tuxpa.in/a/gosora/extend"
	qgen "git.tuxpa.in/a/gosora/query_gen"
	"git.tuxpa.in/a/gosora/routes"
	"git.tuxpa.in/a/gosora/uutils"
	"github.com/fsnotify/fsnotify"

	//"github.com/lucas-clemente/quic-go/http3"
	"github.com/pkg/errors"
)

var router *GenRouter

// TODO: Wrap the globals in here so we can pass pointers to them to subpackages
var globs *Globs

type Globs struct {
    stmts *Stmts
}

// Temporary alias for renderTemplate
func init() {
    c.RenderTemplateAlias = routes.RenderTemplate
}

func afterDBInit() (err error) {
    if err := storeInit(); err != nil {
        return err
    }
    log.Print("Exitted storeInit")

    c.GzipStartEtag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-ng\""
    c.StartEtag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-n\""

    var uids []int
    tc := c.Topics.GetCache()
    if tc != nil {
        log.Print("Preloading topics")
        // Preload ten topics to get the wheels going
        var count = 10
        if tc.GetCapacity() <= 10 {
            count = 2
            if tc.GetCapacity() <= 2 {
                count = 0
            }
        }
        group, err := c.Groups.Get(c.GuestUser.Group)
        if err != nil {
            return err
        }

        // TODO: Use the same cached data for both the topic list and the topic fetches...
        tList, _, _, err := c.TopicList.GetListByCanSee(group.CanSee, 1, 0, nil)
        if err != nil {
            return err
        }
        ctList := make([]*c.TopicsRow, len(tList))
        copy(ctList, tList)

        tList, _, _, err = c.TopicList.GetListByCanSee(group.CanSee, 2, 0, nil)
        if err != nil {
            return err
        }
        for _, tItem := range tList {
            ctList = append(ctList, tItem)
        }

        tList, _, _, err = c.TopicList.GetListByCanSee(group.CanSee, 3, 0, nil)
        if err != nil {
            return err
        }
        for _, tItem := range tList {
            ctList = append(ctList, tItem)
        }

        if count > len(ctList) {
            count = len(ctList)
        }
        for i := 0; i < count; i++ {
            _, _ = c.Topics.Get(ctList[i].ID)
        }
    }

    uc := c.Users.GetCache()
    if uc != nil {
        // Preload associated users too...
        for _, uid := range uids {
            _, _ = c.Users.Get(uid)
        }
    }

    log.Print("Exitted afterDBInit")
    return nil
}

// Experimenting with a new error package here to try to reduce the amount of debugging we have to do
// TODO: Dynamically register these items to avoid maintaining as much code here?
func storeInit() (e error) {
    acc := qgen.NewAcc()
    ws := errors.WithStack
    var rcache c.ReplyCache
    if c.Config.ReplyCache == "static" {
        rcache = c.NewMemoryReplyCache(c.Config.ReplyCacheCapacity)
    }
    c.Rstore, e = c.NewSQLReplyStore(acc, rcache)
    if e != nil {
        return ws(e)
    }
    c.Prstore, e = c.NewSQLProfileReplyStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Likes, e = c.NewDefaultLikeStore(acc)
    if e != nil {
        return ws(e)
    }
    c.ForumActionStore, e = c.NewDefaultForumActionStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Convos, e = c.NewDefaultConversationStore(acc)
    if e != nil {
        return ws(e)
    }
    c.UserBlocks, e = c.NewDefaultBlockStore(acc)
    if e != nil {
        return ws(e)
    }
    c.GroupPromotions, e = c.NewDefaultGroupPromotionStore(acc)
    if e != nil {
        return ws(e)
    }

    if e = p.InitPhrases(c.Site.Language); e != nil {
        return ws(e)
    }
    if e = c.InitEmoji(); e != nil {
        return ws(e)
    }
    if e = c.InitWeakPasswords(); e != nil {
        return ws(e)
    }

    log.Print("Loading the static files.")
    if e = c.Themes.LoadStaticFiles(); e != nil {
        return ws(e)
    }
    if e = c.StaticFiles.Init(); e != nil {
        return ws(e)
    }
    if e = c.StaticFiles.JSTmplInit(); e != nil {
        return ws(e)
    }

    log.Print("Initialising the widgets")
    c.Widgets = c.NewDefaultWidgetStore()
    if e = c.InitWidgets(); e != nil {
        return ws(e)
    }

    log.Print("Initialising the menu item list")
    c.Menus = c.NewDefaultMenuStore()
    if e = c.Menus.Load(1); e != nil { // 1 = the default menu
        return ws(e)
    }
    menuHold, e := c.Menus.Get(1)
    if e != nil {
        return ws(e)
    }
    fmt.Printf("menuHold: %+v\n", menuHold)
    var b bytes.Buffer
    menuHold.Build(&b, &c.GuestUser, "/")
    fmt.Println("menuHold output: ", string(b.Bytes()))

    log.Print("Initialising the authentication system")
    c.Auth, e = c.NewDefaultAuth()
    if e != nil {
        return ws(e)
    }

    log.Print("Initialising the stores")
    c.WordFilters, e = c.NewDefaultWordFilterStore(acc)
    if e != nil {
        return ws(e)
    }
    c.MFAstore, e = c.NewSQLMFAStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Pages, e = c.NewDefaultPageStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Reports, e = c.NewDefaultReportStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Emails, e = c.NewDefaultEmailStore(acc)
    if e != nil {
        return ws(e)
    }
    c.LoginLogs, e = c.NewLoginLogStore(acc)
    if e != nil {
        return ws(e)
    }
    c.RegLogs, e = c.NewRegLogStore(acc)
    if e != nil {
        return ws(e)
    }
    c.ModLogs, e = c.NewModLogStore(acc)
    if e != nil {
        return ws(e)
    }
    c.AdminLogs, e = c.NewAdminLogStore(acc)
    if e != nil {
        return ws(e)
    }
    c.IPSearch, e = c.NewDefaultIPSearcher()
    if e != nil {
        return ws(e)
    }
    if c.Config.Search == "" || c.Config.Search == "sql" {
        c.RepliesSearch, e = c.NewSQLSearcher(acc)
        if e != nil {
            return ws(e)
        }
    }
    c.Subscriptions, e = c.NewDefaultSubscriptionStore()
    if e != nil {
        return ws(e)
    }
    c.Attachments, e = c.NewDefaultAttachmentStore(acc)
    if e != nil {
        return ws(e)
    }
    c.Polls, e = c.NewDefaultPollStore(c.NewMemoryPollCache(100)) // TODO: Max number of polls held in cache, make this a config item
    if e != nil {
        return ws(e)
    }
    c.TopicList, e = c.NewDefaultTopicList(acc)
    if e != nil {
        return ws(e)
    }
    c.PasswordResetter, e = c.NewDefaultPasswordResetter(acc)
    if e != nil {
        return ws(e)
    }
    c.Analytics = c.NewDefaultAnalytics()
    c.Activity, e = c.NewDefaultActivityStream(acc)
    if e != nil {
        return ws(e)
    }
    c.ActivityMatches, e = c.NewDefaultActivityStreamMatches(acc)
    if e != nil {
        return ws(e)
    }
    // TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins
    c.Thumbnailer = c.NewCaireThumbnailer()
    c.Recalc, e = c.NewDefaultRecalc(acc)
    if e != nil {
        return ws(e)
    }

    log.Print("Initialising the meta store")
    c.Meta, e = meta.NewDefaultMetaStore(acc)
    if e != nil {
        return ws(e)
    }

    log.Print("Initialising the view counters")
    if !c.Config.DisableAnalytics {
        co.GlobalViewCounter, e = co.NewGlobalViewCounter(acc)
        if e != nil {
            return ws(e)
        }
        co.AgentViewCounter, e = co.NewDefaultAgentViewCounter(acc)
        if e != nil {
            return ws(e)
        }
        co.OSViewCounter, e = co.NewDefaultOSViewCounter(acc)
        if e != nil {
            return ws(e)
        }
        co.LangViewCounter, e = co.NewDefaultLangViewCounter(acc)
        if e != nil {
            return ws(e)
        }
        if !c.Config.RefNoTrack {
            co.ReferrerTracker, e = co.NewDefaultReferrerTracker()
            if e != nil {
                return ws(e)
            }
        }
        co.MemoryCounter, e = co.NewMemoryCounter(acc)
        if e != nil {
            return ws(e)
        }
        co.PerfCounter, e = co.NewDefaultPerfCounter(acc)
        if e != nil {
            return ws(e)
        }
    }
    co.RouteViewCounter, e = co.NewDefaultRouteViewCounter(acc)
    if e != nil {
        return ws(e)
    }
    co.PostCounter, e = co.NewPostCounter()
    if e != nil {
        return ws(e)
    }
    co.TopicCounter, e = co.NewTopicCounter()
    if e != nil {
        return ws(e)
    }
    co.TopicViewCounter, e = co.NewDefaultTopicViewCounter()
    if e != nil {
        return ws(e)
    }
    co.ForumViewCounter, e = co.NewDefaultForumViewCounter()
    if e != nil {
        return ws(e)
    }

    return nil
}

// TODO: Split this function up
func main() {
    // TODO: Recover from panics
    defer func() {
        if r := recover(); r != nil {
            log.Print(r)
            debug.PrintStack()
            log.Fatal("Fatal error.")
        }
    }()
    c.StartTime = time.Now()

    // TODO: Have a file for each run with the time/date the server started as the file name?
    // TODO: Log panics with recover()
    f, err := os.OpenFile("./logs/ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
    if err != nil {
        log.Fatal(err)
    }
    //c.LogWriter = io.MultiWriter(os.Stderr, f)
    c.LogWriter = io.MultiWriter(os.Stdout, f)
    c.ErrLogWriter = io.MultiWriter(os.Stderr, f)
    log.SetOutput(c.LogWriter)
    c.ErrLogger = log.New(c.ErrLogWriter, "", log.LstdFlags)
    log.Print("Running Gosora v" + c.SoftwareVersion.String())
    fmt.Println("")

    // TODO: Add a flag for enabling the profiler
    if false {
        f, err := os.Create(c.Config.LogDir + "cpu.prof")
        if err != nil {
            log.Fatal(err)
        }
        pprof.StartCPUProfile(f)
    }

    err = mime.AddExtensionType(".avif", "image/avif")
    if err != nil {
        log.Fatal(err)
    }

    jsToken, err := c.GenerateSafeString(80)
    if err != nil {
        log.Fatal(err)
    }
    c.JSTokenBox.Store(jsToken)

    log.Print("Loading the configuration data")
    err = c.LoadConfig()
    if err != nil {
        log.Fatal(err)
    }
    log.Print("Processing configuration data")
    err = c.ProcessConfig()
    if err != nil {
        log.Fatal(err)
    }
    if c.Config.DisableStdout {
        c.LogWriter = f
        log.SetOutput(c.LogWriter)
    }
    if c.Config.DisableStderr {
        c.ErrLogWriter = f
        c.ErrLogger = log.New(c.ErrLogWriter, "", log.LstdFlags)
    }
    c.Tasks = c.NewScheduledTasks()

    err = c.InitTemplates()
    if err != nil {
        log.Fatal(err)
    }
    c.Themes, err = c.NewThemeList()
    if err != nil {
        log.Fatal(err)
    }
    c.TopicListThaw = c.NewSingleServerThaw()

    err = InitDatabase()
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    buildTemplates := flag.Bool("build-templates", false, "build the templates")
    flag.Parse()
    if *buildTemplates {
        if err = c.CompileTemplates(); err != nil {
            log.Fatal(err)
        }
        if err = c.CompileJSTemplates(); err != nil {
            log.Fatal(err)
        }
        return
    }

    err = afterDBInit()
    if err != nil {
        log.Fatalf("%+v", err)
    }
    err = c.VerifyConfig()
    if err != nil {
        log.Fatal(err)
    }

    if !c.Dev.NoFsnotify {
        log.Print("Initialising the file watcher")
        watcher, err := fsnotify.NewWatcher()
        if err != nil {
            log.Fatal(err)
        }
        defer watcher.Close()

        go func() {
            defer c.EatPanics()
            var ErrFileSkip = errors.New("skip mod file")
            modifiedFileEvent := func(path string) error {
                pathBits := strings.Split(path, "\\")
                if len(pathBits) == 0 {
                    return nil
                }
                if pathBits[0] == "themes" {
                    var themeName string
                    if len(pathBits) >= 2 {
                        themeName = pathBits[1]
                    }
                    if len(pathBits) >= 3 && pathBits[2] == "public" {
                        // TODO: Handle new themes freshly plopped into the folder?
                        theme, ok := c.Themes[themeName]
                        if ok {
                            return theme.LoadStaticFiles()
                        }
                    }
                }
                return ErrFileSkip
            }

            // TODO: Expand this to more types of files
            var err error
            for {
                select {
                case ev := <-watcher.Events:
                    // TODO: Handle file deletes (and renames more graciously by removing the old version of it)
                    if ev.Op&fsnotify.Write == fsnotify.Write {
                        err = modifiedFileEvent(ev.Name)
                        if err != ErrFileSkip {
                            log.Println("modified file:", ev.Name)
                        } else {
                            err = nil
                        }
                    } else if ev.Op&fsnotify.Create == fsnotify.Create {
                        log.Println("new file:", ev.Name)
                        err = modifiedFileEvent(ev.Name)
                    } else {
                        log.Println("unknown event:", ev)
                        err = nil
                    }
                    if err != nil {
                        c.LogError(err)
                    }
                case err = <-watcher.Errors:
                    c.LogWarning(err)
                }
            }
        }()

        // TODO: Keep tabs on the (non-resource) theme stuff, and the langpacks
        err = watcher.Add("./public")
        if err != nil {
            log.Fatal(err)
        }
        err = watcher.Add("./templates")
        if err != nil {
            log.Fatal(err)
        }
        for _, theme := range c.Themes {
            err = watcher.Add("./themes/" + theme.Name + "/public")
            if err != nil {
                log.Fatal(err)
            }
        }
    }

    /*if err = c.StaticFiles.GenJS(); err != nil {
        c.LogError(err)
    }*/

    log.Print("Checking for init tasks")
    if err = sched(); err != nil {
        c.LogError(err)
    }

    log.Print("Initialising the task system")

    // Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests
    // TODO: Could we expand this to attachments and other things too?
    thumbChan := make(chan bool)
    go c.ThumbTask(thumbChan)
    if err = tickLoop(thumbChan); err != nil {
        c.LogError(err)
    }
    go TickLoop.Loop()

    // Resource Management Goroutine
    go func() {
        defer c.EatPanics()
        uc, tc := c.Users.GetCache(), c.Topics.GetCache()
        if uc == nil && tc == nil {
            return
        }

        var lastEvictedCount int
        var couldNotDealloc bool
        secondTicker := time.NewTicker(time.Second)
        for {
            select {
            case <-secondTicker.C:
                // TODO: Add a LastRequested field to cached User structs to avoid evicting the same things which wind up getting loaded again anyway?
                if uc != nil {
                    ucap := uc.GetCapacity()
                    if uc.Length() <= ucap || c.Users.Count() <= ucap {
                        couldNotDealloc = false
                        continue
                    }
                    lastEvictedCount = uc.DeallocOverflow(couldNotDealloc)
                    couldNotDealloc = (lastEvictedCount == 0)
                }
            }
        }
    }()

    log.Print("Initialising the router")
    router, err = NewGenRouter(&RouterConfig{
        Uploads: http.FileServer(http.Dir("./uploads")),
    })
    if err != nil {
        log.Fatal(err)
    }

    log.Print("Initialising the plugins")
    c.InitPlugins()

    log.Print("Setting up the signal handler")
    sigs := make(chan os.Signal, 1)
    signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
    go func() {
        defer c.EatPanics()
        sig := <-sigs
        log.Print("Received a signal to shutdown: ", sig)
        // TODO: Gracefully shutdown the HTTP server
        tw, cn := c.NewTickWatch(), uutils.Nanotime()
        tw.Name = "shutdown"
        tw.Set(&tw.Start, cn)
        tw.Set(&tw.DBCheck, cn)
        tw.Run()
        n, e := func() (string, error) {
            if e := runHook("before_shutdown_tick"); e != nil {
                return "before_shutdown_tick ", e
            }
            tw.Set(&tw.StartHook, uutils.Nanotime())
            log.Print("Running shutdown tasks")
            if e := c.Tasks.Shutdown.Run(); e != nil {
                return "shutdown tasks ", e
            }
            tw.Set(&tw.Tasks, uutils.Nanotime())
            log.Print("Ran shutdown tasks")
            if e := runHook("after_shutdown_tick"); e != nil {
                return "after_shutdown_tick ", e
            }
            tw.Set(&tw.EndHook, uutils.Nanotime())
            return "", nil
        }()
        if e != nil {
            log.Print(n+" err:", e)
        }
        tw.Stop()
        log.Print("Stopping server")
        c.StoppedServer("Stopped server")
    }()

    // Start up the WebSocket ticks
    c.WsHub.Start()

    if false {
        f, e := os.Create(c.Config.LogDir + "cpu.prof")
        if e != nil {
            log.Fatal(e)
        }
        pprof.StartCPUProfile(f)
    }

    //if profiling {
    //    pprof.StopCPUProfile()
    //}
    startServer()
    args := <-c.StopServerChan
    if false {
        pprof.StopCPUProfile()
        f, err := os.Create(c.Config.LogDir + "mem.prof")
        if err != nil {
            log.Fatal(err)
        }
        defer f.Close()

        runtime.GC()
        err = pprof.WriteHeapProfile(f)
        if err != nil {
            log.Fatal(err)
        }
    }
    // Why did the server stop?
    log.Fatal(args...)
}

func startServer() {
    // We might not need timeouts, if we're behind a reverse-proxy like Nginx
    newServer := func(addr string, h http.Handler) *http.Server {
        f := func(timeout, dval int) int {
            if timeout == 0 {
                timeout = dval
            } else if timeout == -1 {
                timeout = 0
            }
            return timeout
        }
        rtime := f(c.Config.ReadTimeout, 8)
        wtime := f(c.Config.WriteTimeout, 10)
        itime := f(c.Config.IdleTimeout, 120)
        return &http.Server{
            Addr:      addr,
            Handler:   h,
            ConnState: c.ConnWatch.StateChange,

            ReadTimeout:  time.Duration(rtime) * time.Second,
            WriteTimeout: time.Duration(wtime) * time.Second,
            IdleTimeout:  time.Duration(itime) * time.Second,

            TLSConfig: &tls.Config{
                PreferServerCipherSuites: true,
                CurvePreferences: []tls.CurveID{
                    tls.CurveP256,
                    tls.X25519,
                },
            },
        }
    }

    // TODO: Let users run *both* HTTP and HTTPS
    log.Print("Initialising the HTTP server")
    /*if c.Dev.QuicPort != 0 {
        sQuicPort := strconv.Itoa(c.Dev.QuicPort)
        log.Print("Listening on quic port " + sQuicPort)
        go func() {
            defer c.EatPanics()
            c.StoppedServer(http3.ListenAndServeQUIC(":"+sQuicPort, c.Config.SslFullchain, c.Config.SslPrivkey, router))
        }()
    }*/

    if !c.Site.EnableSsl {
        if c.Site.Port == "" {
            c.Site.Port = "80"
        }
        log.Print("Listening on port " + c.Site.Port)
        go func() {
            defer c.EatPanics()
            c.StoppedServer(newServer(":"+c.Site.Port, router).ListenAndServe())
        }()
        return
    }

    if c.Site.Port == "" {
        c.Site.Port = "443"
    }
    if c.Site.Port == "80" || c.Site.Port == "443" {
        // We should also run the server on port 80
        // TODO: Redirect to port 443
        go func() {
            defer c.EatPanics()
            log.Print("Listening on port 80")
            c.StoppedServer(newServer(":80", &HTTPSRedirect{}).ListenAndServe())
        }()
    }
    log.Printf("Listening on port %s", c.Site.Port)
    go func() {
        defer c.EatPanics()
        c.StoppedServer(newServer(":"+c.Site.Port, router).ListenAndServeTLS(c.Config.SslFullchain, c.Config.SslPrivkey))
    }()
}