From d47540391aee6fa8d9a159f7dff70eab60081701 Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 24 Mar 2021 18:08:37 +1000 Subject: [PATCH] Add Config.LogDir and Config.DisableSuspLog settings. Reduce default Config.PostIPCutoff from 120 to 90. Add another startup log call. --- common/site.go | 9 ++++++++- common/template_init.go | 4 ++-- common/templates/templates.go | 12 +++++++++--- docs/configuration.md | 6 +++++- gen_router.go | 5 +++++ main.go | 9 +++++---- router.go | 13 ++++++++++--- router_gen/main.go | 5 +++++ routes/panel/debug.go | 2 +- 9 files changed, 50 insertions(+), 15 deletions(-) diff --git a/common/site.go b/common/site.go index 923aeecc..d9f4a7a0 100644 --- a/common/site.go +++ b/common/site.go @@ -134,6 +134,9 @@ type config struct { ReadTimeout int WriteTimeout int IdleTimeout int + + LogDir string + DisableSuspLog bool } type devConfig struct { @@ -295,7 +298,7 @@ func ProcessConfig() (err error) { guestAvatar = GuestAvatar{buildNoavatar(0, 200), buildNoavatar(0, 48)} if Config.PostIPCutoff == 0 { - Config.PostIPCutoff = 120 // Default cutoff + Config.PostIPCutoff = 90 // Default cutoff } if Config.LogPruneCutoff == 0 { Config.LogPruneCutoff = 180 // Default cutoff @@ -326,6 +329,10 @@ func ProcessConfig() (err error) { // TODO: Set the alternate hash algo, e.g. argon2 } + if Config.LogDir == "" { + Config.LogDir = "./logs/" + } + // We need this in here rather than verifyConfig as switchToTestDB() currently overwrites the values it verifies if DbConfig.TestDbname == DbConfig.Dbname { return errors.New("Your test database can't have the same name as your production database") diff --git a/common/template_init.go b/common/template_init.go index fc051c5f..c5c32608 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -177,7 +177,7 @@ func CompileTemplates() error { SuperDebug: Dev.TemplateDebug, DockToID: DockToID, } - c := tmpl.NewCTemplateSet("normal") + c := tmpl.NewCTemplateSet("normal", "./logs/") c.SetConfig(config) c.SetBaseImportMap(map[string]string{ "io": "io", @@ -482,7 +482,7 @@ func CompileJSTemplates() error { PackageName: "tmpl", DockToID: DockToID, } - c := tmpl.NewCTemplateSet("js") + c := tmpl.NewCTemplateSet("js", "./logs/") c.SetConfig(config) c.SetBuildTags("!no_templategen") c.SetOverrideTrack(overriden) diff --git a/common/templates/templates.go b/common/templates/templates.go index 69ad0ab5..909c3015 100644 --- a/common/templates/templates.go +++ b/common/templates/templates.go @@ -49,6 +49,7 @@ type CTemplateConfig struct { type CTemplateSet struct { templateList map[string]*parse.Tree fileDir string + logDir string funcMap map[string]interface{} importMap map[string]string //templateFragmentCount map[string]int @@ -79,8 +80,12 @@ type CTemplateSet struct { lang string } -func NewCTemplateSet(in string) *CTemplateSet { - f, err := os.OpenFile("./logs/tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) +func NewCTemplateSet(in string, logDir ...string) *CTemplateSet { + var llogDir string + if len(logDir) > 0 { + llogDir = logDir[0] + } + f, err := os.OpenFile(llogDir+"tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { panic(err) } @@ -88,6 +93,7 @@ func NewCTemplateSet(in string) *CTemplateSet { config: CTemplateConfig{ PackageName: "main", }, + logDir: llogDir, baseImportMap: map[string]string{}, overridenRoots: map[string]map[string]bool{}, funcMap: map[string]interface{}{ @@ -163,7 +169,7 @@ func (c *CTemplateSet) SetPerThemeTmpls(perThemeTmpls map[string]bool) { } func (c *CTemplateSet) ResetLogs(in string) { - f, err := os.OpenFile("./logs/tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + f, err := os.OpenFile(c.logDir+"tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { panic(err) } diff --git a/docs/configuration.md b/docs/configuration.md index 91770ef2..f1739bb3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,7 +88,7 @@ ServerCount - The number of instances you're running. This setting is currently LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature. -PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. +PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 90 and -1 disables this feature. PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently 90 and -1 disables this feature. @@ -146,6 +146,10 @@ WriteTimeout - The number of seconds that a route is allowed to run for before t IdleTimeout - The number of seconds that a Keep-Alive connection will be kept open before being closed. You might to tweak this, if you use Cloudflare or similar. Defaults to 120. +LogDir - The directory in which logs are stored. Default: ./logs/ + +DisableSuspLog - Whether suspicious requests are logged in the suspicious request logs. Enabling this may make a site faster. Defaults to false. + Related: https://support.cloudflare.com/hc/en-us/articles/212794707-General-Best-Practices-for-Load-Balancing-at-your-origin-with-Cloudflare diff --git a/gen_router.go b/gen_router.go index 47616cc2..54d02f99 100644 --- a/gen_router.go +++ b/gen_router.go @@ -930,6 +930,9 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { } func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { + if c.Config.DisableSuspLog { + return + } var sb strings.Builder if pre != "" { sb.WriteString("Suspicious Request\n") @@ -2792,6 +2795,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } co.RouteViewCounter.Bump3(174, cn) + if !c.Config.DisableSuspLog { lp := strings.ToLower(req.URL.Path) if strings.Contains(lp,"w") { if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { @@ -2803,6 +2807,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } + } r.DumpRequest(req,"Bad Route") ae := req.Header.Get("Accept-Encoding") diff --git a/main.go b/main.go index 178fa681..09607c2a 100644 --- a/main.go +++ b/main.go @@ -64,6 +64,7 @@ func afterDBInit() (err error) { var uids []int tc := c.Topics.GetCache() if tc != nil { + log.Print("Preloading topics") // Preload ten topics to get the wheels going var count = 10 if tc.GetCapacity() <= 10 { @@ -356,7 +357,7 @@ func main() { // TODO: Have a file for each run with the time/date the server started as the file name? // TODO: Log panics with recover() - f, err := os.OpenFile("./logs/ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + f, err := os.OpenFile(c.Config.LogDir+"ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { log.Fatal(err) } @@ -367,7 +368,7 @@ func main() { // TODO: Add a flag for enabling the profiler if false { - f, err := os.Create("./logs/cpu.prof") + f, err := os.Create(c.Config.LogDir + "cpu.prof") if err != nil { log.Fatal(err) } @@ -581,7 +582,7 @@ func main() { c.WsHub.Start() if false { - f, err := os.Create("./logs/cpu.prof") + f, err := os.Create(c.Config.LogDir + "cpu.prof") if err != nil { log.Fatal(err) } @@ -595,7 +596,7 @@ func main() { args := <-c.StopServerChan if false { pprof.StopCPUProfile() - f, err := os.Create("./logs/mem.prof") + f, err := os.Create(c.Config.LogDir + "mem.prof") if err != nil { log.Fatal(err) } diff --git a/router.go b/router.go index 817cebdb..13971971 100644 --- a/router.go +++ b/router.go @@ -73,7 +73,7 @@ func (r *GenRouter) DailyTick() error { } stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) - f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + f, err = os.OpenFile(c.Config.LogDir+"reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { return err } @@ -87,7 +87,7 @@ func (r *GenRouter) DailyTick() error { func NewGenRouter(uploads http.Handler) (*GenRouter, error) { stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) createLog := func(name, stimestr string) (*RouterLog, error) { - f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + f, err := os.OpenFile(c.Config.LogDir+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) { if err != nil { return nil, err } - f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + f3, err := os.OpenFile(c.Config.LogDir+"reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) if err != nil { return nil, err } @@ -243,6 +243,13 @@ func (r *GenRouter) susp1(req *http.Request) bool { } func (r *GenRouter) suspScan(req *http.Request) { + if c.Config.DisableSuspLog { + if c.Dev.FullReqLog { + r.DumpRequest(req, "") + } + return + } + // TODO: Cover more suspicious strings and at a lower layer than this var ch rune var susp bool diff --git a/router_gen/main.go b/router_gen/main.go index 9e12a685..ca41676f 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -519,6 +519,9 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { } func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { + if c.Config.DisableSuspLog { + return + } var sb strings.Builder if pre != "" { sb.WriteString("Suspicious Request\n") @@ -956,6 +959,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * } co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) + if !c.Config.DisableSuspLog { lp := strings.ToLower(req.URL.Path) if strings.Contains(lp,"w") { if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { @@ -967,6 +971,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } + } r.DumpRequest(req,"Bad Route") ae := req.Header.Get("Accept-Encoding") diff --git a/routes/panel/debug.go b/routes/panel/debug.go index 93b33b78..0317516d 100644 --- a/routes/panel/debug.go +++ b/routes/panel/debug.go @@ -116,7 +116,7 @@ func Debug(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { staticSize := dirSize("./public/") attachSize := dirSize("./attachs/") uploadsSize := dirSize("./uploads/") - logsSize := dirSize("./logs/") + logsSize := dirSize(c.Config.LogDir) backupsSize := dirSize("./backups/") if fErr != nil { return c.InternalError(fErr, w, r)