Add Config.LogDir and Config.DisableSuspLog settings.

Reduce default Config.PostIPCutoff from 120 to 90.
Add another startup log call.
This commit is contained in:
Azareal 2021-03-24 18:08:37 +10:00
parent 1d747a6f68
commit d47540391a
9 changed files with 50 additions and 15 deletions

View File

@ -134,6 +134,9 @@ type config struct {
ReadTimeout int ReadTimeout int
WriteTimeout int WriteTimeout int
IdleTimeout int IdleTimeout int
LogDir string
DisableSuspLog bool
} }
type devConfig struct { type devConfig struct {
@ -295,7 +298,7 @@ func ProcessConfig() (err error) {
guestAvatar = GuestAvatar{buildNoavatar(0, 200), buildNoavatar(0, 48)} guestAvatar = GuestAvatar{buildNoavatar(0, 200), buildNoavatar(0, 48)}
if Config.PostIPCutoff == 0 { if Config.PostIPCutoff == 0 {
Config.PostIPCutoff = 120 // Default cutoff Config.PostIPCutoff = 90 // Default cutoff
} }
if Config.LogPruneCutoff == 0 { if Config.LogPruneCutoff == 0 {
Config.LogPruneCutoff = 180 // Default cutoff Config.LogPruneCutoff = 180 // Default cutoff
@ -326,6 +329,10 @@ func ProcessConfig() (err error) {
// TODO: Set the alternate hash algo, e.g. argon2 // TODO: Set the alternate hash algo, e.g. argon2
} }
if Config.LogDir == "" {
Config.LogDir = "./logs/"
}
// We need this in here rather than verifyConfig as switchToTestDB() currently overwrites the values it verifies // We need this in here rather than verifyConfig as switchToTestDB() currently overwrites the values it verifies
if DbConfig.TestDbname == DbConfig.Dbname { if DbConfig.TestDbname == DbConfig.Dbname {
return errors.New("Your test database can't have the same name as your production database") return errors.New("Your test database can't have the same name as your production database")

View File

@ -177,7 +177,7 @@ func CompileTemplates() error {
SuperDebug: Dev.TemplateDebug, SuperDebug: Dev.TemplateDebug,
DockToID: DockToID, DockToID: DockToID,
} }
c := tmpl.NewCTemplateSet("normal") c := tmpl.NewCTemplateSet("normal", "./logs/")
c.SetConfig(config) c.SetConfig(config)
c.SetBaseImportMap(map[string]string{ c.SetBaseImportMap(map[string]string{
"io": "io", "io": "io",
@ -482,7 +482,7 @@ func CompileJSTemplates() error {
PackageName: "tmpl", PackageName: "tmpl",
DockToID: DockToID, DockToID: DockToID,
} }
c := tmpl.NewCTemplateSet("js") c := tmpl.NewCTemplateSet("js", "./logs/")
c.SetConfig(config) c.SetConfig(config)
c.SetBuildTags("!no_templategen") c.SetBuildTags("!no_templategen")
c.SetOverrideTrack(overriden) c.SetOverrideTrack(overriden)

View File

@ -49,6 +49,7 @@ type CTemplateConfig struct {
type CTemplateSet struct { type CTemplateSet struct {
templateList map[string]*parse.Tree templateList map[string]*parse.Tree
fileDir string fileDir string
logDir string
funcMap map[string]interface{} funcMap map[string]interface{}
importMap map[string]string importMap map[string]string
//templateFragmentCount map[string]int //templateFragmentCount map[string]int
@ -79,8 +80,12 @@ type CTemplateSet struct {
lang string lang string
} }
func NewCTemplateSet(in string) *CTemplateSet { func NewCTemplateSet(in string, logDir ...string) *CTemplateSet {
f, err := os.OpenFile("./logs/tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) var llogDir string
if len(logDir) > 0 {
llogDir = logDir[0]
}
f, err := os.OpenFile(llogDir+"tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -88,6 +93,7 @@ func NewCTemplateSet(in string) *CTemplateSet {
config: CTemplateConfig{ config: CTemplateConfig{
PackageName: "main", PackageName: "main",
}, },
logDir: llogDir,
baseImportMap: map[string]string{}, baseImportMap: map[string]string{},
overridenRoots: map[string]map[string]bool{}, overridenRoots: map[string]map[string]bool{},
funcMap: map[string]interface{}{ funcMap: map[string]interface{}{
@ -163,7 +169,7 @@ func (c *CTemplateSet) SetPerThemeTmpls(perThemeTmpls map[string]bool) {
} }
func (c *CTemplateSet) ResetLogs(in string) { func (c *CTemplateSet) ResetLogs(in string) {
f, err := os.OpenFile("./logs/tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) f, err := os.OpenFile(c.logDir+"tmpls-"+in+"-"+strconv.FormatInt(time.Now().Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
panic(err) panic(err)
} }

View File

@ -88,7 +88,7 @@ ServerCount - The number of instances you're running. This setting is currently
LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature. LastIPCutoff - The number of months which need to pass before the last IP stored for a user is automatically deleted. Capped at 12. 0 defaults to whatever the current default is, currently 3 and -1 disables this feature.
PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 120 and -1 disables this feature. PostIPCutoff - The number of days which need to pass before the IP data for a post is automatically deleted. 0 defaults to whatever the current default is, currently 90 and -1 disables this feature.
PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently 90 and -1 disables this feature. PollIPCutoff - The number of days which need to pass before the IP data for a poll is automatically deleted. 0 defaults to whatever the current default is, currently 90 and -1 disables this feature.
@ -146,6 +146,10 @@ WriteTimeout - The number of seconds that a route is allowed to run for before t
IdleTimeout - The number of seconds that a Keep-Alive connection will be kept open before being closed. You might to tweak this, if you use Cloudflare or similar. Defaults to 120. IdleTimeout - The number of seconds that a Keep-Alive connection will be kept open before being closed. You might to tweak this, if you use Cloudflare or similar. Defaults to 120.
LogDir - The directory in which logs are stored. Default: ./logs/
DisableSuspLog - Whether suspicious requests are logged in the suspicious request logs. Enabling this may make a site faster. Defaults to false.
Related: https://support.cloudflare.com/hc/en-us/articles/212794707-General-Best-Practices-for-Load-Balancing-at-your-origin-with-Cloudflare Related: https://support.cloudflare.com/hc/en-us/articles/212794707-General-Best-Practices-for-Load-Balancing-at-your-origin-with-Cloudflare

View File

@ -930,6 +930,9 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
if c.Config.DisableSuspLog {
return
}
var sb strings.Builder var sb strings.Builder
if pre != "" { if pre != "" {
sb.WriteString("Suspicious Request\n") sb.WriteString("Suspicious Request\n")
@ -2792,6 +2795,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
} }
co.RouteViewCounter.Bump3(174, cn) co.RouteViewCounter.Bump3(174, cn)
if !c.Config.DisableSuspLog {
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"w") { if strings.Contains(lp,"w") {
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
@ -2803,6 +2807,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req) return c.MicroNotFound(w,req)
} }
}
r.DumpRequest(req,"Bad Route") r.DumpRequest(req,"Bad Route")
ae := req.Header.Get("Accept-Encoding") ae := req.Header.Get("Accept-Encoding")

View File

@ -64,6 +64,7 @@ func afterDBInit() (err error) {
var uids []int var uids []int
tc := c.Topics.GetCache() tc := c.Topics.GetCache()
if tc != nil { if tc != nil {
log.Print("Preloading topics")
// Preload ten topics to get the wheels going // Preload ten topics to get the wheels going
var count = 10 var count = 10
if tc.GetCapacity() <= 10 { if tc.GetCapacity() <= 10 {
@ -356,7 +357,7 @@ func main() {
// TODO: Have a file for each run with the time/date the server started as the file name? // TODO: Have a file for each run with the time/date the server started as the file name?
// TODO: Log panics with recover() // TODO: Log panics with recover()
f, err := os.OpenFile("./logs/ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) f, err := os.OpenFile(c.Config.LogDir+"ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -367,7 +368,7 @@ func main() {
// TODO: Add a flag for enabling the profiler // TODO: Add a flag for enabling the profiler
if false { if false {
f, err := os.Create("./logs/cpu.prof") f, err := os.Create(c.Config.LogDir + "cpu.prof")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -581,7 +582,7 @@ func main() {
c.WsHub.Start() c.WsHub.Start()
if false { if false {
f, err := os.Create("./logs/cpu.prof") f, err := os.Create(c.Config.LogDir + "cpu.prof")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -595,7 +596,7 @@ func main() {
args := <-c.StopServerChan args := <-c.StopServerChan
if false { if false {
pprof.StopCPUProfile() pprof.StopCPUProfile()
f, err := os.Create("./logs/mem.prof") f, err := os.Create(c.Config.LogDir + "mem.prof")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

View File

@ -73,7 +73,7 @@ func (r *GenRouter) DailyTick() error {
} }
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
f, err = os.OpenFile("./logs/reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) f, err = os.OpenFile(c.Config.LogDir+"reqs-susp-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
return err return err
} }
@ -87,7 +87,7 @@ func (r *GenRouter) DailyTick() error {
func NewGenRouter(uploads http.Handler) (*GenRouter, error) { func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
stimestr := strconv.FormatInt(c.StartTime.Unix(), 10) stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
createLog := func(name, stimestr string) (*RouterLog, error) { createLog := func(name, stimestr string) (*RouterLog, error) {
f, err := os.OpenFile("./logs/"+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) f, err := os.OpenFile(c.Config.LogDir+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -106,7 +106,7 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
f3, err := os.OpenFile("./logs/reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) f3, err := os.OpenFile(c.Config.LogDir+"reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -243,6 +243,13 @@ func (r *GenRouter) susp1(req *http.Request) bool {
} }
func (r *GenRouter) suspScan(req *http.Request) { func (r *GenRouter) suspScan(req *http.Request) {
if c.Config.DisableSuspLog {
if c.Dev.FullReqLog {
r.DumpRequest(req, "")
}
return
}
// TODO: Cover more suspicious strings and at a lower layer than this // TODO: Cover more suspicious strings and at a lower layer than this
var ch rune var ch rune
var susp bool var susp bool

View File

@ -519,6 +519,9 @@ func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
if c.Config.DisableSuspLog {
return
}
var sb strings.Builder var sb strings.Builder
if pre != "" { if pre != "" {
sb.WriteString("Suspicious Request\n") sb.WriteString("Suspicious Request\n")
@ -956,6 +959,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
} }
co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn)
if !c.Config.DisableSuspLog {
lp := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lp,"w") { if strings.Contains(lp,"w") {
if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") {
@ -967,6 +971,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req) return c.MicroNotFound(w,req)
} }
}
r.DumpRequest(req,"Bad Route") r.DumpRequest(req,"Bad Route")
ae := req.Header.Get("Accept-Encoding") ae := req.Header.Get("Accept-Encoding")

View File

@ -116,7 +116,7 @@ func Debug(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError {
staticSize := dirSize("./public/") staticSize := dirSize("./public/")
attachSize := dirSize("./attachs/") attachSize := dirSize("./attachs/")
uploadsSize := dirSize("./uploads/") uploadsSize := dirSize("./uploads/")
logsSize := dirSize("./logs/") logsSize := dirSize(c.Config.LogDir)
backupsSize := dirSize("./backups/") backupsSize := dirSize("./backups/")
if fErr != nil { if fErr != nil {
return c.InternalError(fErr, w, r) return c.InternalError(fErr, w, r)