From faf215f388b6cf1c4122e2dd020f645ff169d851 Mon Sep 17 00:00:00 2001 From: Azareal Date: Mon, 13 Apr 2020 13:28:52 +1000 Subject: [PATCH] try speeding up static files with brotli track ref routes for a bit to debug them hit more bots with MicroNotFound --- common/files.go | 92 +++++++++++++++++++++++++++++++++++++++------- common/theme.go | 17 ++++++++- gen_router.go | 7 ++++ go.mod | 1 + go.sum | 2 + router_gen/main.go | 7 ++++ routes/misc.go | 18 +++++++-- 7 files changed, 127 insertions(+), 17 deletions(-) diff --git a/common/files.go b/common/files.go index ede72b64..ba777444 100644 --- a/common/files.go +++ b/common/files.go @@ -17,6 +17,7 @@ import ( "sync" tmpl "github.com/Azareal/Gosora/tmpl_client" + "github.com/andybalholm/brotli" ) type SFileList map[string]SFile @@ -25,15 +26,22 @@ var StaticFiles SFileList = make(map[string]SFile) var staticFileMutex sync.RWMutex type SFile struct { - Data []byte - GzipData []byte - Sha256 string - OName string - Pos int64 - Length int64 - StrLength string - GzipLength int64 - StrGzipLength string + // TODO: Move these to the end? + Data []byte + GzipData []byte + BrData []byte + + Sha256 string + OName string + Pos int64 + + Length int64 + StrLength string + GzipLength int64 + StrGzipLength string + BrLength int64 + StrBrLength string + Mimetype string Info os.FileInfo FormattedModTime string @@ -254,6 +262,21 @@ func (list SFileList) JSTmplInit() error { path = tmplName + ".js" DebugLog("js path: ", path) ext := filepath.Ext("/tmpl_client/" + path) + + brData, err := CompressBytesBrotli(data) + if err != nil { + return err + } + // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses + if len(brData) >= (len(data) + 110) { + brData = nil + } else { + diff := len(data) - len(brData) + if diff <= len(data)/100 { + brData = nil + } + } + gzipData, err := CompressBytesGzip(data) if err != nil { return err @@ -273,7 +296,7 @@ func (list SFileList) JSTmplInit() error { hasher.Write(data) checksum := hex.EncodeToString(hasher.Sum(nil)) - list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) + list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) DebugLogf("Added the '%s' static file.", path) return nil @@ -304,8 +327,22 @@ func (list SFileList) Init() error { checksum := hex.EncodeToString(hasher.Sum(nil)) // Avoid double-compressing images - var gzipData []byte + var gzipData, brData []byte if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" { + brData, err = CompressBytesBrotli(data) + if err != nil { + return err + } + // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses + if len(brData) >= (len(data) + 130) { + brData = nil + } else { + diff := len(data) - len(brData) + if diff <= len(data)/100 { + brData = nil + } + } + gzipData, err = CompressBytesGzip(data) if err != nil { return err @@ -321,7 +358,7 @@ func (list SFileList) Init() error { } } - list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)}) + list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)}) DebugLogf("Added the '%s' static file.", path) return nil @@ -344,6 +381,21 @@ func (list SFileList) Add(path, prefix string) error { ext := filepath.Ext(path) path = strings.TrimPrefix(path, prefix) + + brData, err := CompressBytesBrotli(data) + if err != nil { + return err + } + // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses + if len(brData) >= (len(data) + 130) { + brData = nil + } else { + diff := len(data) - len(brData) + if diff <= len(data)/100 { + brData = nil + } + } + gzipData, err := CompressBytesGzip(data) if err != nil { return err @@ -363,7 +415,7 @@ func (list SFileList) Add(path, prefix string) error { hasher.Write(data) checksum := hex.EncodeToString(hasher.Sum(nil)) - list.Set("/s"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) + list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) DebugLogf("Added the '%s' static file", path) return nil @@ -398,3 +450,17 @@ func CompressBytesGzip(in []byte) ([]byte, error) { } return buff.Bytes(), nil } + +func CompressBytesBrotli(in []byte) ([]byte, error) { + var buff bytes.Buffer + br := brotli.NewWriterLevel(&buff, brotli.BestCompression) + _, err := br.Write(in) + if err != nil { + return nil, err + } + err = br.Close() + if err != nil { + return nil, err + } + return buff.Bytes(), nil +} diff --git a/common/theme.go b/common/theme.go index e9a955ee..cfcc4bd5 100644 --- a/common/theme.go +++ b/common/theme.go @@ -250,6 +250,21 @@ func (t *Theme) AddThemeStaticFiles() error { } path = strings.TrimPrefix(path, "themes/"+t.Name+"/public") + + brData, err := CompressBytesBrotli(data) + if err != nil { + return err + } + // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses + if len(brData) >= (len(data) + 130) { + brData = nil + } else { + diff := len(data) - len(brData) + if diff <= len(data)/100 { + brData = nil + } + } + gzipData, err := CompressBytesGzip(data) if err != nil { return err @@ -269,7 +284,7 @@ func (t *Theme) AddThemeStaticFiles() error { hasher.Write(data) checksum := hex.EncodeToString(hasher.Sum(nil)) - StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) + StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, brData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) DebugLog("Added the '/" + t.Name + path + "' static file for theme " + t.Name + ".") return nil diff --git a/gen_router.go b/gen_router.go index 562e0f72..aa3c4592 100644 --- a/gen_router.go +++ b/gen_router.go @@ -1244,6 +1244,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { portless := strings.Split(ref,":")[0] // TODO: Handle c.Site.Host in uppercase too? if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host { + r.DumpRequest(req,"Ref Route") co.ReferrerTracker.Bump(ref) } } @@ -2865,7 +2866,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } + r.DumpRequest(req,"Bad Route") + ae := req.Header.Get("Accept-Encoding") + likelyBot := ae == "gzip" || ae == "" + if likelyBot { + return c.MicroNotFound(w,req) + } return c.NotFound(w,req,nil) } return err diff --git a/go.mod b/go.mod index ef6803b7..90eff6f5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( cloud.google.com/go v0.31.0 // indirect github.com/Azareal/gopsutil v0.0.0-20170716174751-0763ca4e911d github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect + github.com/andybalholm/brotli v1.0.0 github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f github.com/fortytw2/leaktest v1.3.0 // indirect github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 2a73bb4d..9eee9293 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/router_gen/main.go b/router_gen/main.go index b90cfd25..120b9c9b 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -915,6 +915,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { portless := strings.Split(ref,":")[0] // TODO: Handle c.Site.Host in uppercase too? if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host { + r.DumpRequest(req,"Ref Route") co.ReferrerTracker.Bump(ref) } } @@ -1048,7 +1049,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user * r.SuspiciousRequest(req,"Bad Route") return c.MicroNotFound(w,req) } + r.DumpRequest(req,"Bad Route") + ae := req.Header.Get("Accept-Encoding") + likelyBot := ae == "gzip" || ae == "" + if likelyBot { + return c.MicroNotFound(w,req) + } return c.NotFound(w,req,nil) } return err diff --git a/routes/misc.go b/routes/misc.go index 772f25a1..d7165019 100644 --- a/routes/misc.go +++ b/routes/misc.go @@ -36,8 +36,16 @@ func StaticFile(w http.ResponseWriter, r *http.Request) { } else { h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000 } - - if file.GzipLength > 300 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + ae := r.Header.Get("Accept-Encoding") + + if file.BrLength > 300 && strings.Contains(ae, "br") { + h.Set("Content-Encoding", "br") + h.Set("Content-Length", file.StrBrLength) + http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.BrData)) + return + } + + if file.GzipLength > 300 && strings.Contains(ae, "gzip") { h.Set("Content-Encoding", "gzip") h.Set("Content-Length", file.StrGzipLength) http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.GzipData)) @@ -65,7 +73,11 @@ func StaticFile(w http.ResponseWriter, r *http.Request) { } h.Set("Vary", "Accept-Encoding") - if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + if file.BrLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "br") { + h.Set("Content-Encoding", "br") + h.Set("Content-Length", file.StrBrLength) + io.Copy(w, bytes.NewReader(file.BrData)) // Use w.Write instead? + } else if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { h.Set("Content-Encoding", "gzip") h.Set("Content-Length", file.StrGzipLength) io.Copy(w, bytes.NewReader(file.GzipData)) // Use w.Write instead?