try speeding up static files with brotli

track ref routes for a bit to debug them
hit more bots with MicroNotFound

common/files.go

@@ -17,6 +17,7 @@ import (
 	"sync"
 
 	tmpl "github.com/Azareal/Gosora/tmpl_client"
+	"github.com/andybalholm/brotli"
 )
 
 type SFileList map[string]SFile
@@ -25,15 +26,22 @@ var StaticFiles SFileList = make(map[string]SFile)
 var staticFileMutex sync.RWMutex
 
 type SFile struct {
-	Data             []byte
+	// TODO: Move these to the end?
-	GzipData         []byte
+	Data     []byte
-	Sha256           string
+	GzipData []byte
-	OName            string
+	BrData   []byte
-	Pos              int64
+
-	Length           int64
+	Sha256 string
-	StrLength        string
+	OName  string
-	GzipLength       int64
+	Pos    int64
-	StrGzipLength    string
+
+	Length        int64
+	StrLength     string
+	GzipLength    int64
+	StrGzipLength string
+	BrLength      int64
+	StrBrLength   string
+
 	Mimetype         string
 	Info             os.FileInfo
 	FormattedModTime string
@@ -254,6 +262,21 @@ func (list SFileList) JSTmplInit() error {
 		path = tmplName + ".js"
 		DebugLog("js path: ", path)
 		ext := filepath.Ext("/tmpl_client/" + path)
+
+		brData, err := CompressBytesBrotli(data)
+		if err != nil {
+			return err
+		}
+		// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
+		if len(brData) >= (len(data) + 110) {
+			brData = nil
+		} else {
+			diff := len(data) - len(brData)
+			if diff <= len(data)/100 {
+				brData = nil
+			}
+		}
+
 		gzipData, err := CompressBytesGzip(data)
 		if err != nil {
 			return err
@@ -273,7 +296,7 @@ func (list SFileList) JSTmplInit() error {
 		hasher.Write(data)
 		checksum := hex.EncodeToString(hasher.Sum(nil))
 
-		list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
+		list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
 
 		DebugLogf("Added the '%s' static file.", path)
 		return nil
@@ -304,8 +327,22 @@ func (list SFileList) Init() error {
 		checksum := hex.EncodeToString(hasher.Sum(nil))
 
 		// Avoid double-compressing images
-		var gzipData []byte
+		var gzipData, brData []byte
 		if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" {
+			brData, err = CompressBytesBrotli(data)
+			if err != nil {
+				return err
+			}
+			// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
+			if len(brData) >= (len(data) + 130) {
+				brData = nil
+			} else {
+				diff := len(data) - len(brData)
+				if diff <= len(data)/100 {
+					brData = nil
+				}
+			}
+
 			gzipData, err = CompressBytesGzip(data)
 			if err != nil {
 				return err
@@ -321,7 +358,7 @@ func (list SFileList) Init() error {
 			}
 		}
 
-		list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})
+		list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})
 
 		DebugLogf("Added the '%s' static file.", path)
 		return nil
@@ -344,6 +381,21 @@ func (list SFileList) Add(path, prefix string) error {
 
 	ext := filepath.Ext(path)
 	path = strings.TrimPrefix(path, prefix)
+
+	brData, err := CompressBytesBrotli(data)
+	if err != nil {
+		return err
+	}
+	// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
+	if len(brData) >= (len(data) + 130) {
+		brData = nil
+	} else {
+		diff := len(data) - len(brData)
+		if diff <= len(data)/100 {
+			brData = nil
+		}
+	}
+
 	gzipData, err := CompressBytesGzip(data)
 	if err != nil {
 		return err
@@ -363,7 +415,7 @@ func (list SFileList) Add(path, prefix string) error {
 	hasher.Write(data)
 	checksum := hex.EncodeToString(hasher.Sum(nil))
 
-	list.Set("/s"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
+	list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
 
 	DebugLogf("Added the '%s' static file", path)
 	return nil
@@ -398,3 +450,17 @@ func CompressBytesGzip(in []byte) ([]byte, error) {
 	}
 	return buff.Bytes(), nil
 }
+
+func CompressBytesBrotli(in []byte) ([]byte, error) {
+	var buff bytes.Buffer
+	br := brotli.NewWriterLevel(&buff, brotli.BestCompression)
+	_, err := br.Write(in)
+	if err != nil {
+		return nil, err
+	}
+	err = br.Close()
+	if err != nil {
+		return nil, err
+	}
+	return buff.Bytes(), nil
+}

common/theme.go

@@ -250,6 +250,21 @@ func (t *Theme) AddThemeStaticFiles() error {
 		}
 
 		path = strings.TrimPrefix(path, "themes/"+t.Name+"/public")
+
+		brData, err := CompressBytesBrotli(data)
+		if err != nil {
+			return err
+		}
+		// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
+		if len(brData) >= (len(data) + 130) {
+			brData = nil
+		} else {
+			diff := len(data) - len(brData)
+			if diff <= len(data)/100 {
+				brData = nil
+			}
+		}
+
 		gzipData, err := CompressBytesGzip(data)
 		if err != nil {
 			return err
@@ -269,7 +284,7 @@ func (t *Theme) AddThemeStaticFiles() error {
 		hasher.Write(data)
 		checksum := hex.EncodeToString(hasher.Sum(nil))
 
-		StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
+		StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, brData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
 
 		DebugLog("Added the '/" + t.Name + path + "' static file for theme " + t.Name + ".")
 		return nil

gen_router.go

@@ -1244,6 +1244,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 				portless := strings.Split(ref,":")[0]
 				// TODO: Handle c.Site.Host in uppercase too?
 				if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
+					r.DumpRequest(req,"Ref Route")
 					co.ReferrerTracker.Bump(ref)
 				}
 			}
@@ -2865,7 +2866,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
 				r.SuspiciousRequest(req,"Bad Route")
 				return c.MicroNotFound(w,req)
 			}
+
 			r.DumpRequest(req,"Bad Route")
+			ae := req.Header.Get("Accept-Encoding")
+			likelyBot := ae == "gzip" || ae == ""
+			if likelyBot {
+				return c.MicroNotFound(w,req)
+			}
 			return c.NotFound(w,req,nil)
 	}
 	return err

go.mod

@@ -4,6 +4,7 @@ require (
 	cloud.google.com/go v0.31.0 // indirect
 	github.com/Azareal/gopsutil v0.0.0-20170716174751-0763ca4e911d
 	github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
+	github.com/andybalholm/brotli v1.0.0
 	github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f
 	github.com/fortytw2/leaktest v1.3.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.7

go.sum

@@ -6,6 +6,8 @@ github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8
 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

router_gen/main.go

@@ -915,6 +915,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 				portless := strings.Split(ref,":")[0]
 				// TODO: Handle c.Site.Host in uppercase too?
 				if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
+					r.DumpRequest(req,"Ref Route")
 					co.ReferrerTracker.Bump(ref)
 				}
 			}
@@ -1048,7 +1049,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
 				r.SuspiciousRequest(req,"Bad Route")
 				return c.MicroNotFound(w,req)
 			}
+
 			r.DumpRequest(req,"Bad Route")
+			ae := req.Header.Get("Accept-Encoding")
+			likelyBot := ae == "gzip" || ae == ""
+			if likelyBot {
+				return c.MicroNotFound(w,req)
+			}
 			return c.NotFound(w,req,nil)
 	}
 	return err

routes/misc.go

@@ -36,8 +36,16 @@ func StaticFile(w http.ResponseWriter, r *http.Request) {
 			} else {
 				h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000
 			}
-			
+			ae := r.Header.Get("Accept-Encoding")
-			if file.GzipLength > 300 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+
+			if file.BrLength > 300 && strings.Contains(ae, "br") {
+				h.Set("Content-Encoding", "br")
+				h.Set("Content-Length", file.StrBrLength)
+				http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.BrData))
+				return
+			}
+
+			if file.GzipLength > 300 && strings.Contains(ae, "gzip") {
 				h.Set("Content-Encoding", "gzip")
 				h.Set("Content-Length", file.StrGzipLength)
 				http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.GzipData))
@@ -65,7 +73,11 @@ func StaticFile(w http.ResponseWriter, r *http.Request) {
 	}
 	h.Set("Vary", "Accept-Encoding")
 
-	if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+	if file.BrLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "br") {
+		h.Set("Content-Encoding", "br")
+		h.Set("Content-Length", file.StrBrLength)
+		io.Copy(w, bytes.NewReader(file.BrData)) // Use w.Write instead?
+	} else if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
 		h.Set("Content-Encoding", "gzip")
 		h.Set("Content-Length", file.StrGzipLength)
 		io.Copy(w, bytes.NewReader(file.GzipData)) // Use w.Write instead?