a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

try speeding up static files with brotli 

track ref routes for a bit to debug them
hit more bots with MicroNotFound
This commit is contained in:
Azareal 2020-04-13 13:28:52 +10:00
parent 1dc69ed89e
commit faf215f388
7 changed files with 127 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
common/files.go
View File

@ -17,6 +17,7 @@ import (
"sync"
tmpl "github.com/Azareal/Gosora/tmpl_client"
"github.com/andybalholm/brotli"
)
type SFileList map[string]SFile
@ -25,15 +26,22 @@ var StaticFiles SFileList = make(map[string]SFile)
var staticFileMutex sync.RWMutex
type SFile struct {
Data []byte
GzipData []byte
Sha256 string
OName string
Pos int64
Length int64
StrLength string
GzipLength int64
StrGzipLength string
// TODO: Move these to the end?
Data []byte
GzipData []byte
BrData []byte
Sha256 string
OName string
Pos int64
Length int64
StrLength string
GzipLength int64
StrGzipLength string
BrLength int64
StrBrLength string
Mimetype string
Info os.FileInfo
FormattedModTime string
@ -254,6 +262,21 @@ func (list SFileList) JSTmplInit() error {
path = tmplName + ".js"
DebugLog("js path: ", path)
ext := filepath.Ext("/tmpl_client/" + path)
brData, err := CompressBytesBrotli(data)
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len(brData) >= (len(data) + 110) {
brData = nil
} else {
diff := len(data) - len(brData)
if diff <= len(data)/100 {
brData = nil
}
}
gzipData, err := CompressBytesGzip(data)
if err != nil {
return err
@ -273,7 +296,7 @@ func (list SFileList) JSTmplInit() error {
hasher.Write(data)
checksum := hex.EncodeToString(hasher.Sum(nil))
list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
DebugLogf("Added the '%s' static file.", path)
return nil
@ -304,8 +327,22 @@ func (list SFileList) Init() error {
checksum := hex.EncodeToString(hasher.Sum(nil))
// Avoid double-compressing images
var gzipData []byte
var gzipData, brData []byte
if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" {
brData, err = CompressBytesBrotli(data)
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len(brData) >= (len(data) + 130) {
brData = nil
} else {
diff := len(data) - len(brData)
if diff <= len(data)/100 {
brData = nil
}
}
gzipData, err = CompressBytesGzip(data)
if err != nil {
return err
@ -321,7 +358,7 @@ func (list SFileList) Init() error {
}
}
list.Set("/s/"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})
list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})
DebugLogf("Added the '%s' static file.", path)
return nil
@ -344,6 +381,21 @@ func (list SFileList) Add(path, prefix string) error {
ext := filepath.Ext(path)
path = strings.TrimPrefix(path, prefix)
brData, err := CompressBytesBrotli(data)
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len(brData) >= (len(data) + 130) {
brData = nil
} else {
diff := len(data) - len(brData)
if diff <= len(data)/100 {
brData = nil
}
}
gzipData, err := CompressBytesGzip(data)
if err != nil {
return err
@ -363,7 +415,7 @@ func (list SFileList) Add(path, prefix string) error {
hasher.Write(data)
checksum := hex.EncodeToString(hasher.Sum(nil))
list.Set("/s"+path, SFile{data, gzipData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
list.Set("/s/"+path, SFile{data, gzipData, brData, checksum, path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
DebugLogf("Added the '%s' static file", path)
return nil
@ -398,3 +450,17 @@ func CompressBytesGzip(in []byte) ([]byte, error) {
}
return buff.Bytes(), nil
}
func CompressBytesBrotli(in []byte) ([]byte, error) {
var buff bytes.Buffer
br := brotli.NewWriterLevel(&buff, brotli.BestCompression)
_, err := br.Write(in)
if err != nil {
return nil, err
}
err = br.Close()
if err != nil {
return nil, err
}
return buff.Bytes(), nil
}

17
common/theme.go
View File

@ -250,6 +250,21 @@ func (t *Theme) AddThemeStaticFiles() error {
}
path = strings.TrimPrefix(path, "themes/"+t.Name+"/public")
brData, err := CompressBytesBrotli(data)
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len(brData) >= (len(data) + 130) {
brData = nil
} else {
diff := len(data) - len(brData)
if diff <= len(data)/100 {
brData = nil
}
}
gzipData, err := CompressBytesGzip(data)
if err != nil {
return err
@ -269,7 +284,7 @@ func (t *Theme) AddThemeStaticFiles() error {
hasher.Write(data)
checksum := hex.EncodeToString(hasher.Sum(nil))
StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
StaticFiles.Set("/s/"+t.Name+path, SFile{data, gzipData, brData, checksum, t.Name + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
DebugLog("Added the '/" + t.Name + path + "' static file for theme " + t.Name + ".")
return nil

7
gen_router.go
View File

@ -1244,6 +1244,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
portless := strings.Split(ref,":")[0]
// TODO: Handle c.Site.Host in uppercase too?
if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
r.DumpRequest(req,"Ref Route")
co.ReferrerTracker.Bump(ref)
}
}
@ -2865,7 +2866,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req)
}
r.DumpRequest(req,"Bad Route")
ae := req.Header.Get("Accept-Encoding")
likelyBot := ae == "gzip" || ae == ""
if likelyBot {
return c.MicroNotFound(w,req)
}
return c.NotFound(w,req,nil)
}
return err

1
go.mod
View File

@ -4,6 +4,7 @@ require (
cloud.google.com/go v0.31.0 // indirect
github.com/Azareal/gopsutil v0.0.0-20170716174751-0763ca4e911d
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
github.com/andybalholm/brotli v1.0.0
github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f
github.com/fortytw2/leaktest v1.3.0 // indirect
github.com/fsnotify/fsnotify v1.4.7

2
go.sum
View File

@ -6,6 +6,8 @@ github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

7
router_gen/main.go
View File

@ -915,6 +915,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
portless := strings.Split(ref,":")[0]
// TODO: Handle c.Site.Host in uppercase too?
if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
r.DumpRequest(req,"Ref Route")
co.ReferrerTracker.Bump(ref)
}
}
@ -1048,7 +1049,13 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *
r.SuspiciousRequest(req,"Bad Route")
return c.MicroNotFound(w,req)
}
r.DumpRequest(req,"Bad Route")
ae := req.Header.Get("Accept-Encoding")
likelyBot := ae == "gzip" || ae == ""
if likelyBot {
return c.MicroNotFound(w,req)
}
return c.NotFound(w,req,nil)
}
return err

18
routes/misc.go
View File

@ -36,8 +36,16 @@ func StaticFile(w http.ResponseWriter, r *http.Request) {
} else {
h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000
}
if file.GzipLength > 300 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
ae := r.Header.Get("Accept-Encoding")
if file.BrLength > 300 && strings.Contains(ae, "br") {
h.Set("Content-Encoding", "br")
h.Set("Content-Length", file.StrBrLength)
http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.BrData))
return
}
if file.GzipLength > 300 && strings.Contains(ae, "gzip") {
h.Set("Content-Encoding", "gzip")
h.Set("Content-Length", file.StrGzipLength)
http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.GzipData))
@ -65,7 +73,11 @@ func StaticFile(w http.ResponseWriter, r *http.Request) {
}
h.Set("Vary", "Accept-Encoding")
if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
if file.BrLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "br") {
h.Set("Content-Encoding", "br")
h.Set("Content-Length", file.StrBrLength)
io.Copy(w, bytes.NewReader(file.BrData)) // Use w.Write instead?
} else if file.GzipLength > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
h.Set("Content-Encoding", "gzip")
h.Set("Content-Length", file.StrGzipLength)
io.Copy(w, bytes.NewReader(file.GzipData)) // Use w.Write instead?