From 8f7cda93784816dfbdbedc052c62d1f2dfc4c0a4 Mon Sep 17 00:00:00 2001 From: Azareal Date: Wed, 22 Aug 2018 11:32:07 +1000 Subject: [PATCH] Don't double-compress avatars. Gzip is now disabled for static files when the gains don't justify the costs. Moved StopServerChan and StoppedServer to package common. Added some disabled profiling code. --- common/common.go | 8 ++++++++ common/files.go | 23 +++++++++++++++++++---- common/websockets.go | 1 + gen_router.go | 24 +++++++++++++----------- main.go | 31 ++++++++++++++++++------------- router_gen/main.go | 24 +++++++++++++----------- routes/misc.go | 2 +- 7 files changed, 73 insertions(+), 40 deletions(-) diff --git a/common/common.go b/common/common.go index cfc3b918..5a7f4273 100644 --- a/common/common.go +++ b/common/common.go @@ -96,6 +96,14 @@ func (inits dbInits) Add(init ...func(acc *qgen.Accumulator) error) { DbInits = dbInits(append(DbInits, init...)) } +// TODO: Add a graceful shutdown function +func StoppedServer(msg ...interface{}) { + //log.Print("stopped server") + StopServerChan <- msg +} + +var StopServerChan = make(chan []interface{}) + func DebugDetail(args ...interface{}) { if Dev.SuperDebug { log.Print(args...) diff --git a/common/files.go b/common/files.go index 2d24ea89..d351f056 100644 --- a/common/files.go +++ b/common/files.go @@ -242,12 +242,27 @@ func (list SFileList) Init() error { path = strings.TrimPrefix(path, "public/") var ext = filepath.Ext("/public/" + path) - gzipData, err := compressBytesGzip(data) - if err != nil { - return err + mimetype := mime.TypeByExtension(ext) + + // Avoid double-compressing images + var gzipData []byte + if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" { + gzipData, err = compressBytesGzip(data) + if err != nil { + return err + } + // Don't use Gzip if we get meagre gains from it as it takes longer to process the responses + if len(gzipData) >= (len(data) + 100) { + gzipData = nil + } else { + diff := len(data) - len(gzipData) + if diff <= len(data)/100 { + gzipData = nil + } + } } - list.Set("/static/"+path, SFile{data, gzipData, 0, int64(len(data)), int64(len(gzipData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)}) + list.Set("/static/"+path, SFile{data, gzipData, 0, int64(len(data)), int64(len(gzipData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)}) DebugLogf("Added the '%s' static file.", path) return nil diff --git a/common/websockets.go b/common/websockets.go index 2cd89467..b43c745a 100644 --- a/common/websockets.go +++ b/common/websockets.go @@ -437,6 +437,7 @@ func RouteWebsockets(w http.ResponseWriter, r *http.Request, user User) RouteErr //log.Print("string(Message)", string(message)) messages := bytes.Split(message, []byte("\r")) for _, msg := range messages { + //StoppedServer("Profile end") // A bit of code for me to profile the software //log.Print("Submessage", msg) //log.Print("Submessage", string(msg)) if bytes.HasPrefix(msg, []byte("page ")) { diff --git a/gen_router.go b/gen_router.go index 86e0e7f3..2767bc89 100644 --- a/gen_router.go +++ b/gen_router.go @@ -888,7 +888,19 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // Disable Gzip when SSL is disabled for security reasons? - if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { + // We don't want to double-compress things which are already compressed, so we're moving /uploads here + // TODO: Find a better way of doing this + if prefix == "/uploads" { + if extraData == "" { + common.NotFound(w,req,nil) + return + } + counters.RouteViewCounter.Bump(121) + req.URL.Path += extraData + // TODO: Find a way to propagate errors up from this? + router.UploadHandler(w,req) // TODO: Count these views + return + } else if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Content-Type", "text/html") gz := gzip.NewWriter(w) @@ -2051,16 +2063,6 @@ func (router *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, u if err != nil { router.handleError(err,w,req,user) }*/ - case "/uploads": - if extraData == "" { - common.NotFound(w,req,nil) - return - } - w.Header().Del("Content-Type") - counters.RouteViewCounter.Bump(121) - req.URL.Path += extraData - // TODO: Find a way to propagate errors up from this? - router.UploadHandler(w,req) // TODO: Count these views case "": // Stop the favicons, robots.txt file, etc. resolving to the topics list // TODO: Add support for favicons and robots.txt files diff --git a/main.go b/main.go index 1b3bc589..b37e4d46 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "os/signal" + "runtime/pprof" "strings" "syscall" "time" @@ -227,6 +228,15 @@ func main() { fmt.Println("") common.StartTime = time.Now() + // TODO: Add a flag for enabling the profiler + if false { + f, err := os.Create("./logs/cpuprof.prof") + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + } + jsToken, err := common.GenerateSafeString(80) if err != nil { log.Fatal(err) @@ -377,7 +387,7 @@ func main() { sig := <-sigs // TODO: Gracefully shutdown the HTTP server runTasks(common.ShutdownTasks) - stoppedServer("Received a signal to shutdown: ", sig) + common.StoppedServer("Received a signal to shutdown: ", sig) }() // Start up the WebSocket ticks @@ -387,19 +397,14 @@ func main() { // pprof.StopCPUProfile() //} startServer() - args := <-stopServerChan + args := <-common.StopServerChan + if false { + pprof.StopCPUProfile() + } // Why did the server stop? log.Fatal(args...) } -// TODO: Add a graceful shutdown function -func stoppedServer(msg ...interface{}) { - //log.Print("stopped server") - stopServerChan <- msg -} - -var stopServerChan = make(chan []interface{}) - func startServer() { // We might not need the timeouts, if we're behind a reverse-proxy like Nginx var newServer = func(addr string, handler http.Handler) *http.Server { @@ -429,7 +434,7 @@ func startServer() { } log.Print("Listening on port " + common.Site.Port) go func() { - stoppedServer(newServer(":"+common.Site.Port, router).ListenAndServe()) + common.StoppedServer(newServer(":"+common.Site.Port, router).ListenAndServe()) }() return } @@ -442,11 +447,11 @@ func startServer() { // TODO: Redirect to port 443 go func() { log.Print("Listening on port 80") - stoppedServer(newServer(":80", &routes.HTTPSRedirect{}).ListenAndServe()) + common.StoppedServer(newServer(":80", &routes.HTTPSRedirect{}).ListenAndServe()) }() } log.Printf("Listening on port %s", common.Site.Port) go func() { - stoppedServer(newServer(":"+common.Site.Port, router).ListenAndServeTLS(common.Config.SslFullchain, common.Config.SslPrivkey)) + common.StoppedServer(newServer(":"+common.Site.Port, router).ListenAndServeTLS(common.Config.SslFullchain, common.Config.SslPrivkey)) }() } diff --git a/router_gen/main.go b/router_gen/main.go index 61eb4a19..c7aa09c6 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -665,7 +665,19 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // Disable Gzip when SSL is disabled for security reasons? - if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { + // We don't want to double-compress things which are already compressed, so we're moving /uploads here + // TODO: Find a better way of doing this + if prefix == "/uploads" { + if extraData == "" { + common.NotFound(w,req,nil) + return + } + counters.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile" }}) + req.URL.Path += extraData + // TODO: Find a way to propagate errors up from this? + router.UploadHandler(w,req) // TODO: Count these views + return + } else if prefix != "/ws" && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Content-Type", "text/html") gz := gzip.NewWriter(w) @@ -684,16 +696,6 @@ func (router *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, u if err != nil { router.handleError(err,w,req,user) }*/ - case "/uploads": - if extraData == "" { - common.NotFound(w,req,nil) - return - } - w.Header().Del("Content-Type") - counters.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile" }}) - req.URL.Path += extraData - // TODO: Find a way to propagate errors up from this? - router.UploadHandler(w,req) // TODO: Count these views case "": // Stop the favicons, robots.txt file, etc. resolving to the topics list // TODO: Add support for favicons and robots.txt files diff --git a/routes/misc.go b/routes/misc.go index 3d602dec..5c32a483 100644 --- a/routes/misc.go +++ b/routes/misc.go @@ -37,7 +37,7 @@ func StaticFile(w http.ResponseWriter, r *http.Request) { h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000 h.Set("Vary", "Accept-Encoding") - if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && file.GzipLength > 0 { h.Set("Content-Encoding", "gzip") h.Set("Content-Length", strconv.FormatInt(file.GzipLength, 10)) io.Copy(w, bytes.NewReader(file.GzipData)) // Use w.Write instead?