From d5a58421a71bf60b1c3c2703ac7ff5fbb9e6f61a Mon Sep 17 00:00:00 2001 From: Azareal Date: Sun, 10 Feb 2019 19:32:05 +1000 Subject: [PATCH] The forum page now uses EasyJSON to speed it up slightly. Added SemrushBot to the analytics rather than treating it as an unknown. The word compatible is now skipped in UA to hopefully avoid allocating a little bit more. The tickloop is a little more resilient against hard stale connections now. Removed a temporary benchmark which was used for comparing the built-in JSON library to EasyJSON. Added the semrush phrase. --- gen_router.go | 13 ++++++++----- general_test.go | 4 ---- langs/english.json | 1 + router_gen/main.go | 11 +++++++---- routes/forum.go | 3 +-- tickloop.go | 3 +++ 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/gen_router.go b/gen_router.go index b97c29c4..8de9be60 100644 --- a/gen_router.go +++ b/gen_router.go @@ -486,7 +486,8 @@ var agentMapEnum = map[string]int{ "blank": 25, "malformed": 26, "suspicious": 27, - "zgrab": 28, + "semrush": 28, + "zgrab": 29, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -517,7 +518,8 @@ var reverseAgentMapEnum = map[int]string{ 25: "blank", 26: "malformed", 27: "suspicious", - 28: "zgrab", + 28: "semrush", + 29: "zgrab", } var markToAgent = map[string]string{ "OPR": "opera", @@ -543,6 +545,7 @@ var markToAgent = map[string]string{ "Discordbot": "discord", "Twitterbot": "twitter", "Discourse": "discourse", + "SemrushBot": "semrush", "zgrab": "zgrab", } /*var agentRank = map[string]int{ @@ -762,7 +765,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for _, item := range StringToBytes(ua) { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || item == '~' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { if len(buffer) != 0 { if len(buffer) > 2 { // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append @@ -777,8 +780,8 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { os = "iphone" case "Android": os = "android" - case "like": - // Skip this word + case "like","compatible": + // Skip these words default: items = append(items, string(buffer)) } diff --git a/general_test.go b/general_test.go index 6dc7fdb5..7a4aea7d 100644 --- a/general_test.go +++ b/general_test.go @@ -350,10 +350,6 @@ func BenchmarkTopicsGuestJSRouteParallelWithRouter(b *testing.B) { obRoute(b, "/topics/?js=1") } -func BenchmarkTopicsGuestEJSRouteParallelWithRouter(b *testing.B) { - obRoute(b, "/topics/?ejs=1") -} - func BenchmarkForumsGuestRouteParallelWithRouter(b *testing.B) { obRoute(b, "/forums/") } diff --git a/langs/english.json b/langs/english.json index 06677c73..b24a5468 100644 --- a/langs/english.json +++ b/langs/english.json @@ -190,6 +190,7 @@ "discourse":"Discourse Forum Onebox", "lynx":"Lynx", + "semrush":"SemrushBot", "zgrab":"Zgrab Application Scanner", "suspicious":"Suspicious", "unknown":"Unknown", diff --git a/router_gen/main.go b/router_gen/main.go index 6880f1bf..7a997960 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -221,6 +221,7 @@ func main() { "blank", "malformed", "suspicious", + "semrush", "zgrab", } @@ -255,6 +256,7 @@ func main() { "Twitterbot", "Discourse", + "SemrushBot", "zgrab", } @@ -284,7 +286,8 @@ func main() { "Twitterbot": "twitter", "Discourse": "discourse", - "zgrab": "zgrab", + "SemrushBot": "semrush", + "zgrab": "zgrab", } var fileData = `// Code generated by Gosora's Router Generator. DO NOT EDIT. @@ -554,7 +557,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for _, item := range StringToBytes(ua) { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || item == '~' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { if len(buffer) != 0 { if len(buffer) > 2 { // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append @@ -569,8 +572,8 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { os = "iphone" case "Android": os = "android" - case "like": - // Skip this word + case "like","compatible": + // Skip these words default: items = append(items, string(buffer)) } diff --git a/routes/forum.go b/routes/forum.go index 54da1bd0..218244d3 100644 --- a/routes/forum.go +++ b/routes/forum.go @@ -2,7 +2,6 @@ package routes import ( "database/sql" - "encoding/json" "net/http" "strconv" @@ -114,7 +113,7 @@ func ViewForum(w http.ResponseWriter, r *http.Request, user common.User, header // TODO: Reduce the amount of boilerplate here if r.FormValue("js") == "1" { - outBytes, err := json.Marshal(wsTopicList(topicList, lastPage)) + outBytes, err := wsTopicList(topicList, lastPage).MarshalJSON() if err != nil { return common.InternalError(err, w, r) } diff --git a/tickloop.go b/tickloop.go index 32ba8029..3e8a9b03 100644 --- a/tickloop.go +++ b/tickloop.go @@ -25,6 +25,7 @@ func startTick() (abort bool) { if err != nil { // TODO: There's a bit of a race here, but it doesn't matter if this error appears multiple times in the logs as it's capped at three times, we just want to cut it down 99% of the time if isDBDown == 0 { + db.SetConnMaxLifetime(time.Second) // Drop all the connections and start over common.LogWarning(err) common.LogWarning(errors.New("The database is down")) } @@ -34,6 +35,8 @@ func startTick() (abort bool) { if isDBDown == 1 { log.Print("The database is back") } + //db.SetConnMaxLifetime(time.Second * 60 * 5) // Make this infinite as the temporary lifetime change will purge the stale connections? + db.SetConnMaxLifetime(-1) atomic.StoreInt32(&common.IsDBDown, 0) return false }