a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

push content type for opensearch 

don't insert a last ip on installation
save bytes and allocs in the parser
add attach parser test cases
more benchmarks
This commit is contained in:
Azareal 2020-03-28 07:01:44 +10:00
parent 6b003499a7
commit cd66782129
5 changed files with 155 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
common/parser.go
View File

@ -8,6 +8,7 @@ import (
"io/ioutil"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
@ -24,7 +25,7 @@ var InvalidProfile = []byte("<red>[Invalid Profile]</red>")
var InvalidForum = []byte("<red>[Invalid Forum]</red>")
var unknownMedia = []byte("<red>[Unknown Media]</red>")
var URLOpen = []byte("<a href='")
var URLOpenUser = []byte("<a rel='ugc' href='")
var URLOpenUser = []byte("<a rel='ugc'href='")
var URLOpen2 = []byte("'>")
var bytesSinglequote = []byte("'")
var bytesGreaterthan = []byte(">")
@ -32,8 +33,8 @@ var urlMention = []byte(" class='mention'")
var URLClose = []byte("</a>")
var imageOpen = []byte("<a href=\"")
var imageOpen2 = []byte("\"><img src='")
var imageClose = []byte("' class='postImage'/></a>")
var attachOpen = []byte("<a download class='attach' href=\"")
var imageClose = []byte("'class='postImage'></a>")
var attachOpen = []byte("<a download class='attach'href=\"")
var attachClose = []byte("\">Attachment</a>")
var sidParam = []byte("?sid=")
var stypeParam = []byte("&amp;stype=")
@ -914,12 +915,11 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o
sport = ":" + port
}
media.URL = scheme + "//" + host + sport + path
extarr := strings.Split(path, ".")
if len(extarr) == 0 {
ext := strings.TrimPrefix(filepath.Ext(path), ".")
if len(ext) == 0 {
// TODO: Write a unit test for this
return media, false
}
ext := extarr[len(extarr)-1]
if ImageFileExts.Contains(ext) {
media.Type = "attach"
} else {

119
general_test.go
View File

@ -6,6 +6,7 @@ import (
"log"
"net/http"
"net/http/httptest"
"runtime"
"runtime/debug"
"strconv"
"strings"
@ -216,6 +217,10 @@ func BenchmarkTopicAdminRouteParallelAltAlt(b *testing.B) {
BenchmarkTopicAdminRouteParallel(b)
}
func BenchmarkTopicGuestAdminRouteParallelWithRouterPre(b *testing.B) {
runtime.GC()
}
func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
binit(b)
router, err := NewGenRouter(http.FileServer(http.Dir("./uploads")))
@ -236,6 +241,7 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: c.Year}
sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: c.Year}
path := "/topic/hm." + benchTid
//runtime.GC()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
@ -270,6 +276,67 @@ func BenchmarkTopicGuestAdminRouteParallelWithRouter(b *testing.B) {
cfg.Restore()
}
func BenchmarkTopicGuestAdminRouteParallelWithRouterPre2(b *testing.B) {
runtime.GC()
}
func BenchmarkTopicGuestAdminRouteParallelWithRouterGC(b *testing.B) {
binit(b)
router, err := NewGenRouter(http.FileServer(http.Dir("./uploads")))
if err != nil {
b.Fatal(err)
}
cfg := NewStashConfig()
c.Dev.DebugMode = false
c.Dev.SuperDebug = false
admin, err := c.Users.Get(1)
if err != nil {
b.Fatal(err)
}
if !admin.IsAdmin {
b.Fatal("UID1 is not an admin")
}
uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: c.Year}
sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: c.Year}
path := "/topic/hm." + benchTid
//runtime.GC()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
w := httptest.NewRecorder()
reqAdmin := httptest.NewRequest("get", path, bytes.NewReader(nil))
reqAdmin.AddCookie(&uidCookie)
reqAdmin.AddCookie(&sessionCookie)
reqAdmin.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
reqAdmin.Header.Set("Host", "localhost")
reqAdmin.Host = "localhost"
router.ServeHTTP(w, reqAdmin)
if w.Code != 200 {
b.Log(w.Body)
b.Fatal("HTTP Error!")
}
{
w := httptest.NewRecorder()
req := httptest.NewRequest("GET", path, bytes.NewReader(nil))
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
req.Header.Set("Host", "localhost")
req.Host = "localhost"
router.ServeHTTP(w, req)
if w.Code != 200 {
b.Log(w.Body)
b.Fatal("HTTP Error!")
}
}
}
runtime.GC()
})
cfg.Restore()
}
func BenchmarkTopicGuestRouteParallel(b *testing.B) {
binit(b)
cfg := NewStashConfig()
@ -324,6 +391,54 @@ func BenchmarkTopicGuestRouteParallelDebugMode(b *testing.B) {
cfg.Restore()
}
func BenchmarkAlertsRouteAdminParallelWithRouterGCPre(b *testing.B) {
runtime.GC()
}
func BenchmarkAlertsRouteAdminParallelWithRouterGC(b *testing.B) {
binit(b)
router, err := NewGenRouter(http.FileServer(http.Dir("./uploads")))
if err != nil {
b.Fatal(err)
}
cfg := NewStashConfig()
c.Dev.DebugMode = false
c.Dev.SuperDebug = false
admin, err := c.Users.Get(1)
if err != nil {
b.Fatal(err)
}
if !admin.IsAdmin {
b.Fatal("UID1 is not an admin")
}
uidCookie := http.Cookie{Name: "uid", Value: "1", Path: "/", MaxAge: c.Year}
sessionCookie := http.Cookie{Name: "session", Value: admin.Session, Path: "/", MaxAge: c.Year}
path := "/api/?m=alerts"
//runtime.GC()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
w := httptest.NewRecorder()
reqAdmin := httptest.NewRequest("get", path, bytes.NewReader(nil))
reqAdmin.AddCookie(&uidCookie)
reqAdmin.AddCookie(&sessionCookie)
reqAdmin.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
reqAdmin.Header.Set("Host", "localhost")
reqAdmin.Host = "localhost"
router.ServeHTTP(w, reqAdmin)
if w.Code != 200 {
b.Log(w.Body)
b.Fatal("HTTP Error!")
}
}
runtime.GC()
})
cfg.Restore()
}
func obRoute(b *testing.B, path string) {
binit(b)
cfg := NewStashConfig()
@ -370,6 +485,10 @@ func BenchmarkBadRouteGuestRouteParallelWithRouter(b *testing.B) {
obRouteNoError(b, "/garble/haa")
}
func BenchmarkAlertsRouteGuestParallelWithRouter(b *testing.B) {
obRoute(b, "/api/?m=alerts")
}
// TODO: Alternate between member and guest to bust some CPU caches?
func binit(b *testing.B) {

2
install/install.go
View File

@ -37,7 +37,7 @@ func createAdmin() error {
}
// Build the admin user query
adminUserStmt, err := qgen.Builder.SimpleInsert("users", "name, password, salt, email, group, is_super_admin, active, createdAt, lastActiveAt, lastLiked, oldestItemLikedCreatedAt, message, last_ip", "'Admin',?,?,'admin@localhost',1,1,1,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),'','127.0.0.1'")
adminUserStmt, err := qgen.Builder.SimpleInsert("users", "name, password, salt, email, group, is_super_admin, active, createdAt, lastActiveAt, lastLiked, oldestItemLikedCreatedAt, message, last_ip", "'Admin',?,?,'admin@localhost',1,1,1,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),'',''")
if err != nil {
return err
}

42
parser_test.go
View File

@ -145,7 +145,7 @@ func TestParser(t *testing.T) {
l := &METriList{nil}
url := "github.com/Azareal/Gosora"
eurl := "<a rel='ugc' href='//" + url + "'>" + url + "</a>"
eurl := "<a rel='ugc'href='//" + url + "'>" + url + "</a>"
l.Add("", "")
l.Add("haha", "haha")
l.Add("<b>t</b>", "<b>t</b>")
@ -194,18 +194,18 @@ func TestParser(t *testing.T) {
l.Add("ss", "ss")
l.Add("haha\nhaha\nhaha", "haha<br>haha<br>haha")
l.Add("//"+url, eurl)
l.Add("//a", "<a rel='ugc' href='//a'>a</a>")
l.Add(" //a", " <a rel='ugc' href='//a'>a</a>")
l.Add("//a ", "<a rel='ugc' href='//a'>a</a> ")
l.Add(" //a ", " <a rel='ugc' href='//a'>a</a> ")
l.Add("d //a ", "d <a rel='ugc' href='//a'>a</a> ")
l.Add("ddd ddd //a ", "ddd ddd <a rel='ugc' href='//a'>a</a> ")
l.Add("https://"+url, "<a rel='ugc' href='https://"+url+"'>"+url+"</a>")
l.Add("https://t", "<a rel='ugc' href='https://t'>t</a>")
l.Add("http://"+url, "<a rel='ugc' href='http://"+url+"'>"+url+"</a>")
l.Add("//a", "<a rel='ugc'href='//a'>a</a>")
l.Add(" //a", " <a rel='ugc'href='//a'>a</a>")
l.Add("//a ", "<a rel='ugc'href='//a'>a</a> ")
l.Add(" //a ", " <a rel='ugc'href='//a'>a</a> ")
l.Add("d //a ", "d <a rel='ugc'href='//a'>a</a> ")
l.Add("ddd ddd //a ", "ddd ddd <a rel='ugc'href='//a'>a</a> ")
l.Add("https://"+url, "<a rel='ugc'href='https://"+url+"'>"+url+"</a>")
l.Add("https://t", "<a rel='ugc'href='https://t'>t</a>")
l.Add("http://"+url, "<a rel='ugc'href='http://"+url+"'>"+url+"</a>")
l.Add("#http://"+url, "#http://"+url)
l.Add("@http://"+url, "<red>[Invalid Profile]</red>ttp://"+url)
l.Add("//"+url+"\n", "<a rel='ugc' href='//"+url+"'>"+url+"</a><br>")
l.Add("//"+url+"\n", "<a rel='ugc'href='//"+url+"'>"+url+"</a><br>")
l.Add("\n//"+url, "<br>"+eurl)
l.Add("\n//"+url+"\n", "<br>"+eurl+"<br>")
l.Add("\n//"+url+"\n\n", "<br>"+eurl+"<br><br>")
@ -222,10 +222,24 @@ func TestParser(t *testing.T) {
fs = "https://" + c.Site.URL
}
l.Add("//"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>")
// TODO: Strip redundant slashes?
l.Add("//"+u+"/", "<a href='"+fs+"/'>"+c.Site.URL+"/</a>")
l.Add("//"+u+"//", "<a href='"+fs+"//'>"+c.Site.URL+"//</a>")
l.Add("//"+u+"\n", "<a href='"+fs+"'>"+c.Site.URL+"</a><br>")
l.Add("//"+u+"\n//"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a><br><a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("http://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("https://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("//"+u+"/attachs/sha256hash.png?sid=1&stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs/s?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs/?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs/sha256hash.?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs/sha256hash.png", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash.png?sid=1", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash.png?stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
}
local("localhost")
local("127.0.0.1")
@ -259,9 +273,9 @@ func TestParser(t *testing.T) {
l.Add("@ #tid-@", "<red>[Invalid Profile]</red>#tid-@")
l.Add("#tid-1 #tid-1", "<a href='/topic/1'>#tid-1</a> <a href='/topic/1'>#tid-1</a>")
l.Add("#tid-0", "<red>[Invalid Topic]</red>")
l.Add("https://"+url+"/#tid-1", "<a rel='ugc' href='https://"+url+"/#tid-1'>"+url+"/#tid-1</a>")
l.Add("https://"+url+"/?hi=2", "<a rel='ugc' href='https://"+url+"/?hi=2'>"+url+"/?hi=2</a>")
l.Add("https://"+url+"/?hi=2#t=1", "<a rel='ugc' href='https://"+url+"/?hi=2#t=1'>"+url+"/?hi=2#t=1</a>")
l.Add("https://"+url+"/#tid-1", "<a rel='ugc'href='https://"+url+"/#tid-1'>"+url+"/#tid-1</a>")
l.Add("https://"+url+"/?hi=2", "<a rel='ugc'href='https://"+url+"/?hi=2'>"+url+"/?hi=2</a>")
l.Add("https://"+url+"/?hi=2#t=1", "<a rel='ugc'href='https://"+url+"/?hi=2#t=1'>"+url+"/?hi=2#t=1</a>")
l.Add("#fid-1", "<a href='/forum/1'>#fid-1</a>")
l.Add(" #fid-1", " <a href='/forum/1'>#fid-1</a>")
l.Add("#fid-0", "<red>[Invalid Forum]</red>")

1
routes/api.go
View File

@ -254,6 +254,7 @@ func APIMe(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError {
}
func OpenSearchXml(w http.ResponseWriter, r *http.Request) c.RouteError {
w.Header().Set("Content-Type", "application/xml")
furl := "http"
if c.Config.SslSchema {
furl += "s"