a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add a few more parser tests 

save bytes in the parser
save allocs in external image embeds in the parser
save alloc bytes in convos
This commit is contained in:
Azareal 2020-03-28 14:48:42 +10:00
parent d3b2721746
commit 48b0b410c9
3 changed files with 37 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
common/conversations.go
View File

@ -168,13 +168,13 @@ type DefaultConversationStore struct {
func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationStore, error) { func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationStore, error) {
co := "conversations" co := "conversations"
return &DefaultConversationStore{ return &DefaultConversationStore{
get: acc.Select(co).Columns("createdBy, createdAt, lastReplyBy, lastReplyAt").Where("cid=?").Prepare(), get: acc.Select(co).Columns("createdBy,createdAt,lastReplyBy,lastReplyAt").Where("cid=?").Prepare(),
getUser: acc.SimpleInnerJoin("conversations_participants AS cp", "conversations AS c", "cp.cid, c.createdBy, c.createdAt, c.lastReplyBy, c.lastReplyAt", "cp.cid=c.cid", "cp.uid=?", "c.lastReplyAt DESC, c.createdAt DESC, c.cid DESC", "?,?"), getUser: acc.SimpleInnerJoin("conversations_participants AS cp", "conversations AS c", "cp.cid, c.createdBy, c.createdAt, c.lastReplyBy, c.lastReplyAt", "cp.cid=c.cid", "cp.uid=?", "c.lastReplyAt DESC, c.createdAt DESC, c.cid DESC", "?,?"),
getUserCount: acc.Count("conversations_participants").Where("uid=?").Prepare(), getUserCount: acc.Count("conversations_participants").Where("uid=?").Prepare(),
delete: acc.Delete(co).Where("cid=?").Prepare(), delete: acc.Delete(co).Where("cid=?").Prepare(),
deletePosts: acc.Delete("conversations_posts").Where("cid=?").Prepare(), deletePosts: acc.Delete("conversations_posts").Where("cid=?").Prepare(),
deleteParticipants: acc.Delete("conversations_participants").Where("cid=?").Prepare(), deleteParticipants: acc.Delete("conversations_participants").Where("cid=?").Prepare(),
create: acc.Insert(co).Columns("createdBy, createdAt, lastReplyBy, lastReplyAt").Fields("?,UTC_TIMESTAMP(),?,UTC_TIMESTAMP()").Prepare(), create: acc.Insert(co).Columns("createdBy,createdAt,lastReplyBy,lastReplyAt").Fields("?,UTC_TIMESTAMP(),?,UTC_TIMESTAMP()").Prepare(),
addParticipant: acc.Insert("conversations_participants").Columns("uid,cid").Fields("?,?").Prepare(), addParticipant: acc.Insert("conversations_participants").Columns("uid,cid").Fields("?,?").Prepare(),
count: acc.Count(co).Prepare(), count: acc.Count(co).Prepare(),
}, acc.FirstError() }, acc.FirstError()
@ -251,10 +251,13 @@ func (s *DefaultConversationStore) GetUserExtra(uid, offset int) (cos []*Convers
var q string var q string
idList := make([]interface{}, len(raw)) idList := make([]interface{}, len(raw))
for i, co := range raw { for i, co := range raw {
if i == 0 {
q = "?"
} else {
q += ",?"
}
idList[i] = strconv.Itoa(co.ID) idList[i] = strconv.Itoa(co.ID)
q += "?,"
} }
q = q[0 : len(q)-1]
rows, err := qgen.NewAcc().Select("conversations_participants").Columns("uid,cid").Where("cid IN(" + q + ")").Query(idList...) rows, err := qgen.NewAcc().Select("conversations_participants").Columns("uid,cid").Where("cid IN(" + q + ")").Query(idList...)
if err != nil { if err != nil {

9
common/parser.go
View File

@ -29,7 +29,7 @@ var URLOpenUser = []byte("<a rel='ugc'href='")
var URLOpen2 = []byte("'>") var URLOpen2 = []byte("'>")
var bytesSinglequote = []byte("'") var bytesSinglequote = []byte("'")
var bytesGreaterthan = []byte(">") var bytesGreaterthan = []byte(">")
var urlMention = []byte(" class='mention'") var urlMention = []byte("'class='mention'")
var URLClose = []byte("</a>") var URLClose = []byte("</a>")
var imageOpen = []byte("<a href=\"") var imageOpen = []byte("<a href=\"")
var imageOpen2 = []byte("\"><img src='") var imageOpen2 = []byte("\"><img src='")
@ -557,7 +557,6 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse
sb.Write(URLOpen) sb.Write(URLOpen)
sb.WriteString(menUser.Link) sb.WriteString(menUser.Link)
sb.Write(bytesSinglequote)
sb.Write(urlMention) sb.Write(urlMention)
sb.Write(bytesGreaterthan) sb.Write(bytesGreaterthan)
sb.WriteByte('@') sb.WriteByte('@')
@ -937,15 +936,15 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o
if ok && len(video) >= 1 && video[0] != "" { if ok && len(video) >= 1 && video[0] != "" {
media.Type = "raw" media.Type = "raw"
// TODO: Filter the URL to make sure no nasties end up in there // TODO: Filter the URL to make sure no nasties end up in there
media.Body = "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/" + video[0] + "' frameborder=0 allowfullscreen></iframe>" media.Body = "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/" + video[0] + "'frameborder=0 allowfullscreen></iframe>"
return media, true return media, true
} }
} }
if lastFrag := pathFrags[len(pathFrags)-1]; lastFrag != "" { if lastFrag := pathFrags[len(pathFrags)-1]; lastFrag != "" {
// TODO: Write a function for getting the file extension of a string // TODO: Write a function for getting the file extension of a string
if extarr := strings.Split(lastFrag, "."); len(extarr) >= 2 { ext := strings.TrimPrefix(filepath.Ext(lastFrag),".")
ext := extarr[len(extarr)-1] if len(ext) != 0 {
if ImageFileExts.Contains(ext) { if ImageFileExts.Contains(ext) {
media.Type = "image" media.Type = "image"
var sport string var sport string

45
parser_test.go
View File

@ -231,6 +231,7 @@ func TestParser(t *testing.T) {
l.Add("//"+u+"\n//"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a><br><a href='"+fs+"'>"+c.Site.URL+"</a>") l.Add("//"+u+"\n//"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a><br><a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("http://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>") l.Add("http://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("https://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>") l.Add("https://"+u, "<a href='"+fs+"'>"+c.Site.URL+"</a>")
l.Add("//"+u+"/attachs/sha256hash.png?sid=1&stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>") l.Add("//"+u+"/attachs/sha256hash.png?sid=1&stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash?sid=1&stype=forums", "<red>[Invalid URL]</red>") l.Add("//"+u+"/attachs/sha256hash?sid=1&stype=forums", "<red>[Invalid URL]</red>")
l.Add("//"+u+"/attachs/s?sid=1&stype=forums", "<red>[Invalid URL]</red>") l.Add("//"+u+"/attachs/s?sid=1&stype=forums", "<red>[Invalid URL]</red>")
@ -240,30 +241,36 @@ func TestParser(t *testing.T) {
l.Add("//"+u+"/attachs/sha256hash.png", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>") l.Add("//"+u+"/attachs/sha256hash.png", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash.png?sid=1", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>") l.Add("//"+u+"/attachs/sha256hash.png?sid=1", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash.png?stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>") l.Add("//"+u+"/attachs/sha256hash.png?stype=forums", "<a href=\""+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums\"><img src='"+fs+"/attachs/sha256hash.png?sid=1&amp;stype=forums'class='postImage'></a>")
l.Add("//"+u+"/attachs/sha256hash.txt?sid=1&stype=forums", "<a download class='attach'href=\""+fs+"/attachs/sha256hash.txt?sid=1&amp;stype=forums\">Attachment</a>")
l.Add("//example.com/image.png", "<a href=\"//example.com/image.png\"><img src='//example.com/image.png'class='postImage'></a>")
l.Add("https://example.com/image.png", "<a href=\"https://example.com/image.png\"><img src='https://example.com/image.png'class='postImage'></a>")
l.Add("http://example.com/image.png", "<a href=\"http://example.com/image.png\"><img src='http://example.com/image.png'class='postImage'></a>")
} }
local("localhost") local("localhost")
local("127.0.0.1") local("127.0.0.1")
local("[::1]") local("[::1]")
l.Add("https://www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
//l.Add("https://www.youtube.com/watch?v=;","<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/;' frameborder=0 allowfullscreen></iframe>") //l.Add("https://www.youtube.com/watch?v=;","<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/;'frameborder=0 allowfullscreen></iframe>")
l.Add("https://www.youtube.com/watch?v=d;", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/d' frameborder=0 allowfullscreen></iframe>") l.Add("https://www.youtube.com/watch?v=d;", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/d'frameborder=0 allowfullscreen></iframe>")
l.Add("https://www.youtube.com/watch?v=d;d", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/d' frameborder=0 allowfullscreen></iframe>") l.Add("https://www.youtube.com/watch?v=d;d", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/d'frameborder=0 allowfullscreen></iframe>")
l.Add("https://www.youtube.com/watch?v=alert()", "<red>[Invalid URL]</red>()") l.Add("https://www.youtube.com/watch?v=alert()", "<red>[Invalid URL]</red>()")
l.Add("https://www.youtube.com/watch?v=alert()()", "<red>[Invalid URL]</red>()()") l.Add("https://www.youtube.com/watch?v=alert()()", "<red>[Invalid URL]</red>()()")
l.Add("https://www.youtube.com/watch?v=js:alert()", "<red>[Invalid URL]</red>()") l.Add("https://www.youtube.com/watch?v=js:alert()", "<red>[Invalid URL]</red>()")
l.Add("https://www.youtube.com/watch?v='+><script>alert(\"\")</script><+'", "<red>[Invalid URL]</red>'+><script>alert(\"\")</script><+'") l.Add("https://www.youtube.com/watch?v='+><script>alert(\"\")</script><+'", "<red>[Invalid URL]</red>'+><script>alert(\"\")</script><+'")
l.Add("https://www.youtube.com/watch?v='+onready='alert(\"\")'+'", "<red>[Invalid URL]</red>'+onready='alert(\"\")'+'") l.Add("https://www.youtube.com/watch?v='+onready='alert(\"\")'+'", "<red>[Invalid URL]</red>'+onready='alert(\"\")'+'")
l.Add(" https://www.youtube.com/watch?v=lalalalala", " <iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add(" https://www.youtube.com/watch?v=lalalalala", " <iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("https://www.youtube.com/watch?v=lalalalala tt", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe> tt") l.Add("https://www.youtube.com/watch?v=lalalalala tt", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe> tt")
l.Add("https://www.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://www.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("https://gaming.youtube.com/watch?v=lalalalala", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://gaming.youtube.com/watch?v=lalalalala", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("https://gaming.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://gaming.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("https://m.youtube.com/watch?v=lalalalala", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://m.youtube.com/watch?v=lalalalala", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("https://m.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("https://m.youtube.com/watch?v=lalalalala&d=haha", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("http://www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("http://www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("//www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") l.Add("//www.youtube.com/watch?v=lalalalala", "<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
//l.Add("www.youtube.com/watch?v=lalalalala","<iframe class='postIframe' src='https://www.youtube-nocookie.com/embed/lalalalala' frameborder=0 allowfullscreen></iframe>") //l.Add("www.youtube.com/watch?v=lalalalala","<iframe class='postIframe'src='https://www.youtube-nocookie.com/embed/lalalalala'frameborder=0 allowfullscreen></iframe>")
l.Add("#tid-1", "<a href='/topic/1'>#tid-1</a>") l.Add("#tid-1", "<a href='/topic/1'>#tid-1</a>")
l.Add("##tid-1", "##tid-1") l.Add("##tid-1", "##tid-1")
@ -305,11 +312,11 @@ func TestParser(t *testing.T) {
l.Add("@2 t", "<red>[Invalid Profile]</red> t") l.Add("@2 t", "<red>[Invalid Profile]</red> t")
l.Add("@2 ", "<red>[Invalid Profile]</red> ") l.Add("@2 ", "<red>[Invalid Profile]</red> ")
l.Add("@2 @2", "<red>[Invalid Profile]</red> <red>[Invalid Profile]</red>") l.Add("@2 @2", "<red>[Invalid Profile]</red> <red>[Invalid Profile]</red>")
l.Add("@1", "<a href='/user/admin.1' class='mention'>@Admin</a>") l.Add("@1", "<a href='/user/admin.1'class='mention'>@Admin</a>")
l.Add(" @1", " <a href='/user/admin.1' class='mention'>@Admin</a>") l.Add(" @1", " <a href='/user/admin.1'class='mention'>@Admin</a>")
l.Add("@1t", "<a href='/user/admin.1' class='mention'>@Admin</a>t") l.Add("@1t", "<a href='/user/admin.1'class='mention'>@Admin</a>t")
l.Add("@1 ", "<a href='/user/admin.1' class='mention'>@Admin</a> ") l.Add("@1 ", "<a href='/user/admin.1'class='mention'>@Admin</a> ")
l.Add("@1 @1", "<a href='/user/admin.1' class='mention'>@Admin</a> <a href='/user/admin.1' class='mention'>@Admin</a>") l.Add("@1 @1", "<a href='/user/admin.1'class='mention'>@Admin</a> <a href='/user/admin.1'class='mention'>@Admin</a>")
l.Add("@0", "<red>[Invalid Profile]</red>") l.Add("@0", "<red>[Invalid Profile]</red>")
l.Add("@-1", "<red>[Invalid Profile]</red>1") l.Add("@-1", "<red>[Invalid Profile]</red>1")