From c32498d543864ada7eb452ff989b2d8372286fc1 Mon Sep 17 00:00:00 2001 From: Azareal Date: Thu, 30 Apr 2020 16:28:01 +1000 Subject: [PATCH] use a video player for uploaded videos optimise the parser --- common/parser.go | 127 +++++++++++++++++++++++++++++++++++------------ parser_test.go | 7 ++- 2 files changed, 102 insertions(+), 32 deletions(-) diff --git a/common/parser.go b/common/parser.go index 11b61d74..59ca1526 100644 --- a/common/parser.go +++ b/common/parser.go @@ -28,19 +28,26 @@ var URLOpen = []byte("") var bytesSinglequote = []byte("'") -var bytesGreaterthan = []byte(">") +var bytesGreaterThan = []byte(">") var urlMention = []byte("'class='mention'") var URLClose = []byte("") +var videoOpen = []byte("") var imageOpen = []byte("") -var attachOpen = []byte("Attachment") +var attachOpen = []byte("Attachment") var sidParam = []byte("?sid=") var stypeParam = []byte("&stype=") var urlPattern = `(?s)([ {1}])((http|https|ftp|mailto)*)(:{??)\/\/([\.a-zA-Z\/]+)([ {1}])` var urlReg *regexp.Regexp +const imageSizeHint = len("") +const videoSizeHint = len("") + len("?sid=") + len("&stype=") + 8 +const mentionSizeHint = len("@") + len("") + func init() { urlReg = regexp.MustCompile(urlPattern) } @@ -492,8 +499,14 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse LogError(err) return "" } - for _, filter := range wordFilters { - msg = strings.Replace(msg, filter.Find, filter.Replace, -1) + for _, f := range wordFilters { + msg = strings.Replace(msg, f.Find, f.Replace, -1) + } + + if len(msg) < 2 { + msg = strings.Replace(msg, "\n", "
", -1) + msg = GetHookTable().Sshook("parse_assign", msg) + return msg } // Search for URLs, mentions and hashlinks in the messages... @@ -513,7 +526,9 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse break } //fmt.Println("s2") - if msg[i] == '#' { + ch := msg[i] + switch ch { + case '#': //fmt.Println("msg[i+1]:", msg[i+1]) //fmt.Println("string(msg[i+1]):", string(msg[i+1])) hashType := hashLinkTypes[msg[i+1]] @@ -540,31 +555,37 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse hashLinkMap[hashType](&sb, msg, &i) lastItem = i i-- - } else if msg[i] == '@' { + case '@': sb.WriteString(msg[lastItem:i]) i++ start := i uid, intLen := CoerceIntString(msg[start:]) i += intLen - menUser, err := Users.Get(uid) - if err != nil { - sb.Write(InvalidProfile) - lastItem = i - i-- - continue + var menUser *User + if uid != 0 && user.ID == uid { + menUser = user + } else { + menUser = Users.Getn(uid) + if menUser == nil { + sb.Write(InvalidProfile) + lastItem = i + i-- + continue + } } + sb.Grow(mentionSizeHint + len(menUser.Link) + len(menUser.Name)) sb.Write(URLOpen) sb.WriteString(menUser.Link) sb.Write(urlMention) - sb.Write(bytesGreaterthan) + sb.Write(bytesGreaterThan) sb.WriteByte('@') sb.WriteString(menUser.Name) sb.Write(URLClose) lastItem = i i-- - } else if msg[i] == 'h' || msg[i] == 'f' || msg[i] == 'g' || msg[i] == '/' { + case 'h', 'f', 'g', '/': //fmt.Println("s3") if len(msg) > i+5 && msg[i+1] == 't' && msg[i+2] == 't' && msg[i+3] == 'p' { if len(msg) > i+6 && msg[i+4] == 's' && msg[i+5] == ':' && msg[i+6] == '/' { @@ -574,10 +595,17 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse } else { continue } - } else if len(msg) > i+4 && msg[i+1] == 't' && msg[i+2] == 'p' && msg[i+3] == ':' && msg[i+4] == '/' { - // Do nothing - } else if len(msg) > i+4 && msg[i+1] == 'i' && msg[i+2] == 't' && msg[i+3] == ':' && msg[i+4] == '/' { - // Do nothing + } else if len(msg) > i+4 { + fch := msg[i+1] + if fch == 't' && msg[i+2] == 'p' && msg[i+3] == ':' && msg[i+4] == '/' { + // Do nothing + } else if fch == 'i' && msg[i+2] == 't' && msg[i+3] == ':' && msg[i+4] == '/' { + // Do nothing + } else if fch == '/' { + // Do nothing + } else { + continue + } } else if msg[i+1] == '/' { // Do nothing } else { @@ -626,7 +654,7 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse //fmt.Println("p2") addImage := func(url string) { - sb.Grow(len(imageOpen) + len(url) + len(url) + len(imageOpen2) + len(imageClose)) + sb.Grow(imageSizeHint + len(url) + len(url)) sb.Write(imageOpen) sb.WriteString(url) sb.Write(imageOpen2) @@ -638,13 +666,32 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse // TODO: Reduce the amount of code duplication // TODO: Avoid allocating a string for media.Type? - if media.Type == "attach" { + switch media.Type { + case AImage: addImage(media.URL + "?sid=" + strconv.Itoa(sectionID) + "&stype=" + sectionType) continue - } else if media.Type == "image" { + case AVideo: + sb.Grow(videoSizeHint + (len(media.URL) + len(sectionType)*2)) + sb.Write(videoOpen) + sb.WriteString(media.URL) + sb.Write(sidParam) + sb.WriteString(strconv.Itoa(sectionID)) + sb.Write(stypeParam) + sb.WriteString(sectionType) + sb.Write(videoOpen2) + sb.WriteString(media.URL) + sb.Write(sidParam) + sb.WriteString(strconv.Itoa(sectionID)) + sb.Write(stypeParam) + sb.WriteString(sectionType) + sb.Write(videoClose) + i += urlLen + lastItem = i + continue + case EImage: addImage(media.URL) continue - } else if media.Type == "aother" { + case AOther: sb.Write(attachOpen) sb.WriteString(media.URL) sb.Write(sidParam) @@ -655,12 +702,15 @@ func ParseMessage(msg string, sectionID int, sectionType string, settings *Parse i += urlLen lastItem = i continue - } else if media.Type == "raw" { + case ERaw: sb.WriteString(media.Body) i += urlLen lastItem = i continue - } else if media.Type != "" { + case ENone: + // Do nothing + // TODO: Add support for media plugins + default: sb.Write(unknownMedia) i += urlLen continue @@ -863,7 +913,8 @@ func PartialURLStringLen2(data string) int { } type MediaEmbed struct { - Type string //image + //Type string //image + Type int URL string FURL string Body string @@ -871,6 +922,17 @@ type MediaEmbed struct { Trusted bool // samesite urls } +const ( + ENone = iota + ERaw + EImage + AImage + AVideo + AOther +) + +var LastEmbedID = AOther + // TODO: Write a test for this func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, ok bool) { if !validateURLString(data) { @@ -920,9 +982,11 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o return media, false } if ImageFileExts.Contains(ext) { - media.Type = "attach" + media.Type = AImage + } else if WebVideoFileExts.Contains(ext) { + media.Type = AVideo } else { - media.Type = "aother" + media.Type = AOther } return media, true } @@ -934,7 +998,7 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o if strings.HasSuffix(host, ".youtube.com") && path == "/watch" { video, ok := query["v"] if ok && len(video) >= 1 && video[0] != "" { - media.Type = "raw" + media.Type = ERaw // TODO: Filter the URL to make sure no nasties end up in there media.Body = "" return media, true @@ -943,10 +1007,10 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o if lastFrag := pathFrags[len(pathFrags)-1]; lastFrag != "" { // TODO: Write a function for getting the file extension of a string - ext := strings.TrimPrefix(filepath.Ext(lastFrag),".") + ext := strings.TrimPrefix(filepath.Ext(lastFrag), ".") if len(ext) != 0 { if ImageFileExts.Contains(ext) { - media.Type = "image" + media.Type = EImage var sport string if port != "443" && port != "80" && port != "" { sport = ":" + port @@ -954,6 +1018,7 @@ func parseMediaString(data string, settings *ParseSettings) (media MediaEmbed, o media.URL = scheme + "//" + host + sport + path return media, true } + // TODO: Support external videos } } } diff --git a/parser_test.go b/parser_test.go index 168e26b3..19a606ad 100644 --- a/parser_test.go +++ b/parser_test.go @@ -232,6 +232,11 @@ func TestParser(t *testing.T) { l.Add("http://"+u, ""+c.Site.URL+"") l.Add("https://"+u, ""+c.Site.URL+"") + l.Add("//"+u+"/attachs/sha256hash.webm?sid=1&stype=forums", "") + l.Add("//"+u+"/attachs/sha256hash.webm", "") + l.Add("//"+u+"/attachs/sha256hash.webm?sid=1", "") + l.Add("//"+u+"/attachs/sha256hash.webm?stype=forums", "") + l.Add("//"+u+"/attachs/sha256hash.png?sid=1&stype=forums", "") l.Add("//"+u+"/attachs/sha256hash?sid=1&stype=forums", "[Invalid URL]") l.Add("//"+u+"/attachs/s?sid=1&stype=forums", "[Invalid URL]") @@ -242,7 +247,7 @@ func TestParser(t *testing.T) { l.Add("//"+u+"/attachs/sha256hash.png?sid=1", "") l.Add("//"+u+"/attachs/sha256hash.png?stype=forums", "") - l.Add("//"+u+"/attachs/sha256hash.txt?sid=1&stype=forums", "Attachment") + l.Add("//"+u+"/attachs/sha256hash.txt?sid=1&stype=forums", "Attachment") l.Add("//example.com/image.png", "") l.Add("https://example.com/image.png", "")