511 lines
17 KiB
Go
511 lines
17 KiB
Go
package common
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
|
|
tmpl "git.tuxpa.in/a/gosora/tmpl_client"
|
|
"github.com/andybalholm/brotli"
|
|
)
|
|
|
|
//type SFileList map[string]*SFile
|
|
//type SFileListShort map[string]*SFile
|
|
|
|
var StaticFiles = SFileList{"/s/", make(map[string]*SFile), make(map[string]*SFile)}
|
|
|
|
//var StaticFilesShort SFileList = make(map[string]*SFile)
|
|
var staticFileMutex sync.RWMutex
|
|
|
|
// ? Is it efficient to have two maps for this?
|
|
type SFileList struct {
|
|
Prefix string
|
|
Long map[string]*SFile
|
|
Short map[string]*SFile
|
|
}
|
|
|
|
type SFile struct {
|
|
// TODO: Move these to the end?
|
|
Data []byte
|
|
GzipData []byte
|
|
BrData []byte
|
|
|
|
Sha256 string
|
|
Sha256I string
|
|
OName string
|
|
Pos int64
|
|
|
|
Length int64
|
|
StrLength string
|
|
GzipLength int64
|
|
StrGzipLength string
|
|
BrLength int64
|
|
StrBrLength string
|
|
|
|
Mimetype string
|
|
Info os.FileInfo
|
|
FormattedModTime string
|
|
}
|
|
|
|
type CSSData struct {
|
|
Phrases map[string]string
|
|
}
|
|
|
|
func (l SFileList) JSTmplInit() error {
|
|
DebugLog("Initialising the client side templates")
|
|
return filepath.Walk("./tmpl_client", func(path string, f os.FileInfo, err error) error {
|
|
if f.IsDir() || strings.HasSuffix(path, "tmpl_list.go") || strings.HasSuffix(path, "stub.go") {
|
|
return nil
|
|
}
|
|
path = strings.Replace(path, "\\", "/", -1)
|
|
DebugLog("Processing client template " + path)
|
|
data, err := ioutil.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path = strings.TrimPrefix(path, "tmpl_client/")
|
|
tmplName := strings.TrimSuffix(path, ".jgo")
|
|
shortName := strings.TrimPrefix(tmplName, "tmpl_")
|
|
|
|
replace := func(data []byte, replaceThis, withThis string) []byte {
|
|
return bytes.Replace(data, []byte(replaceThis), []byte(withThis), -1)
|
|
}
|
|
rep := func(replaceThis, withThis string) {
|
|
data = replace(data, replaceThis, withThis)
|
|
}
|
|
|
|
startIndex, hasFunc := skipAllUntilCharsExist(data, 0, []byte("if(tmplInits===undefined)"))
|
|
if !hasFunc {
|
|
return errors.New("no init map found")
|
|
}
|
|
data = data[startIndex-len([]byte("if(tmplInits===undefined)")):]
|
|
rep("// nolint", "")
|
|
//rep("func ", "function ")
|
|
rep("func ", "function ")
|
|
rep(" error {\n", " {\nlet o=\"\"\n")
|
|
funcIndex, hasFunc := skipAllUntilCharsExist(data, 0, []byte("function Tmpl_"))
|
|
if !hasFunc {
|
|
return errors.New("no template function found")
|
|
}
|
|
spaceIndex, hasSpace := skipUntilIfExists(data, funcIndex, ' ')
|
|
if !hasSpace {
|
|
return errors.New("no spaces found after the template function name")
|
|
}
|
|
endBrace, hasBrace := skipUntilIfExists(data, spaceIndex, ')')
|
|
if !hasBrace {
|
|
return errors.New("no right brace found after the template function name")
|
|
}
|
|
fmt.Println("spaceIndex: ", spaceIndex)
|
|
fmt.Println("endBrace: ", endBrace)
|
|
fmt.Println("string(data[spaceIndex:endBrace]): ", string(data[spaceIndex:endBrace]))
|
|
|
|
preLen := len(data)
|
|
rep(string(data[spaceIndex:endBrace]), "")
|
|
rep("))\n", " \n")
|
|
endBrace -= preLen - len(data) // Offset it as we've deleted portions
|
|
fmt.Println("new endBrace: ", endBrace)
|
|
fmt.Println("data: ", string(data))
|
|
|
|
/*showPos := func(data []byte, index int) (out string) {
|
|
out = "["
|
|
for j, char := range data {
|
|
if index == j {
|
|
out += "[" + string(char) + "] "
|
|
} else {
|
|
out += string(char) + " "
|
|
}
|
|
}
|
|
return out + "]"
|
|
}*/
|
|
|
|
// ? Can we just use a regex? I'm thinking of going more efficient, or just outright rolling wasm, this is a temp hack in a place where performance doesn't particularly matter
|
|
each := func(phrase string, h func(index int)) {
|
|
//fmt.Println("find each '" + phrase + "'")
|
|
index := endBrace
|
|
if index < 0 {
|
|
panic("index under zero: " + strconv.Itoa(index))
|
|
}
|
|
var foundIt bool
|
|
for {
|
|
//fmt.Println("in index: ", index)
|
|
//fmt.Println("pos: ", showPos(data, index))
|
|
index, foundIt = skipAllUntilCharsExist(data, index, []byte(phrase))
|
|
if !foundIt {
|
|
break
|
|
}
|
|
h(index)
|
|
}
|
|
}
|
|
each("strconv.Itoa(", func(index int) {
|
|
braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
|
|
if hasEndBrace {
|
|
data[braceAt] = ' ' // Blank it
|
|
}
|
|
})
|
|
each("[]byte(", func(index int) {
|
|
braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
|
|
if hasEndBrace {
|
|
data[braceAt] = ' ' // Blank it
|
|
}
|
|
})
|
|
each("StringToBytes(", func(index int) {
|
|
braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
|
|
if hasEndBrace {
|
|
data[braceAt] = ' ' // Blank it
|
|
}
|
|
})
|
|
each("w.Write(", func(index int) {
|
|
braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
|
|
if hasEndBrace {
|
|
data[braceAt] = ' ' // Blank it
|
|
}
|
|
})
|
|
each("RelativeTime(", func(index int) {
|
|
braceAt, _ := skipUntilIfExistsOrLine(data, index, 10)
|
|
if data[braceAt-1] == ' ' {
|
|
data[braceAt-1] = ' ' // Blank it
|
|
}
|
|
})
|
|
each("if ", func(index int) {
|
|
//fmt.Println("if index: ", index)
|
|
braceAt, hasBrace := skipUntilIfExistsOrLine(data, index, '{')
|
|
if hasBrace {
|
|
if data[braceAt-1] != ' ' {
|
|
panic("couldn't find space before brace, found ' " + string(data[braceAt-1]) + "' instead")
|
|
}
|
|
data[braceAt-1] = ')' // Drop a brace here to satisfy JS
|
|
}
|
|
})
|
|
each("for _, item := range ", func(index int) {
|
|
//fmt.Println("for index: ", index)
|
|
braceAt, hasBrace := skipUntilIfExists(data, index, '{')
|
|
if hasBrace {
|
|
if data[braceAt-1] != ' ' {
|
|
panic("couldn't find space before brace, found ' " + string(data[braceAt-1]) + "' instead")
|
|
}
|
|
data[braceAt-1] = ')' // Drop a brace here to satisfy JS
|
|
}
|
|
})
|
|
rep("for _, item := range ", "for(item of ")
|
|
rep("w.Write([]byte(", "o += ")
|
|
rep("w.Write(StringToBytes(", "o += ")
|
|
rep("w.Write(", "o += ")
|
|
rep("+= c.", "+= ")
|
|
rep("strconv.Itoa(", "")
|
|
rep("strconv.FormatInt(", "")
|
|
rep(" c.", "")
|
|
rep("phrases.", "")
|
|
rep(", 10;", "")
|
|
|
|
//rep("var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const plist = tmplPhrases[\""+tmplName+"\"];")
|
|
//rep("//var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const "+shortName+"_phrase_arr = tmplPhrases[\""+tmplName+"\"];")
|
|
rep("//var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const pl=tmplPhrases[\""+tmplName+"\"];")
|
|
rep(shortName+"_phrase_arr", "pl")
|
|
rep(shortName+"_phrase", "pl")
|
|
rep("tmpl_"+shortName+"_vars", "t_v")
|
|
|
|
rep("var c_v_", "let c_v_")
|
|
rep(`t_vars, ok := tmpl_i.`, `/*`)
|
|
rep("[]byte(", "")
|
|
rep("StringToBytes(", "")
|
|
rep("RelativeTime(t_v.", "t_v.Relative")
|
|
// TODO: Format dates properly on the client side
|
|
rep(".Format(\"2006-01-02 15:04:05\"", "")
|
|
rep(", 10", "")
|
|
rep("if ", "if(")
|
|
rep("return nil", "return o")
|
|
rep(" )", ")")
|
|
rep(" \n", "\n")
|
|
rep("\n", ";\n")
|
|
rep("{;", "{")
|
|
rep("};", "}")
|
|
rep("[;", "[")
|
|
rep(",;", ",")
|
|
rep("=;", "=")
|
|
rep(`,
|
|
});
|
|
}`, "\n\t];")
|
|
rep(`=
|
|
}`, "=[]")
|
|
rep("o += ", "o+=")
|
|
rep(shortName+"_frags[", "fr[")
|
|
rep("function Tmpl_"+shortName+"(t_v) {", "var Tmpl_"+shortName+"=(t_v)=>{")
|
|
|
|
fragset := tmpl.GetFrag(shortName)
|
|
if fragset != nil {
|
|
//sfrags := []byte("let " + shortName + "_frags=[\n")
|
|
sfrags := []byte("{const fr=[")
|
|
for i, frags := range fragset {
|
|
//sfrags = append(sfrags, []byte(shortName+"_frags.push(`"+string(frags)+"`);\n")...)
|
|
//sfrags = append(sfrags, []byte("`"+string(frags)+"`,\n")...)
|
|
if i == 0 {
|
|
sfrags = append(sfrags, []byte("`"+string(frags)+"`")...)
|
|
} else {
|
|
sfrags = append(sfrags, []byte(",`"+string(frags)+"`")...)
|
|
}
|
|
}
|
|
//sfrags = append(sfrags, []byte("];\n")...)
|
|
sfrags = append(sfrags, []byte("];")...)
|
|
data = append(sfrags, data...)
|
|
}
|
|
rep("\n;", "\n")
|
|
rep(";;", ";")
|
|
|
|
data = append(data, '}')
|
|
for name, _ := range Themes {
|
|
if strings.HasSuffix(shortName, "_"+name) {
|
|
data = append(data, "var Tmpl_"+strings.TrimSuffix(shortName, "_"+name)+"=Tmpl_"+shortName+";"...)
|
|
break
|
|
}
|
|
}
|
|
|
|
path = tmplName + ".js"
|
|
DebugLog("js path: ", path)
|
|
ext := filepath.Ext("/tmpl_client/" + path)
|
|
|
|
brData, err := CompressBytesBrotli(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
|
|
if len(brData) >= (len(data) + 110) {
|
|
brData = nil
|
|
} else {
|
|
diff := len(data) - len(brData)
|
|
if diff <= len(data)/100 {
|
|
brData = nil
|
|
}
|
|
}
|
|
|
|
gzipData, err := CompressBytesGzip(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
|
|
if len(gzipData) >= (len(data) + 120) {
|
|
gzipData = nil
|
|
} else {
|
|
diff := len(data) - len(gzipData)
|
|
if diff <= len(data)/100 {
|
|
gzipData = nil
|
|
}
|
|
}
|
|
|
|
// Get a checksum for CSPs and cache busting
|
|
hasher := sha256.New()
|
|
hasher.Write(data)
|
|
sum := hasher.Sum(nil)
|
|
checksum := hex.EncodeToString(sum)
|
|
integrity := base64.StdEncoding.EncodeToString(sum)
|
|
|
|
l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
|
|
|
|
DebugLogf("Added the '%s' static file.", path)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (l SFileList) Init() error {
|
|
return filepath.Walk("./public", func(path string, f os.FileInfo, err error) error {
|
|
if f.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
path = strings.Replace(path, "\\", "/", -1)
|
|
data, err := ioutil.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
path = strings.TrimPrefix(path, "public/")
|
|
ext := filepath.Ext("/public/" + path)
|
|
if ext == ".js" {
|
|
data = bytes.Replace(data, []byte("\r"), []byte(""), -1)
|
|
}
|
|
mimetype := mime.TypeByExtension(ext)
|
|
|
|
// Get a checksum for CSPs and cache busting
|
|
hasher := sha256.New()
|
|
hasher.Write(data)
|
|
sum := hasher.Sum(nil)
|
|
checksum := hex.EncodeToString(sum)
|
|
integrity := base64.StdEncoding.EncodeToString(sum)
|
|
|
|
// Avoid double-compressing images
|
|
var gzipData, brData []byte
|
|
if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" {
|
|
brData, err = CompressBytesBrotli(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
|
|
if len(brData) >= (len(data) + 130) {
|
|
brData = nil
|
|
} else {
|
|
diff := len(data) - len(brData)
|
|
if diff <= len(data)/100 {
|
|
brData = nil
|
|
}
|
|
}
|
|
|
|
gzipData, err = CompressBytesGzip(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
|
|
if len(gzipData) >= (len(data) + 150) {
|
|
gzipData = nil
|
|
} else {
|
|
diff := len(data) - len(gzipData)
|
|
if diff <= len(data)/100 {
|
|
gzipData = nil
|
|
}
|
|
}
|
|
}
|
|
|
|
l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})
|
|
|
|
DebugLogf("Added the '%s' static file.", path)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (l SFileList) Add(path, prefix string) error {
|
|
data, err := ioutil.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fi, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f, err := fi.Stat()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ext := filepath.Ext(path)
|
|
path = strings.TrimPrefix(path, prefix)
|
|
|
|
brData, err := CompressBytesBrotli(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
|
|
if len(brData) >= (len(data) + 130) {
|
|
brData = nil
|
|
} else {
|
|
diff := len(data) - len(brData)
|
|
if diff <= len(data)/100 {
|
|
brData = nil
|
|
}
|
|
}
|
|
|
|
gzipData, err := CompressBytesGzip(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
|
|
if len(gzipData) >= (len(data) + 150) {
|
|
gzipData = nil
|
|
} else {
|
|
diff := len(data) - len(gzipData)
|
|
if diff <= len(data)/100 {
|
|
gzipData = nil
|
|
}
|
|
}
|
|
|
|
// Get a checksum for CSPs and cache busting
|
|
hasher := sha256.New()
|
|
hasher.Write(data)
|
|
sum := hasher.Sum(nil)
|
|
checksum := hex.EncodeToString(sum)
|
|
integrity := base64.StdEncoding.EncodeToString(sum)
|
|
|
|
l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})
|
|
|
|
DebugLogf("Added the '%s' static file", path)
|
|
return nil
|
|
}
|
|
|
|
func (l SFileList) Get(path string) (file *SFile, exists bool) {
|
|
staticFileMutex.RLock()
|
|
defer staticFileMutex.RUnlock()
|
|
file, exists = l.Long[path]
|
|
return file, exists
|
|
}
|
|
|
|
// fetch without /s/ to avoid allocing in pages.go
|
|
func (l SFileList) GetShort(name string) (file *SFile, exists bool) {
|
|
staticFileMutex.RLock()
|
|
defer staticFileMutex.RUnlock()
|
|
file, exists = l.Short[name]
|
|
return file, exists
|
|
}
|
|
|
|
func (l SFileList) Set(name string, data *SFile) {
|
|
staticFileMutex.Lock()
|
|
defer staticFileMutex.Unlock()
|
|
// TODO: Propagate errors back up
|
|
uurl, err := url.Parse(name)
|
|
if err != nil {
|
|
return
|
|
}
|
|
l.Long[uurl.Path] = data
|
|
l.Short[strings.TrimPrefix(strings.TrimPrefix(name, l.Prefix), "/")] = data
|
|
}
|
|
|
|
var gzipBestCompress sync.Pool
|
|
|
|
func CompressBytesGzip(in []byte) (b []byte, err error) {
|
|
var buf bytes.Buffer
|
|
ii := gzipBestCompress.Get()
|
|
var gz *gzip.Writer
|
|
if ii == nil {
|
|
gz, err = gzip.NewWriterLevel(&buf, gzip.BestCompression)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
gz = ii.(*gzip.Writer)
|
|
gz.Reset(&buf)
|
|
}
|
|
_, err = gz.Write(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
err = gz.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gzipBestCompress.Put(gz)
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func CompressBytesBrotli(in []byte) ([]byte, error) {
|
|
var buff bytes.Buffer
|
|
br := brotli.NewWriterLevel(&buff, brotli.BestCompression)
|
|
_, err := br.Write(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
err = br.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return buff.Bytes(), nil
|
|
}
|