2017-11-11 04:06:16 +00:00
package common
2017-02-10 13:39:13 +00:00
2017-06-25 09:56:39 +00:00
import (
"bytes"
2018-09-26 08:05:25 +00:00
"compress/gzip"
2019-03-11 08:47:45 +00:00
"crypto/sha256"
2020-07-31 05:33:29 +00:00
"encoding/base64"
2019-03-11 08:47:45 +00:00
"encoding/hex"
2018-05-14 08:56:56 +00:00
"errors"
2018-06-24 13:49:29 +00:00
"fmt"
2017-06-25 09:56:39 +00:00
"io/ioutil"
2018-09-26 08:05:25 +00:00
"mime"
2017-06-25 09:56:39 +00:00
"net/http"
2020-07-30 08:10:29 +00:00
"net/url"
2017-09-03 04:50:31 +00:00
"os"
"path/filepath"
2018-09-26 08:05:25 +00:00
"strconv"
"strings"
"sync"
2018-05-14 08:56:56 +00:00
2020-02-18 23:04:14 +00:00
tmpl "github.com/Azareal/Gosora/tmpl_client"
2020-04-13 03:28:52 +00:00
"github.com/andybalholm/brotli"
2017-06-25 09:56:39 +00:00
)
2016-12-05 07:21:17 +00:00
2020-07-26 04:36:40 +00:00
//type SFileList map[string]*SFile
//type SFileListShort map[string]*SFile
2017-11-11 04:06:16 +00:00
2020-07-30 08:10:29 +00:00
var StaticFiles = SFileList { "/s/" , make ( map [ string ] * SFile ) , make ( map [ string ] * SFile ) }
2020-07-26 04:36:40 +00:00
//var StaticFilesShort SFileList = make(map[string]*SFile)
2017-12-01 02:04:29 +00:00
var staticFileMutex sync . RWMutex
2017-11-11 04:06:16 +00:00
2020-07-26 04:36:40 +00:00
// ? Is it efficient to have two maps for this?
type SFileList struct {
2020-07-30 08:10:29 +00:00
Prefix string
Long map [ string ] * SFile
Short map [ string ] * SFile
2020-07-26 04:36:40 +00:00
}
2017-09-03 04:50:31 +00:00
type SFile struct {
2020-04-13 03:28:52 +00:00
// TODO: Move these to the end?
Data [ ] byte
GzipData [ ] byte
BrData [ ] byte
Sha256 string
2020-07-31 05:33:29 +00:00
Sha256I string
2020-04-13 03:28:52 +00:00
OName string
Pos int64
Length int64
StrLength string
GzipLength int64
StrGzipLength string
BrLength int64
StrBrLength string
2017-09-03 04:50:31 +00:00
Mimetype string
Info os . FileInfo
2016-12-05 07:21:17 +00:00
FormattedModTime string
}
2017-09-18 17:03:52 +00:00
type CSSData struct {
2018-03-11 09:33:49 +00:00
Phrases map [ string ] string
2016-12-05 07:21:17 +00:00
}
2020-07-26 04:36:40 +00:00
func ( l SFileList ) JSTmplInit ( ) error {
2018-05-15 05:59:52 +00:00
DebugLog ( "Initialising the client side templates" )
2018-05-14 10:21:18 +00:00
return filepath . Walk ( "./tmpl_client" , func ( path string , f os . FileInfo , err error ) error {
2020-03-22 23:14:08 +00:00
if f . IsDir ( ) || strings . HasSuffix ( path , "tmpl_list.go" ) || strings . HasSuffix ( path , "stub.go" ) {
2018-05-14 08:56:56 +00:00
return nil
}
path = strings . Replace ( path , "\\" , "/" , - 1 )
DebugLog ( "Processing client template " + path )
data , err := ioutil . ReadFile ( path )
if err != nil {
return err
}
2018-06-24 13:49:29 +00:00
path = strings . TrimPrefix ( path , "tmpl_client/" )
2019-04-28 10:08:05 +00:00
tmplName := strings . TrimSuffix ( path , ".jgo" )
2020-03-22 23:14:08 +00:00
shortName := strings . TrimPrefix ( tmplName , "tmpl_" )
2018-06-24 13:49:29 +00:00
Cascade delete attachments properly.
Cascade delete replied to topic events for replies properly.
Cascade delete likes on topic posts properly.
Cascade delete replies and their children properly.
Recalculate user stats properly when items are deleted.
Users can now unlike topic opening posts.
Add a recalculator to fix abnormalities across upgrades.
Try fixing a last_ip daily update bug.
Add Existable interface.
Add Delete method to LikeStore.
Add Each, Exists, Create, CountUser, CountMegaUser and CountBigUser methods to ReplyStore.
Add CountUser, CountMegaUser, CountBigUser methods to TopicStore.
Add Each method to UserStore.
Add Add, Delete and DeleteResource methods to SubscriptionStore.
Add Delete, DeleteByParams, DeleteByParamsExtra and AidsByParamsExtra methods to ActivityStream.
Add Exists method to ProfileReplyStore.
Add DropColumn, RenameColumn and ChangeColumn to the database adapters.
Shorten ipaddress column names to ip.
- topics table.
- replies table
- users_replies table.
- polls_votes table.
Add extra column to activity_stream table.
Fix an issue upgrading sites to MariaDB 10.3 from older versions of Gosora. Please report any other issues you find.
You need to run the updater / patcher for this commit.
2020-01-31 07:22:08 +00:00
replace := func ( data [ ] byte , replaceThis , withThis string ) [ ] byte {
2018-05-14 08:56:56 +00:00
return bytes . Replace ( data , [ ] byte ( replaceThis ) , [ ] byte ( withThis ) , - 1 )
}
2020-03-22 02:37:32 +00:00
rep := func ( replaceThis , withThis string ) {
data = replace ( data , replaceThis , withThis )
}
2018-05-14 08:56:56 +00:00
2019-11-07 03:53:29 +00:00
startIndex , hasFunc := skipAllUntilCharsExist ( data , 0 , [ ] byte ( "if(tmplInits===undefined)" ) )
2018-05-14 08:56:56 +00:00
if ! hasFunc {
2019-11-07 03:53:29 +00:00
return errors . New ( "no init map found" )
2018-05-14 08:56:56 +00:00
}
2019-11-07 03:53:29 +00:00
data = data [ startIndex - len ( [ ] byte ( "if(tmplInits===undefined)" ) ) : ]
2020-03-22 02:37:32 +00:00
rep ( "// nolint" , "" )
2020-03-22 23:14:08 +00:00
//rep("func ", "function ")
2020-03-22 02:37:32 +00:00
rep ( "func " , "function " )
2020-03-22 23:14:08 +00:00
rep ( " error {\n" , " {\nlet o=\"\"\n" )
funcIndex , hasFunc := skipAllUntilCharsExist ( data , 0 , [ ] byte ( "function Tmpl_" ) )
2018-06-24 13:49:29 +00:00
if ! hasFunc {
return errors . New ( "no template function found" )
}
spaceIndex , hasSpace := skipUntilIfExists ( data , funcIndex , ' ' )
2018-05-14 08:56:56 +00:00
if ! hasSpace {
return errors . New ( "no spaces found after the template function name" )
}
endBrace , hasBrace := skipUntilIfExists ( data , spaceIndex , ')' )
if ! hasBrace {
return errors . New ( "no right brace found after the template function name" )
}
2018-06-24 13:49:29 +00:00
fmt . Println ( "spaceIndex: " , spaceIndex )
fmt . Println ( "endBrace: " , endBrace )
fmt . Println ( "string(data[spaceIndex:endBrace]): " , string ( data [ spaceIndex : endBrace ] ) )
2018-05-14 08:56:56 +00:00
preLen := len ( data )
2020-03-22 02:37:32 +00:00
rep ( string ( data [ spaceIndex : endBrace ] ) , "" )
rep ( "))\n" , " \n" )
2018-05-14 08:56:56 +00:00
endBrace -= preLen - len ( data ) // Offset it as we've deleted portions
2018-06-24 13:49:29 +00:00
fmt . Println ( "new endBrace: " , endBrace )
fmt . Println ( "data: " , string ( data ) )
2018-05-14 08:56:56 +00:00
2019-10-26 23:11:09 +00:00
/ * showPos := func ( data [ ] byte , index int ) ( out string ) {
2018-05-14 08:56:56 +00:00
out = "["
for j , char := range data {
if index == j {
out += "[" + string ( char ) + "] "
} else {
out += string ( char ) + " "
}
}
return out + "]"
2018-05-15 05:59:52 +00:00
} * /
2018-05-14 08:56:56 +00:00
// ? Can we just use a regex? I'm thinking of going more efficient, or just outright rolling wasm, this is a temp hack in a place where performance doesn't particularly matter
Cascade delete attachments properly.
Cascade delete replied to topic events for replies properly.
Cascade delete likes on topic posts properly.
Cascade delete replies and their children properly.
Recalculate user stats properly when items are deleted.
Users can now unlike topic opening posts.
Add a recalculator to fix abnormalities across upgrades.
Try fixing a last_ip daily update bug.
Add Existable interface.
Add Delete method to LikeStore.
Add Each, Exists, Create, CountUser, CountMegaUser and CountBigUser methods to ReplyStore.
Add CountUser, CountMegaUser, CountBigUser methods to TopicStore.
Add Each method to UserStore.
Add Add, Delete and DeleteResource methods to SubscriptionStore.
Add Delete, DeleteByParams, DeleteByParamsExtra and AidsByParamsExtra methods to ActivityStream.
Add Exists method to ProfileReplyStore.
Add DropColumn, RenameColumn and ChangeColumn to the database adapters.
Shorten ipaddress column names to ip.
- topics table.
- replies table
- users_replies table.
- polls_votes table.
Add extra column to activity_stream table.
Fix an issue upgrading sites to MariaDB 10.3 from older versions of Gosora. Please report any other issues you find.
You need to run the updater / patcher for this commit.
2020-01-31 07:22:08 +00:00
each := func ( phrase string , h func ( index int ) ) {
2018-05-15 05:59:52 +00:00
//fmt.Println("find each '" + phrase + "'")
2019-10-26 23:11:09 +00:00
index := endBrace
2018-06-24 13:49:29 +00:00
if index < 0 {
panic ( "index under zero: " + strconv . Itoa ( index ) )
}
2018-05-14 08:56:56 +00:00
var foundIt bool
for {
2018-05-15 05:59:52 +00:00
//fmt.Println("in index: ", index)
//fmt.Println("pos: ", showPos(data, index))
2018-05-14 08:56:56 +00:00
index , foundIt = skipAllUntilCharsExist ( data , index , [ ] byte ( phrase ) )
if ! foundIt {
break
}
Cascade delete attachments properly.
Cascade delete replied to topic events for replies properly.
Cascade delete likes on topic posts properly.
Cascade delete replies and their children properly.
Recalculate user stats properly when items are deleted.
Users can now unlike topic opening posts.
Add a recalculator to fix abnormalities across upgrades.
Try fixing a last_ip daily update bug.
Add Existable interface.
Add Delete method to LikeStore.
Add Each, Exists, Create, CountUser, CountMegaUser and CountBigUser methods to ReplyStore.
Add CountUser, CountMegaUser, CountBigUser methods to TopicStore.
Add Each method to UserStore.
Add Add, Delete and DeleteResource methods to SubscriptionStore.
Add Delete, DeleteByParams, DeleteByParamsExtra and AidsByParamsExtra methods to ActivityStream.
Add Exists method to ProfileReplyStore.
Add DropColumn, RenameColumn and ChangeColumn to the database adapters.
Shorten ipaddress column names to ip.
- topics table.
- replies table
- users_replies table.
- polls_votes table.
Add extra column to activity_stream table.
Fix an issue upgrading sites to MariaDB 10.3 from older versions of Gosora. Please report any other issues you find.
You need to run the updater / patcher for this commit.
2020-01-31 07:22:08 +00:00
h ( index )
2018-05-14 08:56:56 +00:00
}
}
each ( "strconv.Itoa(" , func ( index int ) {
2018-12-27 05:42:41 +00:00
braceAt , hasEndBrace := skipUntilIfExistsOrLine ( data , index , ')' )
2018-05-14 08:56:56 +00:00
if hasEndBrace {
data [ braceAt ] = ' ' // Blank it
}
} )
2018-12-27 05:42:41 +00:00
each ( "[]byte(" , func ( index int ) {
braceAt , hasEndBrace := skipUntilIfExistsOrLine ( data , index , ')' )
2018-05-14 08:56:56 +00:00
if hasEndBrace {
data [ braceAt ] = ' ' // Blank it
}
} )
2018-12-27 05:42:41 +00:00
each ( "StringToBytes(" , func ( index int ) {
braceAt , hasEndBrace := skipUntilIfExistsOrLine ( data , index , ')' )
2018-11-22 07:21:43 +00:00
if hasEndBrace {
data [ braceAt ] = ' ' // Blank it
}
} )
2018-12-27 05:42:41 +00:00
each ( "w.Write(" , func ( index int ) {
braceAt , hasEndBrace := skipUntilIfExistsOrLine ( data , index , ')' )
2018-12-15 04:39:50 +00:00
if hasEndBrace {
data [ braceAt ] = ' ' // Blank it
}
} )
2018-12-27 05:42:41 +00:00
each ( "RelativeTime(" , func ( index int ) {
braceAt , _ := skipUntilIfExistsOrLine ( data , index , 10 )
if data [ braceAt - 1 ] == ' ' {
2019-02-10 05:52:26 +00:00
data [ braceAt - 1 ] = ' ' // Blank it
2018-05-14 08:56:56 +00:00
}
} )
each ( "if " , func ( index int ) {
2018-05-15 05:59:52 +00:00
//fmt.Println("if index: ", index)
2018-12-27 05:42:41 +00:00
braceAt , hasBrace := skipUntilIfExistsOrLine ( data , index , '{' )
2018-05-14 08:56:56 +00:00
if hasBrace {
if data [ braceAt - 1 ] != ' ' {
panic ( "couldn't find space before brace, found ' " + string ( data [ braceAt - 1 ] ) + "' instead" )
}
data [ braceAt - 1 ] = ')' // Drop a brace here to satisfy JS
}
} )
2018-06-24 13:49:29 +00:00
each ( "for _, item := range " , func ( index int ) {
//fmt.Println("for index: ", index)
braceAt , hasBrace := skipUntilIfExists ( data , index , '{' )
if hasBrace {
if data [ braceAt - 1 ] != ' ' {
panic ( "couldn't find space before brace, found ' " + string ( data [ braceAt - 1 ] ) + "' instead" )
}
data [ braceAt - 1 ] = ')' // Drop a brace here to satisfy JS
}
} )
2020-03-22 02:37:32 +00:00
rep ( "for _, item := range " , "for(item of " )
rep ( "w.Write([]byte(" , "o += " )
rep ( "w.Write(StringToBytes(" , "o += " )
rep ( "w.Write(" , "o += " )
rep ( "+= c." , "+= " )
rep ( "strconv.Itoa(" , "" )
rep ( "strconv.FormatInt(" , "" )
rep ( " c." , "" )
rep ( "phrases." , "" )
rep ( ", 10;" , "" )
2020-03-22 00:25:10 +00:00
2020-03-22 02:37:32 +00:00
//rep("var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const plist = tmplPhrases[\""+tmplName+"\"];")
//rep("//var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const "+shortName+"_phrase_arr = tmplPhrases[\""+tmplName+"\"];")
rep ( "//var plist = GetTmplPhrasesBytes(" + shortName + "_tmpl_phrase_id)" , "const pl=tmplPhrases[\"" + tmplName + "\"];" )
rep ( shortName + "_phrase_arr" , "pl" )
2020-04-29 00:26:15 +00:00
rep ( shortName + "_phrase" , "pl" )
2020-03-22 23:14:08 +00:00
rep ( "tmpl_" + shortName + "_vars" , "t_v" )
2020-03-22 00:25:10 +00:00
2020-03-22 02:37:32 +00:00
rep ( "var c_v_" , "let c_v_" )
rep ( ` t_vars, ok := tmpl_i. ` , ` /* ` )
rep ( "[]byte(" , "" )
rep ( "StringToBytes(" , "" )
2020-03-22 23:14:08 +00:00
rep ( "RelativeTime(t_v." , "t_v.Relative" )
2018-12-27 05:42:41 +00:00
// TODO: Format dates properly on the client side
2020-03-22 02:37:32 +00:00
rep ( ".Format(\"2006-01-02 15:04:05\"" , "" )
rep ( ", 10" , "" )
rep ( "if " , "if(" )
rep ( "return nil" , "return o" )
rep ( " )" , ")" )
rep ( " \n" , "\n" )
rep ( "\n" , ";\n" )
rep ( "{;" , "{" )
rep ( "};" , "}" )
rep ( "[;" , "[" )
rep ( ",;" , "," )
rep ( "=;" , "=" )
rep ( ` ,
2018-06-24 13:49:29 +00:00
} ) ;
} ` , "\n\t];" )
2020-03-22 02:37:32 +00:00
rep ( ` =
2020-03-05 04:43:40 +00:00
} ` , "=[]" )
2020-03-22 02:37:32 +00:00
rep ( "o += " , "o+=" )
2020-03-22 23:14:08 +00:00
rep ( shortName + "_frags[" , "fr[" )
2020-04-03 05:53:15 +00:00
rep ( "function Tmpl_" + shortName + "(t_v) {" , "var Tmpl_" + shortName + "=(t_v)=>{" )
2018-05-14 08:56:56 +00:00
2019-02-28 07:28:17 +00:00
fragset := tmpl . GetFrag ( shortName )
if fragset != nil {
2020-03-22 23:14:08 +00:00
//sfrags := []byte("let " + shortName + "_frags=[\n")
sfrags := [ ] byte ( "{const fr=[" )
for i , frags := range fragset {
2020-02-18 23:04:14 +00:00
//sfrags = append(sfrags, []byte(shortName+"_frags.push(`"+string(frags)+"`);\n")...)
2020-03-22 23:14:08 +00:00
//sfrags = append(sfrags, []byte("`"+string(frags)+"`,\n")...)
if i == 0 {
sfrags = append ( sfrags , [ ] byte ( "`" + string ( frags ) + "`" ) ... )
} else {
sfrags = append ( sfrags , [ ] byte ( ",`" + string ( frags ) + "`" ) ... )
}
2019-02-28 07:28:17 +00:00
}
2020-03-22 23:14:08 +00:00
//sfrags = append(sfrags, []byte("];\n")...)
sfrags = append ( sfrags , [ ] byte ( "];" ) ... )
2019-02-28 07:28:17 +00:00
data = append ( sfrags , data ... )
2018-05-14 08:56:56 +00:00
}
2020-03-22 02:37:32 +00:00
rep ( "\n;" , "\n" )
2020-03-22 23:14:08 +00:00
rep ( ";;" , ";" )
2018-05-14 08:56:56 +00:00
2020-03-22 23:14:08 +00:00
data = append ( data , '}' )
2019-02-28 07:28:17 +00:00
for name , _ := range Themes {
if strings . HasSuffix ( shortName , "_" + name ) {
2020-03-22 23:14:08 +00:00
data = append ( data , "var Tmpl_" + strings . TrimSuffix ( shortName , "_" + name ) + "=Tmpl_" + shortName + ";" ... )
2019-02-28 07:28:17 +00:00
break
}
2018-05-14 08:56:56 +00:00
}
path = tmplName + ".js"
DebugLog ( "js path: " , path )
2019-09-29 04:56:39 +00:00
ext := filepath . Ext ( "/tmpl_client/" + path )
2020-04-13 03:28:52 +00:00
brData , err := CompressBytesBrotli ( data )
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len ( brData ) >= ( len ( data ) + 110 ) {
brData = nil
} else {
diff := len ( data ) - len ( brData )
if diff <= len ( data ) / 100 {
brData = nil
}
}
2019-04-27 10:22:39 +00:00
gzipData , err := CompressBytesGzip ( data )
2018-08-21 08:00:35 +00:00
if err != nil {
return err
}
2020-03-12 03:00:04 +00:00
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
2020-03-22 00:25:10 +00:00
if len ( gzipData ) >= ( len ( data ) + 120 ) {
2020-03-12 03:00:04 +00:00
gzipData = nil
} else {
diff := len ( data ) - len ( gzipData )
if diff <= len ( data ) / 100 {
gzipData = nil
}
}
2018-05-14 08:56:56 +00:00
2019-03-11 08:47:45 +00:00
// Get a checksum for CSPs and cache busting
hasher := sha256 . New ( )
hasher . Write ( data )
2020-07-31 05:33:29 +00:00
sum := hasher . Sum ( nil )
checksum := hex . EncodeToString ( sum )
integrity := base64 . StdEncoding . EncodeToString ( sum )
2019-03-11 08:47:45 +00:00
2020-07-31 05:33:29 +00:00
l . Set ( l . Prefix + path , & SFile { data , gzipData , brData , checksum , integrity , l . Prefix + path + "?h=" + checksum , 0 , int64 ( len ( data ) ) , strconv . Itoa ( len ( data ) ) , int64 ( len ( gzipData ) ) , strconv . Itoa ( len ( gzipData ) ) , int64 ( len ( brData ) ) , strconv . Itoa ( len ( brData ) ) , mime . TypeByExtension ( ext ) , f , f . ModTime ( ) . UTC ( ) . Format ( http . TimeFormat ) } )
2018-05-14 08:56:56 +00:00
DebugLogf ( "Added the '%s' static file." , path )
return nil
} )
}
2020-07-26 04:36:40 +00:00
func ( l SFileList ) Init ( ) error {
2017-09-03 04:50:31 +00:00
return filepath . Walk ( "./public" , func ( path string , f os . FileInfo , err error ) error {
2017-06-25 09:56:39 +00:00
if f . IsDir ( ) {
return nil
}
2017-09-03 04:50:31 +00:00
path = strings . Replace ( path , "\\" , "/" , - 1 )
2017-06-25 09:56:39 +00:00
data , err := ioutil . ReadFile ( path )
if err != nil {
return err
}
2017-09-03 04:50:31 +00:00
path = strings . TrimPrefix ( path , "public/" )
2020-03-07 02:59:06 +00:00
ext := filepath . Ext ( "/public/" + path )
2020-04-03 05:53:15 +00:00
if ext == ".js" {
data = bytes . Replace ( data , [ ] byte ( "\r" ) , [ ] byte ( "" ) , - 1 )
}
2018-08-22 01:32:07 +00:00
mimetype := mime . TypeByExtension ( ext )
2019-03-11 08:47:45 +00:00
// Get a checksum for CSPs and cache busting
hasher := sha256 . New ( )
hasher . Write ( data )
2020-07-31 05:33:29 +00:00
sum := hasher . Sum ( nil )
checksum := hex . EncodeToString ( sum )
integrity := base64 . StdEncoding . EncodeToString ( sum )
2019-03-11 08:47:45 +00:00
2018-08-22 01:32:07 +00:00
// Avoid double-compressing images
2020-04-13 03:28:52 +00:00
var gzipData , brData [ ] byte
2018-08-22 01:32:07 +00:00
if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" {
2020-04-13 03:28:52 +00:00
brData , err = CompressBytesBrotli ( data )
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len ( brData ) >= ( len ( data ) + 130 ) {
brData = nil
} else {
diff := len ( data ) - len ( brData )
if diff <= len ( data ) / 100 {
brData = nil
}
}
2019-04-27 10:22:39 +00:00
gzipData , err = CompressBytesGzip ( data )
2018-08-22 01:32:07 +00:00
if err != nil {
return err
}
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
2020-03-12 03:00:04 +00:00
if len ( gzipData ) >= ( len ( data ) + 150 ) {
2018-08-22 01:32:07 +00:00
gzipData = nil
} else {
diff := len ( data ) - len ( gzipData )
if diff <= len ( data ) / 100 {
gzipData = nil
}
}
2018-08-21 08:00:35 +00:00
}
2017-06-25 09:56:39 +00:00
2020-07-31 05:33:29 +00:00
l . Set ( l . Prefix + path , & SFile { data , gzipData , brData , checksum , integrity , l . Prefix + path + "?h=" + checksum , 0 , int64 ( len ( data ) ) , strconv . Itoa ( len ( data ) ) , int64 ( len ( gzipData ) ) , strconv . Itoa ( len ( gzipData ) ) , int64 ( len ( brData ) ) , strconv . Itoa ( len ( brData ) ) , mimetype , f , f . ModTime ( ) . UTC ( ) . Format ( http . TimeFormat ) } )
2017-06-25 09:56:39 +00:00
2018-02-19 04:26:01 +00:00
DebugLogf ( "Added the '%s' static file." , path )
2017-06-25 09:56:39 +00:00
return nil
} )
}
2017-01-07 06:31:04 +00:00
2020-07-26 04:36:40 +00:00
func ( l SFileList ) Add ( path , prefix string ) error {
2017-01-07 06:31:04 +00:00
data , err := ioutil . ReadFile ( path )
if err != nil {
return err
}
fi , err := os . Open ( path )
if err != nil {
return err
}
f , err := fi . Stat ( )
if err != nil {
return err
}
2017-06-25 09:56:39 +00:00
2019-09-29 04:56:39 +00:00
ext := filepath . Ext ( path )
2017-01-07 06:31:04 +00:00
path = strings . TrimPrefix ( path , prefix )
2020-04-13 03:28:52 +00:00
brData , err := CompressBytesBrotli ( data )
if err != nil {
return err
}
// Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
if len ( brData ) >= ( len ( data ) + 130 ) {
brData = nil
} else {
diff := len ( data ) - len ( brData )
if diff <= len ( data ) / 100 {
brData = nil
}
}
2019-04-27 10:22:39 +00:00
gzipData , err := CompressBytesGzip ( data )
2018-08-21 08:00:35 +00:00
if err != nil {
return err
}
2020-03-12 03:00:04 +00:00
// Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
if len ( gzipData ) >= ( len ( data ) + 150 ) {
gzipData = nil
} else {
diff := len ( data ) - len ( gzipData )
if diff <= len ( data ) / 100 {
gzipData = nil
}
}
2017-06-25 09:56:39 +00:00
2019-03-11 08:47:45 +00:00
// Get a checksum for CSPs and cache busting
hasher := sha256 . New ( )
hasher . Write ( data )
2020-07-31 05:33:29 +00:00
sum := hasher . Sum ( nil )
checksum := hex . EncodeToString ( sum )
integrity := base64 . StdEncoding . EncodeToString ( sum )
2019-03-11 08:47:45 +00:00
2020-07-31 05:33:29 +00:00
l . Set ( l . Prefix + path , & SFile { data , gzipData , brData , checksum , integrity , l . Prefix + path + "?h=" + checksum , 0 , int64 ( len ( data ) ) , strconv . Itoa ( len ( data ) ) , int64 ( len ( gzipData ) ) , strconv . Itoa ( len ( gzipData ) ) , int64 ( len ( brData ) ) , strconv . Itoa ( len ( brData ) ) , mime . TypeByExtension ( ext ) , f , f . ModTime ( ) . UTC ( ) . Format ( http . TimeFormat ) } )
2017-06-25 09:56:39 +00:00
2018-02-19 04:26:01 +00:00
DebugLogf ( "Added the '%s' static file" , path )
2017-01-07 06:31:04 +00:00
return nil
2017-02-10 13:39:13 +00:00
}
2020-07-26 04:36:40 +00:00
func ( l SFileList ) Get ( path string ) ( file * SFile , exists bool ) {
staticFileMutex . RLock ( )
defer staticFileMutex . RUnlock ( )
file , exists = l . Long [ path ]
return file , exists
}
// fetch without /s/ to avoid allocing in pages.go
func ( l SFileList ) GetShort ( name string ) ( file * SFile , exists bool ) {
2017-12-01 02:04:29 +00:00
staticFileMutex . RLock ( )
defer staticFileMutex . RUnlock ( )
2020-07-26 04:36:40 +00:00
file , exists = l . Short [ name ]
2017-12-01 02:04:29 +00:00
return file , exists
}
2020-07-26 04:36:40 +00:00
func ( l SFileList ) Set ( name string , data * SFile ) {
2017-12-01 02:04:29 +00:00
staticFileMutex . Lock ( )
defer staticFileMutex . Unlock ( )
2020-07-30 08:10:29 +00:00
// TODO: Propagate errors back up
uurl , err := url . Parse ( name )
if err != nil {
return
}
l . Long [ uurl . Path ] = data
l . Short [ strings . TrimPrefix ( strings . TrimPrefix ( name , l . Prefix ) , "/" ) ] = data
2017-12-01 02:04:29 +00:00
}
2020-04-29 00:26:15 +00:00
var gzipBestCompress sync . Pool
func CompressBytesGzip ( in [ ] byte ) ( b [ ] byte , err error ) {
var buf bytes . Buffer
ii := gzipBestCompress . Get ( )
var gz * gzip . Writer
if ii == nil {
gz , err = gzip . NewWriterLevel ( & buf , gzip . BestCompression )
if err != nil {
return nil , err
}
} else {
gz = ii . ( * gzip . Writer )
gz . Reset ( & buf )
2018-08-21 08:00:35 +00:00
}
_ , err = gz . Write ( in )
if err != nil {
return nil , err
}
err = gz . Close ( )
if err != nil {
return nil , err
}
2020-04-29 00:26:15 +00:00
gzipBestCompress . Put ( gz )
return buf . Bytes ( ) , nil
2017-02-10 13:39:13 +00:00
}
2020-04-13 03:28:52 +00:00
func CompressBytesBrotli ( in [ ] byte ) ( [ ] byte , error ) {
var buff bytes . Buffer
br := brotli . NewWriterLevel ( & buff , brotli . BestCompression )
_ , err := br . Write ( in )
if err != nil {
return nil , err
}
err = br . Close ( )
if err != nil {
return nil , err
}
return buff . Bytes ( ) , nil
}