PNG and JPG avatars are now encoded as JPG images leading to a dramatic drop in the amount of bandwidth used.

Did some work on image thumbnailing, but our dependencies are acting up delaying this from being released.

Fixed the positions of the topic list bits for Nox on mobile.
Removed APNG as an accepted image format, as we don't currently have a good way of optimising these images.
Added a comment regarding the constant time compare for sessions.
Added a warning about putting Gosora in www folders.
Noavatars can now take a width parameters.
Added a bit of missing validation for the avatar uploader.
Refactored the multiple file detector for the avatar uploader.

Added a Run method to accDeleteBuilder.
Added an EachInt method to AccSelectBuilder.
Added a Run method to accInsertBuilder.

Added the users_avatar_queue table, you will need to run the patcher / update script.
You might also want to update the Noavatar field in your config.json file with the new one.
This commit is contained in:
Azareal 2018-07-28 22:52:23 +10:00
parent cee027cc7f
commit 0a628f7201
27 changed files with 356 additions and 80 deletions

View File

@ -65,7 +65,9 @@ On Windows, you might want to try the [GosoraBootstrapper](https://github.com/Az
*Linux* *Linux*
First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path. First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path (but *never* in a folder where the files are automatically served by a webserver).
If you place it in `/www/`, `/public_html/` or any similar folder, then there's a chance that your server might be compromised.
You can navigate to it by typing the following six commands into the console and hitting enter: You can navigate to it by typing the following six commands into the console and hitting enter:

View File

@ -250,6 +250,7 @@ func (auth *DefaultAuth) SessionCheck(w http.ResponseWriter, r *http.Request) (u
return &GuestUser, true return &GuestUser, true
} }
// We need to do a constant time compare, otherwise someone might be able to deduce the session character by character based on how long it takes to do the comparison. Change this at your own peril.
if user.Session == "" || subtle.ConstantTimeCompare([]byte(session), []byte(user.Session)) != 1 { if user.Session == "" || subtle.ConstantTimeCompare([]byte(session), []byte(user.Session)) != 1 {
return &GuestUser, false return &GuestUser, false
} }

View File

@ -43,7 +43,7 @@ type StringList []string
// ? - Should we allow users to upload .php or .go files? It could cause security issues. We could store them with a mangled extension to render them inert // ? - Should we allow users to upload .php or .go files? It could cause security issues. We could store them with a mangled extension to render them inert
// TODO: Let admins manage this from the Control Panel // TODO: Let admins manage this from the Control Panel
var AllowedFileExts = StringList{ var AllowedFileExts = StringList{
"png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", "apng", // images "png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", /*"apng",*/ // images
"txt", "xml", "json", "yaml", "toml", "ini", "md", "html", "rtf", "js", "py", "rb", "css", "scss", "less", "eqcss", "pcss", "java", "ts", "cs", "c", "cc", "cpp", "cxx", "C", "c++", "h", "hh", "hpp", "hxx", "h++", "rs", "rlib", "htaccess", "gitignore", // text "txt", "xml", "json", "yaml", "toml", "ini", "md", "html", "rtf", "js", "py", "rb", "css", "scss", "less", "eqcss", "pcss", "java", "ts", "cs", "c", "cc", "cpp", "cxx", "C", "c++", "h", "hh", "hpp", "hxx", "h++", "rs", "rlib", "htaccess", "gitignore", // text

View File

@ -29,6 +29,7 @@ type ReplyUser struct {
LastEdit int LastEdit int
LastEditBy int LastEditBy int
Avatar string Avatar string
MicroAvatar string
ClassName string ClassName string
ContentLines int ContentLines int
Tag string Tag string

View File

@ -173,7 +173,7 @@ func ProcessConfig() (err error) {
if Config.MaxUsernameLength == 0 { if Config.MaxUsernameLength == 0 {
Config.MaxUsernameLength = 100 Config.MaxUsernameLength = 100
} }
GuestUser.Avatar = BuildAvatar(0, "") GuestUser.Avatar, GuestUser.MicroAvatar = BuildAvatar(0, "")
if Config.HashAlgo != "" { if Config.HashAlgo != "" {
// TODO: Set the alternate hash algo, e.g. argon2 // TODO: Set the alternate hash algo, e.g. argon2

View File

@ -123,10 +123,15 @@ var Template_ip_search_handle = func(pi IPSearchPage, w io.Writer) error {
} }
func tmplInitUsers() (User, User, User) { func tmplInitUsers() (User, User, User) {
user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, BuildAvatar(62, ""), "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0} avatar, microAvatar := BuildAvatar(62, "")
user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, "", avatar, microAvatar, "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0}
// TODO: Do a more accurate level calculation for this? // TODO: Do a more accurate level calculation for this?
user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(1, ""), "", "", "", "", 58, 1000, 0, "127.0.0.1", 0} avatar, microAvatar = BuildAvatar(1, "")
user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(2, ""), "", "", "", "", 42, 900, 0, "::1", 0} user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 58, 1000, 0, "127.0.0.1", 0}
avatar, microAvatar = BuildAvatar(2, "")
user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 42, 900, 0, "::1", 0}
return user, user2, user3 return user, user2, user3
} }
@ -183,9 +188,12 @@ func CompileTemplates() error {
PollOption{0, "Nothing"}, PollOption{0, "Nothing"},
PollOption{1, "Something"}, PollOption{1, "Something"},
}, VoteCount: 7} }, VoteCount: 7}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false} avatar, microAvatar := BuildAvatar(62, "")
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
var replyList []ReplyUser var replyList []ReplyUser
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) // TODO: Do we want the UID on this to be 0?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
var varList = make(map[string]tmpl.VarItem) var varList = make(map[string]tmpl.VarItem)
header.Title = "Topic Name" header.Title = "Topic Name"
@ -367,9 +375,12 @@ func CompileJSTemplates() error {
PollOption{0, "Nothing"}, PollOption{0, "Nothing"},
PollOption{1, "Something"}, PollOption{1, "Something"},
}, VoteCount: 7} }, VoteCount: 7}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false} avatar, microAvatar := BuildAvatar(62, "")
topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
var replyList []ReplyUser var replyList []ReplyUser
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) // TODO: Do we really want the UID here to be zero?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
varList = make(map[string]tmpl.VarItem) varList = make(map[string]tmpl.VarItem)
header.Title = "Topic Name" header.Title = "Topic Name"

View File

@ -1,28 +1,86 @@
package common package common
import (
"image"
_ "image/gif"
"image/jpeg"
_ "image/png"
"os"
)
var Thumbnailer ThumbnailerInt var Thumbnailer ThumbnailerInt
type ThumbnailerInt interface { type ThumbnailerInt interface {
Resize(inPath string, tmpPath string, outPath string, width int) error
} }
type RezThumbnailer struct { type RezThumbnailer struct {
} }
func (thumb *RezThumbnailer) Resize(path string, width int) error { func (thumb *RezThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
// TODO: Sniff the aspect ratio of the image and calculate the dest height accordingly, bug make sure it isn't excessively high // TODO: Sniff the aspect ratio of the image and calculate the dest height accordingly, bug make sure it isn't excessively high
return nil return nil
} }
func (thumb *RezThumbnailer) resize(path string, width int, height int) error { func (thumb *RezThumbnailer) resize(inPath string, outPath string, width int, height int) error {
return nil return nil
} }
// ! Note: CaireThumbnailer can't handle gifs, so we'll have to either cap their sizes or have another resizer deal with them
type CaireThumbnailer struct {
}
func NewCaireThumbnailer() *CaireThumbnailer {
return &CaireThumbnailer{}
}
func precodeImage(inPath string, tmpPath string) error {
imageFile, err := os.Open(inPath)
if err != nil {
return err
}
defer imageFile.Close()
img, _, err := image.Decode(imageFile)
if err != nil {
return err
}
outFile, err := os.Create(tmpPath)
if err != nil {
return err
}
defer outFile.Close()
return jpeg.Encode(outFile, img, nil)
}
func (thumb *CaireThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
err := precodeImage(inPath, tmpPath)
if err != nil {
return err
}
return nil
// TODO: Caire doesn't work. Try something else. Or get them to fix the index out of range. We get enough wins from re-encoding as jpeg anyway
/*imageFile, err := os.Open(tmpPath)
if err != nil {
return err
}
defer imageFile.Close()
outFile, err := os.Create(outPath)
if err != nil {
return err
}
defer outFile.Close()
p := &caire.Processor{NewWidth: width, Scale: true}
return p.Process(imageFile, outFile)*/
}
/* /*
type LilliputThumbnailer struct { type LilliputThumbnailer struct {
}
type ResizeThumbnailer struct {
} }
*/ */

View File

@ -70,6 +70,7 @@ type TopicUser struct {
CreatedByName string CreatedByName string
Group int Group int
Avatar string Avatar string
MicroAvatar string
ContentLines int ContentLines int
ContentHTML string ContentHTML string
Tag string Tag string
@ -366,6 +367,7 @@ func GetTopicUser(tid int) (TopicUser, error) {
tu := TopicUser{ID: tid} tu := TopicUser{ID: tid}
err := topicStmts.getTopicUser.QueryRow(tid).Scan(&tu.Title, &tu.Content, &tu.CreatedBy, &tu.CreatedAt, &tu.IsClosed, &tu.Sticky, &tu.ParentID, &tu.IPAddress, &tu.PostCount, &tu.LikeCount, &tu.Poll, &tu.CreatedByName, &tu.Avatar, &tu.Group, &tu.URLPrefix, &tu.URLName, &tu.Level) err := topicStmts.getTopicUser.QueryRow(tid).Scan(&tu.Title, &tu.Content, &tu.CreatedBy, &tu.CreatedAt, &tu.IsClosed, &tu.Sticky, &tu.ParentID, &tu.IPAddress, &tu.PostCount, &tu.LikeCount, &tu.Poll, &tu.CreatedByName, &tu.Avatar, &tu.Group, &tu.URLPrefix, &tu.URLName, &tu.Level)
tu.Avatar, tu.MicroAvatar = BuildAvatar(tu.CreatedBy, tu.Avatar)
tu.Link = BuildTopicURL(NameToSlug(tu.Title), tu.ID) tu.Link = BuildTopicURL(NameToSlug(tu.Title), tu.ID)
tu.UserLink = BuildProfileURL(NameToSlug(tu.CreatedByName), tu.CreatedBy) tu.UserLink = BuildProfileURL(NameToSlug(tu.CreatedByName), tu.CreatedBy)
tu.Tag = Groups.DirtyGet(tu.Group).Tag tu.Tag = Groups.DirtyGet(tu.Group).Tag
@ -383,6 +385,7 @@ func copyTopicToTopicUser(topic *Topic, user *User) (tu TopicUser) {
tu.CreatedByName = user.Name tu.CreatedByName = user.Name
tu.Group = user.Group tu.Group = user.Group
tu.Avatar = user.Avatar tu.Avatar = user.Avatar
tu.MicroAvatar = user.MicroAvatar
tu.URLPrefix = user.URLPrefix tu.URLPrefix = user.URLPrefix
tu.URLName = user.URLName tu.URLName = user.URLName
tu.Level = user.Level tu.Level = user.Level

View File

@ -14,6 +14,7 @@ import (
"time" "time"
"../query_gen/lib" "../query_gen/lib"
"github.com/go-sql-driver/mysql"
) )
// TODO: Replace any literals with this // TODO: Replace any literals with this
@ -41,7 +42,9 @@ type User struct {
Session string Session string
//AuthToken string //AuthToken string
Loggedin bool Loggedin bool
RawAvatar string
Avatar string Avatar string
MicroAvatar string
Message string Message string
URLPrefix string // Move this to another table? Create a user lite? URLPrefix string // Move this to another table? Create a user lite?
URLName string URLName string
@ -96,6 +99,8 @@ type UserStmts struct {
updateLastIP *sql.Stmt updateLastIP *sql.Stmt
setPassword *sql.Stmt setPassword *sql.Stmt
scheduleAvatarResize *sql.Stmt
} }
var userStmts UserStmts var userStmts UserStmts
@ -123,13 +128,15 @@ func init() {
updateLastIP: acc.SimpleUpdate("users", "last_ip = ?", where), updateLastIP: acc.SimpleUpdate("users", "last_ip = ?", where),
setPassword: acc.Update("users").Set("password = ?, salt = ?").Where(where).Prepare(), setPassword: acc.Update("users").Set("password = ?, salt = ?").Where(where).Prepare(),
scheduleAvatarResize: acc.Insert("users_avatar_queue").Columns("uid").Fields("?").Prepare(),
} }
return acc.FirstError() return acc.FirstError()
}) })
} }
func (user *User) Init() { func (user *User) Init() {
user.Avatar = BuildAvatar(user.ID, user.Avatar) user.Avatar, user.MicroAvatar = BuildAvatar(user.ID, user.RawAvatar)
user.Link = BuildProfileURL(NameToSlug(user.Name), user.ID) user.Link = BuildProfileURL(NameToSlug(user.Name), user.ID)
user.Tag = Groups.DirtyGet(user.Group).Tag user.Tag = Groups.DirtyGet(user.Group).Tag
user.InitPerms() user.InitPerms()
@ -268,6 +275,23 @@ func (user *User) ChangeAvatar(avatar string) (err error) {
return user.bindStmt(userStmts.setAvatar, avatar) return user.bindStmt(userStmts.setAvatar, avatar)
} }
// TODO: Abstract this with an interface so we can scale this with an actual dedicated queue in a real cluster
func (user *User) ScheduleAvatarResize() (err error) {
_, err = userStmts.scheduleAvatarResize.Exec(user.ID)
if err != nil {
// TODO: Do a more generic check so that we're not as tied to MySQL
me, ok := err.(*mysql.MySQLError)
if !ok {
return err
}
// If it's just telling us that the item already exists in the database, then we can ignore it, as it doesn't matter if it's this call or another which schedules the item in the queue
if me.Number != 1062 {
return err
}
}
return nil
}
func (user *User) ChangeGroup(group int) (err error) { func (user *User) ChangeGroup(group int) (err error) {
return user.bindStmt(userStmts.changeGroup, group) return user.bindStmt(userStmts.changeGroup, group)
} }
@ -381,15 +405,25 @@ func (user *User) InitPerms() {
} }
} }
func buildNoavatar(uid int, width int) string {
return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1)
}
// ? Make this part of *User? // ? Make this part of *User?
func BuildAvatar(uid int, avatar string) string { // TODO: Write tests for this
func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar string) {
if avatar != "" { if avatar != "" {
if avatar[0] == '.' { if avatar[0] == '.' {
return "/uploads/avatar_" + strconv.Itoa(uid) + avatar if avatar[1] == '.' {
normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + "_tmp" + avatar[1:]
return normalAvatar, normalAvatar
} }
return avatar normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + avatar
return normalAvatar, normalAvatar
} }
return strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1) return avatar, avatar
}
return buildNoavatar(uid, 200), buildNoavatar(uid, 48)
} }
// TODO: Move this to *User // TODO: Move this to *User

View File

@ -68,7 +68,7 @@ func (mus *DefaultUserStore) DirtyGet(id int) *User {
} }
user = &User{ID: id, Loggedin: true} user = &User{ID: id, Loggedin: true}
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init() user.Init()
if err == nil { if err == nil {
@ -86,7 +86,7 @@ func (mus *DefaultUserStore) Get(id int) (*User, error) {
} }
user = &User{ID: id, Loggedin: true} user = &User{ID: id, Loggedin: true}
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init() user.Init()
if err == nil { if err == nil {
@ -106,7 +106,7 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User
for rows.Next() { for rows.Next() {
user := &User{Loggedin: true} user := &User{Loggedin: true}
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -162,7 +162,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
for rows.Next() { for rows.Next() {
user := &User{Loggedin: true} user := &User{Loggedin: true}
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil { if err != nil {
return list, err return list, err
} }
@ -203,7 +203,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
func (mus *DefaultUserStore) BypassGet(id int) (*User, error) { func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
user := &User{ID: id, Loggedin: true} user := &User{ID: id, Loggedin: true}
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init() user.Init()
return user, err return user, err
@ -211,7 +211,7 @@ func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
func (mus *DefaultUserStore) Reload(id int) error { func (mus *DefaultUserStore) Reload(id int) error {
user := &User{ID: id, Loggedin: true} user := &User{ID: id, Loggedin: true}
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil { if err != nil {
mus.cache.Remove(id) mus.cache.Remove(id)
return err return err

View File

@ -30,7 +30,7 @@
"MinifyTemplates":true, "MinifyTemplates":true,
"BuildSlugs":true, "BuildSlugs":true,
"ServerCount":1, "ServerCount":1,
"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png", "Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
"ItemsPerPage":25 "ItemsPerPage":25
}, },
"Database": { "Database": {

View File

@ -329,12 +329,12 @@ func RouteMemberList(w http.ResponseWriter, r *http.Request, user common.User) c
var guildMembers []Member var guildMembers []Member
for rows.Next() { for rows.Next() {
guildMember := Member{PostCount: 0} guildMember := Member{PostCount: 0}
err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.Avatar) err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.RawAvatar)
if err != nil { if err != nil {
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
guildMember.Link = common.BuildProfileURL(common.NameToSlug(guildMember.User.Name), guildMember.User.ID) guildMember.Link = common.BuildProfileURL(common.NameToSlug(guildMember.User.Name), guildMember.User.ID)
guildMember.User.Avatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.Avatar) guildMember.User.Avatar, guildMember.User.MicroAvatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.RawAvatar)
guildMember.JoinedAt, _ = common.RelativeTimeFromString(guildMember.JoinedAt) guildMember.JoinedAt, _ = common.RelativeTimeFromString(guildMember.JoinedAt)
if guildItem.Owner == guildMember.User.ID { if guildItem.Owner == guildMember.User.ID {
guildMember.RankString = "Owner" guildMember.RankString = "Owner"

View File

@ -2,21 +2,22 @@
package main package main
var dbTablePrimaryKeys = map[string]string{ var dbTablePrimaryKeys = map[string]string{
"users_groups":"gid",
"attachments":"attachID",
"users_replies":"rid",
"menu_items":"miid",
"pages":"pid",
"polls":"pollID",
"activity_stream":"asid",
"users_groups_scheduler":"uid", "users_groups_scheduler":"uid",
"replies":"rid", "replies":"rid",
"attachments":"attachID",
"revisions":"reviseID",
"polls":"pollID",
"users_replies":"rid",
"activity_stream":"asid",
"users":"uid",
"pages":"pid",
"word_filters":"wfid", "word_filters":"wfid",
"menus":"mid", "menus":"mid",
"registration_logs":"rlid", "registration_logs":"rlid",
"users":"uid",
"users_2fa_keys":"uid",
"forums":"fid", "forums":"fid",
"users_2fa_keys":"uid",
"users_avatar_queue":"uid",
"topics":"tid", "topics":"tid",
"revisions":"reviseID", "users_groups":"gid",
"menu_items":"miid",
} }

View File

@ -128,7 +128,7 @@ func main() {
"MinifyTemplates":true, "MinifyTemplates":true,
"BuildSlugs":true, "BuildSlugs":true,
"ServerCount":1, "ServerCount":1,
"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png", "Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
"ItemsPerPage":25 "ItemsPerPage":25
}, },
"Database": { "Database": {
@ -152,7 +152,6 @@ func main() {
} }
}`) }`)
//"Noavatar": "https://api.adorable.io/avatars/{width}/{id}@{site_url}.png" Maybe allow this sort of syntax?
fmt.Println("Opening the configuration file") fmt.Println("Opening the configuration file")
configFile, err := os.Create("./config/config.json") configFile, err := os.Create("./config/config.json")
if err != nil { if err != nil {

57
main.go
View File

@ -16,6 +16,7 @@ import (
"net/http" "net/http"
"os" "os"
"os/signal" "os/signal"
"strconv"
"strings" "strings"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
@ -158,6 +159,8 @@ func afterDBInit() (err error) {
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
// TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins
common.Thumbnailer = common.NewCaireThumbnailer()
log.Print("Initialising the view counters") log.Print("Initialising the view counters")
counters.GlobalViewCounter, err = counters.NewGlobalViewCounter(acc) counters.GlobalViewCounter, err = counters.NewGlobalViewCounter(acc)
@ -309,6 +312,7 @@ func main() {
return nil return nil
} }
// TODO: Expand this to more types of files
var err error var err error
for { for {
select { select {
@ -357,6 +361,58 @@ func main() {
} }
} }
// Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests
// TODO: Could we expand this to attachments and other things too?
thumbChan := make(chan bool)
go func() {
acc := qgen.Builder.Accumulator()
for {
// Put this goroutine to sleep until we have work to do
<-thumbChan
// TODO: Use a real queue
err := acc.Select("users_avatar_queue").Columns("uid").Limit("0,5").EachInt(func(uid int) error {
//log.Print("uid: ", uid)
// TODO: Do a bulk user fetch instead?
user, err := common.Users.Get(uid)
if err != nil {
return errors.WithStack(err)
}
//log.Print("user.RawAvatar: ", user.RawAvatar)
// Has the avatar been removed or already been processed by the thumbnailer?
if len(user.RawAvatar) < 2 || user.RawAvatar[1] == '.' {
_, _ = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
return nil
}
// This means it's an external image, they aren't currently implemented, but this is here for when they are
if user.RawAvatar[0] != '.' {
return nil
}
/*if user.RawAvatar == ".gif" {
return nil
}*/
if user.RawAvatar != ".png" && user.RawAvatar != "jpg" && user.RawAvatar != "jpeg" && user.RawAvatar != "gif" {
return nil
}
err = common.Thumbnailer.Resize("./uploads/avatar_"+strconv.Itoa(user.ID)+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_tmp"+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_w48"+user.RawAvatar, 48)
if err != nil {
return errors.WithStack(err)
}
err = user.ChangeAvatar("." + user.RawAvatar)
if err != nil {
return errors.WithStack(err)
}
_, err = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
return errors.WithStack(err)
})
if err != nil {
common.LogError(err)
}
}
}()
// TODO: Write tests for these // TODO: Write tests for these
// Run this goroutine once every half second // Run this goroutine once every half second
halfSecondTicker := time.NewTicker(time.Second / 2) halfSecondTicker := time.NewTicker(time.Second / 2)
@ -402,6 +458,7 @@ func main() {
continue continue
} }
runHook("before_second_tick") runHook("before_second_tick")
go func() { thumbChan <- true }()
runTasks(common.ScheduledSecondTasks) runTasks(common.ScheduledSecondTasks)
// TODO: Stop hard-coding this // TODO: Stop hard-coding this

View File

@ -15,6 +15,7 @@ func init() {
addPatch(4, patch4) addPatch(4, patch4)
addPatch(5, patch5) addPatch(5, patch5)
addPatch(6, patch6) addPatch(6, patch6)
addPatch(7, patch7)
} }
func patch0(scanner *bufio.Scanner) (err error) { func patch0(scanner *bufio.Scanner) (err error) {
@ -519,3 +520,19 @@ func patch6(scanner *bufio.Scanner) error {
return nil return nil
} }
func patch7(scanner *bufio.Scanner) error {
err := execStmt(qgen.Builder.CreateTable("users_avatar_queue", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
},
[]qgen.DBTableKey{
qgen.DBTableKey{"uid", "primary"},
},
))
if err != nil {
return err
}
return nil
}

View File

@ -12,16 +12,31 @@ type accDeleteBuilder struct {
build *Accumulator build *Accumulator
} }
func (delete *accDeleteBuilder) Where(where string) *accDeleteBuilder { func (builder *accDeleteBuilder) Where(where string) *accDeleteBuilder {
if delete.where != "" { if builder.where != "" {
delete.where += " AND " builder.where += " AND "
} }
delete.where += where builder.where += where
return delete return builder
} }
func (delete *accDeleteBuilder) Prepare() *sql.Stmt { func (builder *accDeleteBuilder) Prepare() *sql.Stmt {
return delete.build.SimpleDelete(delete.table, delete.where) return builder.build.SimpleDelete(builder.table, builder.where)
}
func (builder *accDeleteBuilder) Run(args ...interface{}) (int, error) {
stmt := builder.Prepare()
if stmt == nil {
return 0, builder.build.FirstError()
}
res, err := stmt.Exec(args...)
if err != nil {
return 0, err
}
lastID, err := res.LastInsertId()
return int(lastID), err
} }
type accUpdateBuilder struct { type accUpdateBuilder struct {
@ -169,6 +184,26 @@ func (selectItem *AccSelectBuilder) Each(handle func(*sql.Rows) error) error {
} }
return rows.Err() return rows.Err()
} }
func (selectItem *AccSelectBuilder) EachInt(handle func(int) error) error {
rows, err := selectItem.Query()
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var theInt int
err = rows.Scan(&theInt)
if err != nil {
return err
}
err = handle(theInt)
if err != nil {
return err
}
}
return rows.Err()
}
type accInsertBuilder struct { type accInsertBuilder struct {
table string table string
@ -200,6 +235,21 @@ func (insert *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err e
return res, insert.build.FirstError() return res, insert.build.FirstError()
} }
func (builder *accInsertBuilder) Run(args ...interface{}) (int, error) {
stmt := builder.Prepare()
if stmt == nil {
return 0, builder.build.FirstError()
}
res, err := stmt.Exec(args...)
if err != nil {
return 0, err
}
lastID, err := res.LastInsertId()
return int(lastID), err
}
type accCountBuilder struct { type accCountBuilder struct {
table string table string
where string where string

View File

@ -45,6 +45,7 @@ func (build *Accumulator) recordError(err error) {
} }
func (build *Accumulator) prepare(res string, err error) *sql.Stmt { func (build *Accumulator) prepare(res string, err error) *sql.Stmt {
// TODO: Can we make this less noisy on debug mode?
if LogPrepares { if LogPrepares {
log.Print("res: ", res) log.Print("res: ", res)
} }

View File

@ -127,6 +127,17 @@ func createTables(adapter qgen.Adapter) error {
}, },
) )
// TODO: Can we use a piece of software dedicated to persistent queues for this rather than relying on the database for it?
qgen.Install.CreateTable("users_avatar_queue", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
},
[]qgen.DBTableKey{
qgen.DBTableKey{"uid", "primary"},
},
)
// TODO: Should we add a users prefix to this table to fit the "unofficial convention"?
qgen.Install.CreateTable("emails", "", "", qgen.Install.CreateTable("emails", "", "",
[]qgen.DBTableColumn{ []qgen.DBTableColumn{
qgen.DBTableColumn{"email", "varchar", 200, false, false, ""}, qgen.DBTableColumn{"email", "varchar", 200, false, false, ""},

View File

@ -447,7 +447,22 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
return ferr return ferr
} }
var filename, ext string // We don't want multiple files
// TODO: Are we doing this correctly?
filenameMap := make(map[string]bool)
for _, fheaders := range r.MultipartForm.File {
for _, hdr := range fheaders {
if hdr.Filename == "" {
continue
}
filenameMap[hdr.Filename] = true
}
}
if len(filenameMap) > 1 {
return common.LocalError("You may only upload one avatar", w, r, user)
}
var ext string
for _, fheaders := range r.MultipartForm.File { for _, fheaders := range r.MultipartForm.File {
for _, hdr := range fheaders { for _, hdr := range fheaders {
if hdr.Filename == "" { if hdr.Filename == "" {
@ -459,17 +474,6 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
} }
defer infile.Close() defer infile.Close()
// We don't want multiple files
// TODO: Check the length of r.MultipartForm.File and error rather than doing this x.x
if filename != "" {
if filename != hdr.Filename {
os.Remove("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
return common.LocalError("You may only upload one avatar", w, r, user)
}
} else {
filename = hdr.Filename
}
if ext == "" { if ext == "" {
extarr := strings.Split(hdr.Filename, ".") extarr := strings.Split(hdr.Filename, ".")
if len(extarr) < 2 { if len(extarr) < 2 {
@ -484,8 +488,13 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
} }
ext = reg.ReplaceAllString(ext, "") ext = reg.ReplaceAllString(ext, "")
ext = strings.ToLower(ext) ext = strings.ToLower(ext)
if !common.ImageFileExts.Contains(ext) {
return common.LocalError("You can only use an image for your avatar", w, r, user)
}
} }
// TODO: Centralise this string, so we don't have to change it in two different places when it changes
outfile, err := os.Create("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext) outfile, err := os.Create("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
if err != nil { if err != nil {
return common.LocalError("Upload failed [File Creation Failed]", w, r, user) return common.LocalError("Upload failed [File Creation Failed]", w, r, user)
@ -506,6 +515,11 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
if err != nil { if err != nil {
return common.InternalError(err, w, r) return common.InternalError(err, w, r)
} }
// TODO: Only schedule a resize if the avatar isn't tiny
err = user.ScheduleAvatarResize()
if err != nil {
return common.InternalError(err, w, r)
}
http.Redirect(w, r, "/user/edit/?avatar_updated=1", http.StatusSeeOther) http.Redirect(w, r, "/user/edit/?avatar_updated=1", http.StatusSeeOther)
return nil return nil
} }

View File

@ -37,7 +37,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
var err error var err error
var replyCreatedAt time.Time var replyCreatedAt time.Time
var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyTag, replyClassName string var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyMicroAvatar, replyTag, replyClassName string
var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int
var replyList []common.ReplyUser var replyList []common.ReplyUser
@ -93,7 +93,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
} else { } else {
replyClassName = "" replyClassName = ""
} }
replyAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar) replyAvatar, replyMicroAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar)
if group.Tag != "" { if group.Tag != "" {
replyTag = group.Tag replyTag = group.Tag
@ -109,7 +109,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
// TODO: Add a hook here // TODO: Add a hook here
replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""}) replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
} }
err = rows.Err() err = rows.Err()
if err != nil { if err != nil {

View File

@ -89,9 +89,6 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
} }
topic.RelativeCreatedAt = common.RelativeTime(topic.CreatedAt) topic.RelativeCreatedAt = common.RelativeTime(topic.CreatedAt)
// TODO: Make a function for this? Build a more sophisticated noavatar handling system?
topic.Avatar = common.BuildAvatar(topic.CreatedBy, topic.Avatar)
var poll common.Poll var poll common.Poll
if topic.Poll != 0 { if topic.Poll != 0 {
pPoll, err := common.Polls.Get(topic.Poll) pPoll, err := common.Polls.Get(topic.Poll)
@ -129,6 +126,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
} }
defer rows.Close() defer rows.Close()
// TODO: Factor the user fields out and embed a user struct instead
replyItem := common.ReplyUser{ClassName: ""} replyItem := common.ReplyUser{ClassName: ""}
for rows.Next() { for rows.Next() {
err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType) err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType)
@ -153,7 +151,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
} }
// TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the common.UserStore initialise this? // TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the common.UserStore initialise this?
replyItem.Avatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar) replyItem.Avatar, replyItem.MicroAvatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
replyItem.Tag = postGroup.Tag replyItem.Tag = postGroup.Tag
replyItem.RelativeCreatedAt = common.RelativeTime(replyItem.CreatedAt) replyItem.RelativeCreatedAt = common.RelativeTime(replyItem.CreatedAt)

View File

@ -0,0 +1,4 @@
CREATE TABLE [users_avatar_queue] (
[uid] int not null,
primary key([uid])
);

View File

@ -0,0 +1,4 @@
CREATE TABLE `users_avatar_queue` (
`uid` int not null,
primary key(`uid`)
);

View File

@ -0,0 +1,4 @@
CREATE TABLE `users_avatar_queue` (
`uid` int not null,
primary key(`uid`)
);

View File

@ -1,5 +1,5 @@
{ {
"DBVersion":"7", "DBVersion":"8",
"DynamicFileVersion":"0", "DynamicFileVersion":"0",
"MinGoVersion":"1.10", "MinGoVersion":"1.10",
"MinVersion":"" "MinVersion":""

View File

@ -696,6 +696,12 @@ button, .formbutton {
.topic_list .topic_middle { .topic_list .topic_middle {
display: none; display: none;
} }
.topic_left, .topic_right, .topic_middle {
width: 50%;
}
.topic_right_inside .lastName, .topic_left .rowtopic {
margin-top: -4px;
}
.topic_left img, .topic_right img { .topic_left img, .topic_right img {
height: 32px; height: 32px;
width: 32px; width: 32px;