PNG and JPG avatars are now encoded as JPG images leading to a dramatic drop in the amount of bandwidth used.

Did some work on image thumbnailing, but our dependencies are acting up delaying this from being released.

Fixed the positions of the topic list bits for Nox on mobile.
Removed APNG as an accepted image format, as we don't currently have a good way of optimising these images.
Added a comment regarding the constant time compare for sessions.
Added a warning about putting Gosora in www folders.
Noavatars can now take a width parameters.
Added a bit of missing validation for the avatar uploader.
Refactored the multiple file detector for the avatar uploader.

Added a Run method to accDeleteBuilder.
Added an EachInt method to AccSelectBuilder.
Added a Run method to accInsertBuilder.

Added the users_avatar_queue table, you will need to run the patcher / update script.
You might also want to update the Noavatar field in your config.json file with the new one.

README.md

@@ -65,7 +65,9 @@ On Windows, you might want to try the [GosoraBootstrapper](https://github.com/Az
 
 *Linux*
 
-First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path.
+First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path (but *never* in a folder where the files are automatically served by a webserver).
+
+If you place it in `/www/`, `/public_html/` or any similar folder, then there's a chance that your server might be compromised.
 
 You can navigate to it by typing the following six commands into the console and hitting enter:
 

common/auth.go

@@ -250,6 +250,7 @@ func (auth *DefaultAuth) SessionCheck(w http.ResponseWriter, r *http.Request) (u
 		return &GuestUser, true
 	}
 
+	// We need to do a constant time compare, otherwise someone might be able to deduce the session character by character based on how long it takes to do the comparison. Change this at your own peril.
 	if user.Session == "" || subtle.ConstantTimeCompare([]byte(session), []byte(user.Session)) != 1 {
 		return &GuestUser, false
 	}

common/common.go

@@ -43,7 +43,7 @@ type StringList []string
 // ? - Should we allow users to upload .php or .go files? It could cause security issues. We could store them with a mangled extension to render them inert
 // TODO: Let admins manage this from the Control Panel
 var AllowedFileExts = StringList{
-	"png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", "apng", // images
+	"png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", /*"apng",*/ // images
 
 	"txt", "xml", "json", "yaml", "toml", "ini", "md", "html", "rtf", "js", "py", "rb", "css", "scss", "less", "eqcss", "pcss", "java", "ts", "cs", "c", "cc", "cpp", "cxx", "C", "c++", "h", "hh", "hpp", "hxx", "h++", "rs", "rlib", "htaccess", "gitignore", // text
 

common/reply.go

@@ -29,6 +29,7 @@ type ReplyUser struct {
 	LastEdit          int
 	LastEditBy        int
 	Avatar            string
+	MicroAvatar       string
 	ClassName         string
 	ContentLines      int
 	Tag               string

common/site.go

@@ -173,7 +173,7 @@ func ProcessConfig() (err error) {
 	if Config.MaxUsernameLength == 0 {
 		Config.MaxUsernameLength = 100
 	}
-	GuestUser.Avatar = BuildAvatar(0, "")
+	GuestUser.Avatar, GuestUser.MicroAvatar = BuildAvatar(0, "")
 
 	if Config.HashAlgo != "" {
 		// TODO: Set the alternate hash algo, e.g. argon2

common/template_init.go

@@ -123,10 +123,15 @@ var Template_ip_search_handle = func(pi IPSearchPage, w io.Writer) error {
 }
 
 func tmplInitUsers() (User, User, User) {
-	user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, BuildAvatar(62, ""), "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0}
+	avatar, microAvatar := BuildAvatar(62, "")
+	user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, "", avatar, microAvatar, "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0}
+
 	// TODO: Do a more accurate level calculation for this?
-	user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(1, ""), "", "", "", "", 58, 1000, 0, "127.0.0.1", 0}
+	avatar, microAvatar = BuildAvatar(1, "")
-	user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(2, ""), "", "", "", "", 42, 900, 0, "::1", 0}
+	user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 58, 1000, 0, "127.0.0.1", 0}
+
+	avatar, microAvatar = BuildAvatar(2, "")
+	user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 42, 900, 0, "::1", 0}
 	return user, user2, user3
 }
 
@@ -183,9 +188,12 @@ func CompileTemplates() error {
 		PollOption{0, "Nothing"},
 		PollOption{1, "Something"},
 	}, VoteCount: 7}
-	topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false}
+	avatar, microAvatar := BuildAvatar(62, "")
+	topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
 	var replyList []ReplyUser
-	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
+	// TODO: Do we want the UID on this to be 0?
+	avatar, microAvatar = BuildAvatar(0, "")
+	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
 
 	var varList = make(map[string]tmpl.VarItem)
 	header.Title = "Topic Name"
@@ -367,9 +375,12 @@ func CompileJSTemplates() error {
 		PollOption{0, "Nothing"},
 		PollOption{1, "Something"},
 	}, VoteCount: 7}
-	topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false}
+	avatar, microAvatar := BuildAvatar(62, "")
+	topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
 	var replyList []ReplyUser
-	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
+	// TODO: Do we really want the UID here to be zero?
+	avatar, microAvatar = BuildAvatar(0, "")
+	replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
 
 	varList = make(map[string]tmpl.VarItem)
 	header.Title = "Topic Name"

common/thumbnailer.go

@@ -1,28 +1,86 @@
 package common
 
+import (
+	"image"
+	_ "image/gif"
+	"image/jpeg"
+	_ "image/png"
+	"os"
+)
+
 var Thumbnailer ThumbnailerInt
 
 type ThumbnailerInt interface {
+	Resize(inPath string, tmpPath string, outPath string, width int) error
 }
 
 type RezThumbnailer struct {
 }
 
-func (thumb *RezThumbnailer) Resize(path string, width int) error {
+func (thumb *RezThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
 	// TODO: Sniff the aspect ratio of the image and calculate the dest height accordingly, bug make sure it isn't excessively high
 	return nil
 }
 
-func (thumb *RezThumbnailer) resize(path string, width int, height int) error {
+func (thumb *RezThumbnailer) resize(inPath string, outPath string, width int, height int) error {
 	return nil
 }
 
+// ! Note: CaireThumbnailer can't handle gifs, so we'll have to either cap their sizes or have another resizer deal with them
+type CaireThumbnailer struct {
+}
+
+func NewCaireThumbnailer() *CaireThumbnailer {
+	return &CaireThumbnailer{}
+}
+
+func precodeImage(inPath string, tmpPath string) error {
+	imageFile, err := os.Open(inPath)
+	if err != nil {
+		return err
+	}
+	defer imageFile.Close()
+
+	img, _, err := image.Decode(imageFile)
+	if err != nil {
+		return err
+	}
+
+	outFile, err := os.Create(tmpPath)
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+
+	return jpeg.Encode(outFile, img, nil)
+}
+
+func (thumb *CaireThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
+	err := precodeImage(inPath, tmpPath)
+	if err != nil {
+		return err
+	}
+	return nil
+
+	// TODO: Caire doesn't work. Try something else. Or get them to fix the index out of range. We get enough wins from re-encoding as jpeg anyway
+	/*imageFile, err := os.Open(tmpPath)
+	if err != nil {
+		return err
+	}
+	defer imageFile.Close()
+
+	outFile, err := os.Create(outPath)
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+
+	p := &caire.Processor{NewWidth: width, Scale: true}
+	return p.Process(imageFile, outFile)*/
+}
+
 /*
 type LilliputThumbnailer struct {
 
-}
-
-type ResizeThumbnailer struct {
-
 }
 */

common/topic.go

@@ -70,6 +70,7 @@ type TopicUser struct {
 	CreatedByName string
 	Group         int
 	Avatar        string
+	MicroAvatar   string
 	ContentLines  int
 	ContentHTML   string
 	Tag           string
@@ -366,6 +367,7 @@ func GetTopicUser(tid int) (TopicUser, error) {
 
 	tu := TopicUser{ID: tid}
 	err := topicStmts.getTopicUser.QueryRow(tid).Scan(&tu.Title, &tu.Content, &tu.CreatedBy, &tu.CreatedAt, &tu.IsClosed, &tu.Sticky, &tu.ParentID, &tu.IPAddress, &tu.PostCount, &tu.LikeCount, &tu.Poll, &tu.CreatedByName, &tu.Avatar, &tu.Group, &tu.URLPrefix, &tu.URLName, &tu.Level)
+	tu.Avatar, tu.MicroAvatar = BuildAvatar(tu.CreatedBy, tu.Avatar)
 	tu.Link = BuildTopicURL(NameToSlug(tu.Title), tu.ID)
 	tu.UserLink = BuildProfileURL(NameToSlug(tu.CreatedByName), tu.CreatedBy)
 	tu.Tag = Groups.DirtyGet(tu.Group).Tag
@@ -383,6 +385,7 @@ func copyTopicToTopicUser(topic *Topic, user *User) (tu TopicUser) {
 	tu.CreatedByName = user.Name
 	tu.Group = user.Group
 	tu.Avatar = user.Avatar
+	tu.MicroAvatar = user.MicroAvatar
 	tu.URLPrefix = user.URLPrefix
 	tu.URLName = user.URLName
 	tu.Level = user.Level

common/user.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"../query_gen/lib"
+	"github.com/go-sql-driver/mysql"
 )
 
 // TODO: Replace any literals with this
@@ -41,7 +42,9 @@ type User struct {
 	Session      string
 	//AuthToken    string
 	Loggedin    bool
+	RawAvatar   string
 	Avatar      string
+	MicroAvatar string
 	Message     string
 	URLPrefix   string // Move this to another table? Create a user lite?
 	URLName     string
@@ -96,6 +99,8 @@ type UserStmts struct {
 	updateLastIP   *sql.Stmt
 
 	setPassword *sql.Stmt
+
+	scheduleAvatarResize *sql.Stmt
 }
 
 var userStmts UserStmts
@@ -123,13 +128,15 @@ func init() {
 			updateLastIP: acc.SimpleUpdate("users", "last_ip = ?", where),
 
 			setPassword: acc.Update("users").Set("password = ?, salt = ?").Where(where).Prepare(),
+
+			scheduleAvatarResize: acc.Insert("users_avatar_queue").Columns("uid").Fields("?").Prepare(),
 		}
 		return acc.FirstError()
 	})
 }
 
 func (user *User) Init() {
-	user.Avatar = BuildAvatar(user.ID, user.Avatar)
+	user.Avatar, user.MicroAvatar = BuildAvatar(user.ID, user.RawAvatar)
 	user.Link = BuildProfileURL(NameToSlug(user.Name), user.ID)
 	user.Tag = Groups.DirtyGet(user.Group).Tag
 	user.InitPerms()
@@ -268,6 +275,23 @@ func (user *User) ChangeAvatar(avatar string) (err error) {
 	return user.bindStmt(userStmts.setAvatar, avatar)
 }
 
+// TODO: Abstract this with an interface so we can scale this with an actual dedicated queue in a real cluster
+func (user *User) ScheduleAvatarResize() (err error) {
+	_, err = userStmts.scheduleAvatarResize.Exec(user.ID)
+	if err != nil {
+		// TODO: Do a more generic check so that we're not as tied to MySQL
+		me, ok := err.(*mysql.MySQLError)
+		if !ok {
+			return err
+		}
+		// If it's just telling us that the item already exists in the database, then we can ignore it, as it doesn't matter if it's this call or another which schedules the item in the queue
+		if me.Number != 1062 {
+			return err
+		}
+	}
+	return nil
+}
+
 func (user *User) ChangeGroup(group int) (err error) {
 	return user.bindStmt(userStmts.changeGroup, group)
 }
@@ -381,15 +405,25 @@ func (user *User) InitPerms() {
 	}
 }
 
+func buildNoavatar(uid int, width int) string {
+	return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1)
+}
+
 // ? Make this part of *User?
-func BuildAvatar(uid int, avatar string) string {
+// TODO: Write tests for this
+func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar string) {
 	if avatar != "" {
 		if avatar[0] == '.' {
-			return "/uploads/avatar_" + strconv.Itoa(uid) + avatar
+			if avatar[1] == '.' {
+				normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + "_tmp" + avatar[1:]
+				return normalAvatar, normalAvatar
 			}
-		return avatar
+			normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + avatar
+			return normalAvatar, normalAvatar
 		}
-	return strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1)
+		return avatar, avatar
+	}
+	return buildNoavatar(uid, 200), buildNoavatar(uid, 48)
 }
 
 // TODO: Move this to *User

common/user_store.go

@@ -68,7 +68,7 @@ func (mus *DefaultUserStore) DirtyGet(id int) *User {
 	}
 
 	user = &User{ID: id, Loggedin: true}
-	err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+	err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 
 	user.Init()
 	if err == nil {
@@ -86,7 +86,7 @@ func (mus *DefaultUserStore) Get(id int) (*User, error) {
 	}
 
 	user = &User{ID: id, Loggedin: true}
-	err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+	err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 
 	user.Init()
 	if err == nil {
@@ -106,7 +106,7 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User
 
 	for rows.Next() {
 		user := &User{Loggedin: true}
-		err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+		err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 		if err != nil {
 			return nil, err
 		}
@@ -162,7 +162,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
 
 	for rows.Next() {
 		user := &User{Loggedin: true}
-		err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+		err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 		if err != nil {
 			return list, err
 		}
@@ -203,7 +203,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
 
 func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
 	user := &User{ID: id, Loggedin: true}
-	err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+	err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 
 	user.Init()
 	return user, err
@@ -211,7 +211,7 @@ func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
 
 func (mus *DefaultUserStore) Reload(id int) error {
 	user := &User{ID: id, Loggedin: true}
-	err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
+	err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
 	if err != nil {
 		mus.cache.Remove(id)
 		return err

config/config_example.json

@@ -30,7 +30,7 @@
 		"MinifyTemplates":true,
 		"BuildSlugs":true,
 		"ServerCount":1,
-		"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png",
+		"Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
 		"ItemsPerPage":25
 	},
 	"Database": {

extend/guilds/lib/guilds.go

@@ -329,12 +329,12 @@ func RouteMemberList(w http.ResponseWriter, r *http.Request, user common.User) c
 	var guildMembers []Member
 	for rows.Next() {
 		guildMember := Member{PostCount: 0}
-		err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.Avatar)
+		err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.RawAvatar)
 		if err != nil {
 			return common.InternalError(err, w, r)
 		}
 		guildMember.Link = common.BuildProfileURL(common.NameToSlug(guildMember.User.Name), guildMember.User.ID)
-		guildMember.User.Avatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.Avatar)
+		guildMember.User.Avatar, guildMember.User.MicroAvatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.RawAvatar)
 		guildMember.JoinedAt, _ = common.RelativeTimeFromString(guildMember.JoinedAt)
 		if guildItem.Owner == guildMember.User.ID {
 			guildMember.RankString = "Owner"

gen_tables.go

@@ -2,21 +2,22 @@
 package main
 
 var dbTablePrimaryKeys = map[string]string{
-	"users_groups":"gid",
-	"attachments":"attachID",
-	"users_replies":"rid",
-	"menu_items":"miid",
-	"pages":"pid",
-	"polls":"pollID",
-	"activity_stream":"asid",
 	"users_groups_scheduler":"uid",
 	"replies":"rid",
+	"attachments":"attachID",
+	"revisions":"reviseID",
+	"polls":"pollID",
+	"users_replies":"rid",
+	"activity_stream":"asid",
+	"users":"uid",
+	"pages":"pid",
 	"word_filters":"wfid",
 	"menus":"mid",
 	"registration_logs":"rlid",
-	"users":"uid",
-	"users_2fa_keys":"uid",
 	"forums":"fid",
+	"users_2fa_keys":"uid",
+	"users_avatar_queue":"uid",
 	"topics":"tid",
-	"revisions":"reviseID",
+	"users_groups":"gid",
+	"menu_items":"miid",
 }

install/install.go

@@ -128,7 +128,7 @@ func main() {
 		"MinifyTemplates":true,
 		"BuildSlugs":true,
 		"ServerCount":1,
-		"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png",
+		"Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
 		"ItemsPerPage":25
 	},
 	"Database": {
@@ -152,7 +152,6 @@ func main() {
 	}
 }`)
 
-	//"Noavatar": "https://api.adorable.io/avatars/{width}/{id}@{site_url}.png" Maybe allow this sort of syntax?
 	fmt.Println("Opening the configuration file")
 	configFile, err := os.Create("./config/config.json")
 	if err != nil {

main.go

@@ -16,6 +16,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
@@ -158,6 +159,8 @@ func afterDBInit() (err error) {
 	if err != nil {
 		return errors.WithStack(err)
 	}
+	// TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins
+	common.Thumbnailer = common.NewCaireThumbnailer()
 
 	log.Print("Initialising the view counters")
 	counters.GlobalViewCounter, err = counters.NewGlobalViewCounter(acc)
@@ -309,6 +312,7 @@ func main() {
 			return nil
 		}
 
+		// TODO: Expand this to more types of files
 		var err error
 		for {
 			select {
@@ -357,6 +361,58 @@ func main() {
 		}
 	}
 
+	// Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests
+	// TODO: Could we expand this to attachments and other things too?
+	thumbChan := make(chan bool)
+	go func() {
+		acc := qgen.Builder.Accumulator()
+		for {
+			// Put this goroutine to sleep until we have work to do
+			<-thumbChan
+
+			// TODO: Use a real queue
+			err := acc.Select("users_avatar_queue").Columns("uid").Limit("0,5").EachInt(func(uid int) error {
+				//log.Print("uid: ", uid)
+				// TODO: Do a bulk user fetch instead?
+				user, err := common.Users.Get(uid)
+				if err != nil {
+					return errors.WithStack(err)
+				}
+				//log.Print("user.RawAvatar: ", user.RawAvatar)
+
+				// Has the avatar been removed or already been processed by the thumbnailer?
+				if len(user.RawAvatar) < 2 || user.RawAvatar[1] == '.' {
+					_, _ = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
+					return nil
+				}
+				// This means it's an external image, they aren't currently implemented, but this is here for when they are
+				if user.RawAvatar[0] != '.' {
+					return nil
+				}
+				/*if user.RawAvatar == ".gif" {
+					return nil
+				}*/
+				if user.RawAvatar != ".png" && user.RawAvatar != "jpg" && user.RawAvatar != "jpeg" && user.RawAvatar != "gif" {
+					return nil
+				}
+
+				err = common.Thumbnailer.Resize("./uploads/avatar_"+strconv.Itoa(user.ID)+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_tmp"+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_w48"+user.RawAvatar, 48)
+				if err != nil {
+					return errors.WithStack(err)
+				}
+
+				err = user.ChangeAvatar("." + user.RawAvatar)
+				if err != nil {
+					return errors.WithStack(err)
+				}
+				_, err = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
+				return errors.WithStack(err)
+			})
+			if err != nil {
+				common.LogError(err)
+			}
+		}
+	}()
 	// TODO: Write tests for these
 	// Run this goroutine once every half second
 	halfSecondTicker := time.NewTicker(time.Second / 2)
@@ -402,6 +458,7 @@ func main() {
 					continue
 				}
 				runHook("before_second_tick")
+				go func() { thumbChan <- true }()
 				runTasks(common.ScheduledSecondTasks)
 
 				// TODO: Stop hard-coding this

patcher/patches.go

@@ -15,6 +15,7 @@ func init() {
 	addPatch(4, patch4)
 	addPatch(5, patch5)
 	addPatch(6, patch6)
+	addPatch(7, patch7)
 }
 
 func patch0(scanner *bufio.Scanner) (err error) {
@@ -519,3 +520,19 @@ func patch6(scanner *bufio.Scanner) error {
 
 	return nil
 }
+
+func patch7(scanner *bufio.Scanner) error {
+	err := execStmt(qgen.Builder.CreateTable("users_avatar_queue", "", "",
+		[]qgen.DBTableColumn{
+			qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
+		},
+		[]qgen.DBTableKey{
+			qgen.DBTableKey{"uid", "primary"},
+		},
+	))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

query_gen/lib/acc_builders.go

@@ -12,16 +12,31 @@ type accDeleteBuilder struct {
 	build *Accumulator
 }
 
-func (delete *accDeleteBuilder) Where(where string) *accDeleteBuilder {
+func (builder *accDeleteBuilder) Where(where string) *accDeleteBuilder {
-	if delete.where != "" {
+	if builder.where != "" {
-		delete.where += " AND "
+		builder.where += " AND "
 	}
-	delete.where += where
+	builder.where += where
-	return delete
+	return builder
 }
 
-func (delete *accDeleteBuilder) Prepare() *sql.Stmt {
+func (builder *accDeleteBuilder) Prepare() *sql.Stmt {
-	return delete.build.SimpleDelete(delete.table, delete.where)
+	return builder.build.SimpleDelete(builder.table, builder.where)
+}
+
+func (builder *accDeleteBuilder) Run(args ...interface{}) (int, error) {
+	stmt := builder.Prepare()
+	if stmt == nil {
+		return 0, builder.build.FirstError()
+	}
+
+	res, err := stmt.Exec(args...)
+	if err != nil {
+		return 0, err
+	}
+
+	lastID, err := res.LastInsertId()
+	return int(lastID), err
 }
 
 type accUpdateBuilder struct {
@@ -169,6 +184,26 @@ func (selectItem *AccSelectBuilder) Each(handle func(*sql.Rows) error) error {
 	}
 	return rows.Err()
 }
+func (selectItem *AccSelectBuilder) EachInt(handle func(int) error) error {
+	rows, err := selectItem.Query()
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var theInt int
+		err = rows.Scan(&theInt)
+		if err != nil {
+			return err
+		}
+		err = handle(theInt)
+		if err != nil {
+			return err
+		}
+	}
+	return rows.Err()
+}
 
 type accInsertBuilder struct {
 	table   string
@@ -200,6 +235,21 @@ func (insert *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err e
 	return res, insert.build.FirstError()
 }
 
+func (builder *accInsertBuilder) Run(args ...interface{}) (int, error) {
+	stmt := builder.Prepare()
+	if stmt == nil {
+		return 0, builder.build.FirstError()
+	}
+
+	res, err := stmt.Exec(args...)
+	if err != nil {
+		return 0, err
+	}
+
+	lastID, err := res.LastInsertId()
+	return int(lastID), err
+}
+
 type accCountBuilder struct {
 	table string
 	where string

query_gen/lib/accumulator.go

@@ -45,6 +45,7 @@ func (build *Accumulator) recordError(err error) {
 }
 
 func (build *Accumulator) prepare(res string, err error) *sql.Stmt {
+	// TODO: Can we make this less noisy on debug mode?
 	if LogPrepares {
 		log.Print("res: ", res)
 	}

query_gen/tables.go

@@ -127,6 +127,17 @@ func createTables(adapter qgen.Adapter) error {
 		},
 	)
 
+	// TODO: Can we use a piece of software dedicated to persistent queues for this rather than relying on the database for it?
+	qgen.Install.CreateTable("users_avatar_queue", "", "",
+		[]qgen.DBTableColumn{
+			qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
+		},
+		[]qgen.DBTableKey{
+			qgen.DBTableKey{"uid", "primary"},
+		},
+	)
+
+	// TODO: Should we add a users prefix to this table to fit the "unofficial convention"?
 	qgen.Install.CreateTable("emails", "", "",
 		[]qgen.DBTableColumn{
 			qgen.DBTableColumn{"email", "varchar", 200, false, false, ""},

routes/account.go

@@ -447,7 +447,22 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
 		return ferr
 	}
 
-	var filename, ext string
+	// We don't want multiple files
+	// TODO: Are we doing this correctly?
+	filenameMap := make(map[string]bool)
+	for _, fheaders := range r.MultipartForm.File {
+		for _, hdr := range fheaders {
+			if hdr.Filename == "" {
+				continue
+			}
+			filenameMap[hdr.Filename] = true
+		}
+	}
+	if len(filenameMap) > 1 {
+		return common.LocalError("You may only upload one avatar", w, r, user)
+	}
+
+	var ext string
 	for _, fheaders := range r.MultipartForm.File {
 		for _, hdr := range fheaders {
 			if hdr.Filename == "" {
@@ -459,17 +474,6 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
 			}
 			defer infile.Close()
 
-			// We don't want multiple files
-			// TODO: Check the length of r.MultipartForm.File and error rather than doing this x.x
-			if filename != "" {
-				if filename != hdr.Filename {
-					os.Remove("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
-					return common.LocalError("You may only upload one avatar", w, r, user)
-				}
-			} else {
-				filename = hdr.Filename
-			}
-
 			if ext == "" {
 				extarr := strings.Split(hdr.Filename, ".")
 				if len(extarr) < 2 {
@@ -484,8 +488,13 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
 				}
 				ext = reg.ReplaceAllString(ext, "")
 				ext = strings.ToLower(ext)
+
+				if !common.ImageFileExts.Contains(ext) {
+					return common.LocalError("You can only use an image for your avatar", w, r, user)
+				}
 			}
 
+			// TODO: Centralise this string, so we don't have to change it in two different places when it changes
 			outfile, err := os.Create("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
 			if err != nil {
 				return common.LocalError("Upload failed [File Creation Failed]", w, r, user)
@@ -506,6 +515,11 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
 	if err != nil {
 		return common.InternalError(err, w, r)
 	}
+	// TODO: Only schedule a resize if the avatar isn't tiny
+	err = user.ScheduleAvatarResize()
+	if err != nil {
+		return common.InternalError(err, w, r)
+	}
 	http.Redirect(w, r, "/user/edit/?avatar_updated=1", http.StatusSeeOther)
 	return nil
 }

routes/profile.go

@@ -37,7 +37,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
 
 	var err error
 	var replyCreatedAt time.Time
-	var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyTag, replyClassName string
+	var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyMicroAvatar, replyTag, replyClassName string
 	var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int
 	var replyList []common.ReplyUser
 
@@ -93,7 +93,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
 		} else {
 			replyClassName = ""
 		}
-		replyAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar)
+		replyAvatar, replyMicroAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar)
 
 		if group.Tag != "" {
 			replyTag = group.Tag
@@ -109,7 +109,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
 
 		// TODO: Add a hook here
 
-		replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
+		replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
 	}
 	err = rows.Err()
 	if err != nil {

routes/topic.go

@@ -89,9 +89,6 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
 	}
 	topic.RelativeCreatedAt = common.RelativeTime(topic.CreatedAt)
 
-	// TODO: Make a function for this? Build a more sophisticated noavatar handling system?
-	topic.Avatar = common.BuildAvatar(topic.CreatedBy, topic.Avatar)
-
 	var poll common.Poll
 	if topic.Poll != 0 {
 		pPoll, err := common.Polls.Get(topic.Poll)
@@ -129,6 +126,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
 		}
 		defer rows.Close()
 
+		// TODO: Factor the user fields out and embed a user struct instead
 		replyItem := common.ReplyUser{ClassName: ""}
 		for rows.Next() {
 			err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType)
@@ -153,7 +151,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
 			}
 
 			// TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the common.UserStore initialise this?
-			replyItem.Avatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
+			replyItem.Avatar, replyItem.MicroAvatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
 			replyItem.Tag = postGroup.Tag
 			replyItem.RelativeCreatedAt = common.RelativeTime(replyItem.CreatedAt)
 

schema/mssql/query_users_avatar_queue.sql

@@ -0,0 +1,4 @@
+CREATE TABLE [users_avatar_queue] (
+	[uid] int not null,
+	primary key([uid])
+);

schema/mysql/query_users_avatar_queue.sql

@@ -0,0 +1,4 @@
+CREATE TABLE `users_avatar_queue` (
+	`uid` int not null,
+	primary key(`uid`)
+);

schema/pgsql/query_users_avatar_queue.sql

@@ -0,0 +1,4 @@
+CREATE TABLE `users_avatar_queue` (
+	`uid` int not null,
+	primary key(`uid`)
+);

schema/schema.json

@@ -1,5 +1,5 @@
 {
-	"DBVersion":"7",
+	"DBVersion":"8",
 	"DynamicFileVersion":"0",
 	"MinGoVersion":"1.10",
 	"MinVersion":""

themes/nox/public/main.css

@@ -696,6 +696,12 @@ button, .formbutton {
 	.topic_list .topic_middle {
 		display: none;
 	}
+	.topic_left, .topic_right, .topic_middle {
+		width: 50%;
+	}
+	.topic_right_inside .lastName, .topic_left .rowtopic {
+		margin-top: -4px;
+	}
 	.topic_left img, .topic_right img {
 		height: 32px;
 		width: 32px;