a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

PNG and JPG avatars are now encoded as JPG images leading to a dramatic drop in the amount of bandwidth used. 

Did some work on image thumbnailing, but our dependencies are acting up delaying this from being released.

Fixed the positions of the topic list bits for Nox on mobile.
Removed APNG as an accepted image format, as we don't currently have a good way of optimising these images.
Added a comment regarding the constant time compare for sessions.
Added a warning about putting Gosora in www folders.
Noavatars can now take a width parameters.
Added a bit of missing validation for the avatar uploader.
Refactored the multiple file detector for the avatar uploader.

Added a Run method to accDeleteBuilder.
Added an EachInt method to AccSelectBuilder.
Added a Run method to accInsertBuilder.

Added the users_avatar_queue table, you will need to run the patcher / update script.
You might also want to update the Noavatar field in your config.json file with the new one.
This commit is contained in:
Azareal 2018-07-28 22:52:23 +10:00
parent cee027cc7f
commit 0a628f7201
27 changed files with 356 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -65,7 +65,9 @@ On Windows, you might want to try the [GosoraBootstrapper](https://github.com/Az
*Linux*
First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path.
First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path (but *never* in a folder where the files are automatically served by a webserver).
If you place it in `/www/`, `/public_html/` or any similar folder, then there's a chance that your server might be compromised.
You can navigate to it by typing the following six commands into the console and hitting enter:

1
common/auth.go
View File

@ -250,6 +250,7 @@ func (auth *DefaultAuth) SessionCheck(w http.ResponseWriter, r *http.Request) (u
return &GuestUser, true
}
// We need to do a constant time compare, otherwise someone might be able to deduce the session character by character based on how long it takes to do the comparison. Change this at your own peril.
if user.Session == "" || subtle.ConstantTimeCompare([]byte(session), []byte(user.Session)) != 1 {
return &GuestUser, false
}

2
common/common.go
View File

@ -43,7 +43,7 @@ type StringList []string
// ? - Should we allow users to upload .php or .go files? It could cause security issues. We could store them with a mangled extension to render them inert
// TODO: Let admins manage this from the Control Panel
var AllowedFileExts = StringList{
"png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", "apng", // images
"png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", /*"apng",*/ // images
"txt", "xml", "json", "yaml", "toml", "ini", "md", "html", "rtf", "js", "py", "rb", "css", "scss", "less", "eqcss", "pcss", "java", "ts", "cs", "c", "cc", "cpp", "cxx", "C", "c++", "h", "hh", "hpp", "hxx", "h++", "rs", "rlib", "htaccess", "gitignore", // text

1
common/reply.go
View File

@ -29,6 +29,7 @@ type ReplyUser struct {
LastEdit int
LastEditBy int
Avatar string
MicroAvatar string
ClassName string
ContentLines int
Tag string

2
common/site.go
View File

@ -173,7 +173,7 @@ func ProcessConfig() (err error) {
if Config.MaxUsernameLength == 0 {
Config.MaxUsernameLength = 100
}
GuestUser.Avatar = BuildAvatar(0, "")
GuestUser.Avatar, GuestUser.MicroAvatar = BuildAvatar(0, "")
if Config.HashAlgo != "" {
// TODO: Set the alternate hash algo, e.g. argon2

25
common/template_init.go
View File

@ -123,10 +123,15 @@ var Template_ip_search_handle = func(pi IPSearchPage, w io.Writer) error {
}
func tmplInitUsers() (User, User, User) {
user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, BuildAvatar(62, ""), "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0}
avatar, microAvatar := BuildAvatar(62, "")
user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, "", avatar, microAvatar, "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0}
// TODO: Do a more accurate level calculation for this?
user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(1, ""), "", "", "", "", 58, 1000, 0, "127.0.0.1", 0}
user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(2, ""), "", "", "", "", 42, 900, 0, "::1", 0}
avatar, microAvatar = BuildAvatar(1, "")
user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 58, 1000, 0, "127.0.0.1", 0}
avatar, microAvatar = BuildAvatar(2, "")
user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 42, 900, 0, "::1", 0}
return user, user2, user3
}
@ -183,9 +188,12 @@ func CompileTemplates() error {
PollOption{0, "Nothing"},
PollOption{1, "Something"},
}, VoteCount: 7}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false}
avatar, microAvatar := BuildAvatar(62, "")
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
var replyList []ReplyUser
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
// TODO: Do we want the UID on this to be 0?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
var varList = make(map[string]tmpl.VarItem)
header.Title = "Topic Name"
@ -367,9 +375,12 @@ func CompileJSTemplates() error {
PollOption{0, "Nothing"},
PollOption{1, "Something"},
}, VoteCount: 7}
topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false}
avatar, microAvatar := BuildAvatar(62, "")
topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false}
var replyList []ReplyUser
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
// TODO: Do we really want the UID here to be zero?
avatar, microAvatar = BuildAvatar(0, "")
replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""})
varList = make(map[string]tmpl.VarItem)
header.Title = "Topic Name"

70
common/thumbnailer.go
View File

@ -1,28 +1,86 @@
package common
import (
"image"
_ "image/gif"
"image/jpeg"
_ "image/png"
"os"
)
var Thumbnailer ThumbnailerInt
type ThumbnailerInt interface {
Resize(inPath string, tmpPath string, outPath string, width int) error
}
type RezThumbnailer struct {
}
func (thumb *RezThumbnailer) Resize(path string, width int) error {
func (thumb *RezThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
// TODO: Sniff the aspect ratio of the image and calculate the dest height accordingly, bug make sure it isn't excessively high
return nil
}
func (thumb *RezThumbnailer) resize(path string, width int, height int) error {
func (thumb *RezThumbnailer) resize(inPath string, outPath string, width int, height int) error {
return nil
}
// ! Note: CaireThumbnailer can't handle gifs, so we'll have to either cap their sizes or have another resizer deal with them
type CaireThumbnailer struct {
}
func NewCaireThumbnailer() *CaireThumbnailer {
return &CaireThumbnailer{}
}
func precodeImage(inPath string, tmpPath string) error {
imageFile, err := os.Open(inPath)
if err != nil {
return err
}
defer imageFile.Close()
img, _, err := image.Decode(imageFile)
if err != nil {
return err
}
outFile, err := os.Create(tmpPath)
if err != nil {
return err
}
defer outFile.Close()
return jpeg.Encode(outFile, img, nil)
}
func (thumb *CaireThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error {
err := precodeImage(inPath, tmpPath)
if err != nil {
return err
}
return nil
// TODO: Caire doesn't work. Try something else. Or get them to fix the index out of range. We get enough wins from re-encoding as jpeg anyway
/*imageFile, err := os.Open(tmpPath)
if err != nil {
return err
}
defer imageFile.Close()
outFile, err := os.Create(outPath)
if err != nil {
return err
}
defer outFile.Close()
p := &caire.Processor{NewWidth: width, Scale: true}
return p.Process(imageFile, outFile)*/
}
/*
type LilliputThumbnailer struct {
}
type ResizeThumbnailer struct {
}
*/

3
common/topic.go
View File

@ -70,6 +70,7 @@ type TopicUser struct {
CreatedByName string
Group int
Avatar string
MicroAvatar string
ContentLines int
ContentHTML string
Tag string
@ -366,6 +367,7 @@ func GetTopicUser(tid int) (TopicUser, error) {
tu := TopicUser{ID: tid}
err := topicStmts.getTopicUser.QueryRow(tid).Scan(&tu.Title, &tu.Content, &tu.CreatedBy, &tu.CreatedAt, &tu.IsClosed, &tu.Sticky, &tu.ParentID, &tu.IPAddress, &tu.PostCount, &tu.LikeCount, &tu.Poll, &tu.CreatedByName, &tu.Avatar, &tu.Group, &tu.URLPrefix, &tu.URLName, &tu.Level)
tu.Avatar, tu.MicroAvatar = BuildAvatar(tu.CreatedBy, tu.Avatar)
tu.Link = BuildTopicURL(NameToSlug(tu.Title), tu.ID)
tu.UserLink = BuildProfileURL(NameToSlug(tu.CreatedByName), tu.CreatedBy)
tu.Tag = Groups.DirtyGet(tu.Group).Tag
@ -383,6 +385,7 @@ func copyTopicToTopicUser(topic *Topic, user *User) (tu TopicUser) {
tu.CreatedByName = user.Name
tu.Group = user.Group
tu.Avatar = user.Avatar
tu.MicroAvatar = user.MicroAvatar
tu.URLPrefix = user.URLPrefix
tu.URLName = user.URLName
tu.Level = user.Level

66
common/user.go
View File

@ -14,6 +14,7 @@ import (
"time"
"../query_gen/lib"
"github.com/go-sql-driver/mysql"
)
// TODO: Replace any literals with this
@ -40,17 +41,19 @@ type User struct {
PluginPerms map[string]bool
Session string
//AuthToken string
Loggedin bool
Avatar string
Message string
URLPrefix string // Move this to another table? Create a user lite?
URLName string
Tag string
Level int
Score int
Liked int
LastIP string // ! This part of the UserCache data might fall out of date
TempGroup int
Loggedin bool
RawAvatar string
Avatar string
MicroAvatar string
Message string
URLPrefix string // Move this to another table? Create a user lite?
URLName string
Tag string
Level int
Score int
Liked int
LastIP string // ! This part of the UserCache data might fall out of date
TempGroup int
}
func (user *User) WebSockets() *WsJSONUser {
@ -96,6 +99,8 @@ type UserStmts struct {
updateLastIP *sql.Stmt
setPassword *sql.Stmt
scheduleAvatarResize *sql.Stmt
}
var userStmts UserStmts
@ -123,13 +128,15 @@ func init() {
updateLastIP: acc.SimpleUpdate("users", "last_ip = ?", where),
setPassword: acc.Update("users").Set("password = ?, salt = ?").Where(where).Prepare(),
scheduleAvatarResize: acc.Insert("users_avatar_queue").Columns("uid").Fields("?").Prepare(),
}
return acc.FirstError()
})
}
func (user *User) Init() {
user.Avatar = BuildAvatar(user.ID, user.Avatar)
user.Avatar, user.MicroAvatar = BuildAvatar(user.ID, user.RawAvatar)
user.Link = BuildProfileURL(NameToSlug(user.Name), user.ID)
user.Tag = Groups.DirtyGet(user.Group).Tag
user.InitPerms()
@ -268,6 +275,23 @@ func (user *User) ChangeAvatar(avatar string) (err error) {
return user.bindStmt(userStmts.setAvatar, avatar)
}
// TODO: Abstract this with an interface so we can scale this with an actual dedicated queue in a real cluster
func (user *User) ScheduleAvatarResize() (err error) {
_, err = userStmts.scheduleAvatarResize.Exec(user.ID)
if err != nil {
// TODO: Do a more generic check so that we're not as tied to MySQL
me, ok := err.(*mysql.MySQLError)
if !ok {
return err
}
// If it's just telling us that the item already exists in the database, then we can ignore it, as it doesn't matter if it's this call or another which schedules the item in the queue
if me.Number != 1062 {
return err
}
}
return nil
}
func (user *User) ChangeGroup(group int) (err error) {
return user.bindStmt(userStmts.changeGroup, group)
}
@ -381,15 +405,25 @@ func (user *User) InitPerms() {
}
}
func buildNoavatar(uid int, width int) string {
return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1)
}
// ? Make this part of *User?
func BuildAvatar(uid int, avatar string) string {
// TODO: Write tests for this
func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar string) {
if avatar != "" {
if avatar[0] == '.' {
return "/uploads/avatar_" + strconv.Itoa(uid) + avatar
if avatar[1] == '.' {
normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + "_tmp" + avatar[1:]
return normalAvatar, normalAvatar
}
normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + avatar
return normalAvatar, normalAvatar
}
return avatar
return avatar, avatar
}
return strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1)
return buildNoavatar(uid, 200), buildNoavatar(uid, 48)
}
// TODO: Move this to *User

12
common/user_store.go
View File

@ -68,7 +68,7 @@ func (mus *DefaultUserStore) DirtyGet(id int) *User {
}
user = &User{ID: id, Loggedin: true}
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init()
if err == nil {
@ -86,7 +86,7 @@ func (mus *DefaultUserStore) Get(id int) (*User, error) {
}
user = &User{ID: id, Loggedin: true}
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init()
if err == nil {
@ -106,7 +106,7 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User
for rows.Next() {
user := &User{Loggedin: true}
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil {
return nil, err
}
@ -162,7 +162,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
for rows.Next() {
user := &User{Loggedin: true}
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil {
return list, err
}
@ -203,7 +203,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro
func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
user := &User{ID: id, Loggedin: true}
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
user.Init()
return user, err
@ -211,7 +211,7 @@ func (mus *DefaultUserStore) BypassGet(id int) (*User, error) {
func (mus *DefaultUserStore) Reload(id int) error {
user := &User{ID: id, Loggedin: true}
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup)
if err != nil {
mus.cache.Remove(id)
return err

2
config/config_example.json
View File

@ -30,7 +30,7 @@
"MinifyTemplates":true,
"BuildSlugs":true,
"ServerCount":1,
"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png",
"Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
"ItemsPerPage":25
},
"Database": {

4
extend/guilds/lib/guilds.go
View File

@ -329,12 +329,12 @@ func RouteMemberList(w http.ResponseWriter, r *http.Request, user common.User) c
var guildMembers []Member
for rows.Next() {
guildMember := Member{PostCount: 0}
err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.Avatar)
err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.RawAvatar)
if err != nil {
return common.InternalError(err, w, r)
}
guildMember.Link = common.BuildProfileURL(common.NameToSlug(guildMember.User.Name), guildMember.User.ID)
guildMember.User.Avatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.Avatar)
guildMember.User.Avatar, guildMember.User.MicroAvatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.RawAvatar)
guildMember.JoinedAt, _ = common.RelativeTimeFromString(guildMember.JoinedAt)
if guildItem.Owner == guildMember.User.ID {
guildMember.RankString = "Owner"

21
gen_tables.go
View File

@ -2,21 +2,22 @@
package main
var dbTablePrimaryKeys = map[string]string{
"users_groups":"gid",
"attachments":"attachID",
"users_replies":"rid",
"menu_items":"miid",
"pages":"pid",
"polls":"pollID",
"activity_stream":"asid",
"users_groups_scheduler":"uid",
"replies":"rid",
"attachments":"attachID",
"revisions":"reviseID",
"polls":"pollID",
"users_replies":"rid",
"activity_stream":"asid",
"users":"uid",
"pages":"pid",
"word_filters":"wfid",
"menus":"mid",
"registration_logs":"rlid",
"users":"uid",
"users_2fa_keys":"uid",
"forums":"fid",
"users_2fa_keys":"uid",
"users_avatar_queue":"uid",
"topics":"tid",
"revisions":"reviseID",
"users_groups":"gid",
"menu_items":"miid",
}

3
install/install.go
View File

@ -128,7 +128,7 @@ func main() {
"MinifyTemplates":true,
"BuildSlugs":true,
"ServerCount":1,
"Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png",
"Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png",
"ItemsPerPage":25
},
"Database": {
@ -152,7 +152,6 @@ func main() {
}
}`)
//"Noavatar": "https://api.adorable.io/avatars/{width}/{id}@{site_url}.png" Maybe allow this sort of syntax?
fmt.Println("Opening the configuration file")
configFile, err := os.Create("./config/config.json")
if err != nil {

57
main.go
View File

@ -16,6 +16,7 @@ import (
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
@ -158,6 +159,8 @@ func afterDBInit() (err error) {
if err != nil {
return errors.WithStack(err)
}
// TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins
common.Thumbnailer = common.NewCaireThumbnailer()
log.Print("Initialising the view counters")
counters.GlobalViewCounter, err = counters.NewGlobalViewCounter(acc)
@ -309,6 +312,7 @@ func main() {
return nil
}
// TODO: Expand this to more types of files
var err error
for {
select {
@ -357,6 +361,58 @@ func main() {
}
}
// Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests
// TODO: Could we expand this to attachments and other things too?
thumbChan := make(chan bool)
go func() {
acc := qgen.Builder.Accumulator()
for {
// Put this goroutine to sleep until we have work to do
<-thumbChan
// TODO: Use a real queue
err := acc.Select("users_avatar_queue").Columns("uid").Limit("0,5").EachInt(func(uid int) error {
//log.Print("uid: ", uid)
// TODO: Do a bulk user fetch instead?
user, err := common.Users.Get(uid)
if err != nil {
return errors.WithStack(err)
}
//log.Print("user.RawAvatar: ", user.RawAvatar)
// Has the avatar been removed or already been processed by the thumbnailer?
if len(user.RawAvatar) < 2 || user.RawAvatar[1] == '.' {
_, _ = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
return nil
}
// This means it's an external image, they aren't currently implemented, but this is here for when they are
if user.RawAvatar[0] != '.' {
return nil
}
/*if user.RawAvatar == ".gif" {
return nil
}*/
if user.RawAvatar != ".png" && user.RawAvatar != "jpg" && user.RawAvatar != "jpeg" && user.RawAvatar != "gif" {
return nil
}
err = common.Thumbnailer.Resize("./uploads/avatar_"+strconv.Itoa(user.ID)+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_tmp"+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_w48"+user.RawAvatar, 48)
if err != nil {
return errors.WithStack(err)
}
err = user.ChangeAvatar("." + user.RawAvatar)
if err != nil {
return errors.WithStack(err)
}
_, err = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid)
return errors.WithStack(err)
})
if err != nil {
common.LogError(err)
}
}
}()
// TODO: Write tests for these
// Run this goroutine once every half second
halfSecondTicker := time.NewTicker(time.Second / 2)
@ -402,6 +458,7 @@ func main() {
continue
}
runHook("before_second_tick")
go func() { thumbChan <- true }()
runTasks(common.ScheduledSecondTasks)
// TODO: Stop hard-coding this

17
patcher/patches.go
View File

@ -15,6 +15,7 @@ func init() {
addPatch(4, patch4)
addPatch(5, patch5)
addPatch(6, patch6)
addPatch(7, patch7)
}
func patch0(scanner *bufio.Scanner) (err error) {
@ -519,3 +520,19 @@ func patch6(scanner *bufio.Scanner) error {
return nil
}
func patch7(scanner *bufio.Scanner) error {
err := execStmt(qgen.Builder.CreateTable("users_avatar_queue", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
},
[]qgen.DBTableKey{
qgen.DBTableKey{"uid", "primary"},
},
))
if err != nil {
return err
}
return nil
}

64
query_gen/lib/acc_builders.go
View File

@ -12,16 +12,31 @@ type accDeleteBuilder struct {
build *Accumulator
}
func (delete *accDeleteBuilder) Where(where string) *accDeleteBuilder {
if delete.where != "" {
delete.where += " AND "
func (builder *accDeleteBuilder) Where(where string) *accDeleteBuilder {
if builder.where != "" {
builder.where += " AND "
}
delete.where += where
return delete
builder.where += where
return builder
}
func (delete *accDeleteBuilder) Prepare() *sql.Stmt {
return delete.build.SimpleDelete(delete.table, delete.where)
func (builder *accDeleteBuilder) Prepare() *sql.Stmt {
return builder.build.SimpleDelete(builder.table, builder.where)
}
func (builder *accDeleteBuilder) Run(args ...interface{}) (int, error) {
stmt := builder.Prepare()
if stmt == nil {
return 0, builder.build.FirstError()
}
res, err := stmt.Exec(args...)
if err != nil {
return 0, err
}
lastID, err := res.LastInsertId()
return int(lastID), err
}
type accUpdateBuilder struct {
@ -169,6 +184,26 @@ func (selectItem *AccSelectBuilder) Each(handle func(*sql.Rows) error) error {
}
return rows.Err()
}
func (selectItem *AccSelectBuilder) EachInt(handle func(int) error) error {
rows, err := selectItem.Query()
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var theInt int
err = rows.Scan(&theInt)
if err != nil {
return err
}
err = handle(theInt)
if err != nil {
return err
}
}
return rows.Err()
}
type accInsertBuilder struct {
table string
@ -200,6 +235,21 @@ func (insert *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err e
return res, insert.build.FirstError()
}
func (builder *accInsertBuilder) Run(args ...interface{}) (int, error) {
stmt := builder.Prepare()
if stmt == nil {
return 0, builder.build.FirstError()
}
res, err := stmt.Exec(args...)
if err != nil {
return 0, err
}
lastID, err := res.LastInsertId()
return int(lastID), err
}
type accCountBuilder struct {
table string
where string

1
query_gen/lib/accumulator.go
View File

@ -45,6 +45,7 @@ func (build *Accumulator) recordError(err error) {
}
func (build *Accumulator) prepare(res string, err error) *sql.Stmt {
// TODO: Can we make this less noisy on debug mode?
if LogPrepares {
log.Print("res: ", res)
}

11
query_gen/tables.go
View File

@ -127,6 +127,17 @@ func createTables(adapter qgen.Adapter) error {
},
)
// TODO: Can we use a piece of software dedicated to persistent queues for this rather than relying on the database for it?
qgen.Install.CreateTable("users_avatar_queue", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key
},
[]qgen.DBTableKey{
qgen.DBTableKey{"uid", "primary"},
},
)
// TODO: Should we add a users prefix to this table to fit the "unofficial convention"?
qgen.Install.CreateTable("emails", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"email", "varchar", 200, false, false, ""},

38
routes/account.go
View File

@ -447,7 +447,22 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
return ferr
}
var filename, ext string
// We don't want multiple files
// TODO: Are we doing this correctly?
filenameMap := make(map[string]bool)
for _, fheaders := range r.MultipartForm.File {
for _, hdr := range fheaders {
if hdr.Filename == "" {
continue
}
filenameMap[hdr.Filename] = true
}
}
if len(filenameMap) > 1 {
return common.LocalError("You may only upload one avatar", w, r, user)
}
var ext string
for _, fheaders := range r.MultipartForm.File {
for _, hdr := range fheaders {
if hdr.Filename == "" {
@ -459,17 +474,6 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
}
defer infile.Close()
// We don't want multiple files
// TODO: Check the length of r.MultipartForm.File and error rather than doing this x.x
if filename != "" {
if filename != hdr.Filename {
os.Remove("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
return common.LocalError("You may only upload one avatar", w, r, user)
}
} else {
filename = hdr.Filename
}
if ext == "" {
extarr := strings.Split(hdr.Filename, ".")
if len(extarr) < 2 {
@ -484,8 +488,13 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
}
ext = reg.ReplaceAllString(ext, "")
ext = strings.ToLower(ext)
if !common.ImageFileExts.Contains(ext) {
return common.LocalError("You can only use an image for your avatar", w, r, user)
}
}
// TODO: Centralise this string, so we don't have to change it in two different places when it changes
outfile, err := os.Create("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext)
if err != nil {
return common.LocalError("Upload failed [File Creation Failed]", w, r, user)
@ -506,6 +515,11 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Only schedule a resize if the avatar isn't tiny
err = user.ScheduleAvatarResize()
if err != nil {
return common.InternalError(err, w, r)
}
http.Redirect(w, r, "/user/edit/?avatar_updated=1", http.StatusSeeOther)
return nil
}

6
routes/profile.go
View File

@ -37,7 +37,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
var err error
var replyCreatedAt time.Time
var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyTag, replyClassName string
var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyMicroAvatar, replyTag, replyClassName string
var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int
var replyList []common.ReplyUser
@ -93,7 +93,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
} else {
replyClassName = ""
}
replyAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar)
replyAvatar, replyMicroAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar)
if group.Tag != "" {
replyTag = group.Tag
@ -109,7 +109,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo
// TODO: Add a hook here
replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""})
}
err = rows.Err()
if err != nil {

6
routes/topic.go
View File

@ -89,9 +89,6 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
}
topic.RelativeCreatedAt = common.RelativeTime(topic.CreatedAt)
// TODO: Make a function for this? Build a more sophisticated noavatar handling system?
topic.Avatar = common.BuildAvatar(topic.CreatedBy, topic.Avatar)
var poll common.Poll
if topic.Poll != 0 {
pPoll, err := common.Polls.Get(topic.Poll)
@ -129,6 +126,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
}
defer rows.Close()
// TODO: Factor the user fields out and embed a user struct instead
replyItem := common.ReplyUser{ClassName: ""}
for rows.Next() {
err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType)
@ -153,7 +151,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit
}
// TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the common.UserStore initialise this?
replyItem.Avatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
replyItem.Avatar, replyItem.MicroAvatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar)
replyItem.Tag = postGroup.Tag
replyItem.RelativeCreatedAt = common.RelativeTime(replyItem.CreatedAt)

4
schema/mssql/query_users_avatar_queue.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE [users_avatar_queue] (
[uid] int not null,
primary key([uid])
);

4
schema/mysql/query_users_avatar_queue.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE `users_avatar_queue` (
`uid` int not null,
primary key(`uid`)
);

4
schema/pgsql/query_users_avatar_queue.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE `users_avatar_queue` (
`uid` int not null,
primary key(`uid`)
);

2
schema/schema.json
View File

@ -1,5 +1,5 @@
{
"DBVersion":"7",
"DBVersion":"8",
"DynamicFileVersion":"0",
"MinGoVersion":"1.10",
"MinVersion":""

6
themes/nox/public/main.css
View File

@ -696,6 +696,12 @@ button, .formbutton {
.topic_list .topic_middle {
display: none;
}
.topic_left, .topic_right, .topic_middle {
width: 50%;
}
.topic_right_inside .lastName, .topic_left .rowtopic {
margin-top: -4px;
}
.topic_left img, .topic_right img {
height: 32px;
width: 32px;