diff --git a/common/counters.go b/common/counters.go new file mode 100644 index 00000000..2e8f1196 --- /dev/null +++ b/common/counters.go @@ -0,0 +1,63 @@ +package common + +import ( + "database/sql" + "sync/atomic" + + "../query_gen/lib" +) + +var GlobalViewCounter *BufferedViewCounter + +type BufferedViewCounter struct { + buckets [2]int64 + currentBucket int64 + + insert *sql.Stmt +} + +func NewGlobalViewCounter() (*BufferedViewCounter, error) { + acc := qgen.Builder.Accumulator() + counter := &BufferedViewCounter{ + currentBucket: 0, + insert: acc.SimpleInsert("viewchunks", "count, createdAt", "?,UTC_TIMESTAMP()"), + } + //AddScheduledFifteenMinuteTask(counter.Tick) + AddScheduledSecondTask(counter.Tick) + return counter, acc.FirstError() +} + +func (counter *BufferedViewCounter) Tick() (err error) { + var oldBucket = counter.currentBucket + var nextBucket int64 + if counter.currentBucket == 1 { + nextBucket = 0 + } else { + nextBucket = 1 + } + atomic.AddInt64(&counter.buckets[oldBucket], counter.buckets[nextBucket]) + atomic.StoreInt64(&counter.buckets[nextBucket], 0) + atomic.StoreInt64(&counter.currentBucket, nextBucket) + /*debugLog("counter.buckets[nextBucket]: ", counter.buckets[nextBucket]) + debugLog("counter.buckets[oldBucket]: ", counter.buckets[oldBucket]) + debugLog("counter.currentBucket:", counter.currentBucket) + debugLog("oldBucket:", oldBucket) + debugLog("nextBucket:", nextBucket)*/ + + var previousViewChunk = counter.buckets[oldBucket] + atomic.AddInt64(&counter.buckets[oldBucket], -previousViewChunk) + return counter.insertChunk(previousViewChunk) +} + +func (counter *BufferedViewCounter) Bump() { + atomic.AddInt64(&counter.buckets[counter.currentBucket], 1) +} + +func (counter *BufferedViewCounter) insertChunk(count int64) error { + if count == 0 { + return nil + } + debugLogf("Inserting a viewchunk with a count of %d", count) + _, err := counter.insert.Exec(count) + return err +} diff --git a/common/tasks.go b/common/tasks.go index d9692a17..513a13de 100644 --- a/common/tasks.go +++ b/common/tasks.go @@ -19,6 +19,8 @@ type TaskStmts struct { getSync *sql.Stmt } +var ScheduledSecondTasks []func() error +var ScheduledFifteenMinuteTasks []func() error var taskStmts TaskStmts var lastSync time.Time @@ -33,6 +35,17 @@ func init() { }) } +// AddScheduledSecondTask is not concurrency safe +func AddScheduledSecondTask(task func() error) { + ScheduledSecondTasks = append(ScheduledSecondTasks, task) +} + +// AddScheduledFifteenMinuteTask is not concurrency safe +func AddScheduledFifteenMinuteTask(task func() error) { + ScheduledFifteenMinuteTasks = append(ScheduledFifteenMinuteTasks, task) +} + +// TODO: Use AddScheduledSecondTask func HandleExpiredScheduledGroups() error { rows, err := taskStmts.getExpiredScheduledGroups.Query() if err != nil { @@ -58,6 +71,7 @@ func HandleExpiredScheduledGroups() error { return rows.Err() } +// TODO: Use AddScheduledSecondTask func HandleServerSync() error { var lastUpdate time.Time err := taskStmts.getSync.QueryRow().Scan(&lastUpdate) diff --git a/common/widgets.go b/common/widgets.go index 576806f6..143ff407 100644 --- a/common/widgets.go +++ b/common/widgets.go @@ -165,10 +165,9 @@ func (widget *Widget) Build(hvars interface{}) (string, error) { return widget.Body, nil } - var b bytes.Buffer var headerVars = hvars.(*HeaderVars) err := RunThemeTemplate(headerVars.Theme.Name, widget.Body, hvars, headerVars.Writer) - return string(b.Bytes()), err + return "", err } // TODO: Make a store for this? diff --git a/gen_router.go b/gen_router.go index ad7e7fe5..549c945c 100644 --- a/gen_router.go +++ b/gen_router.go @@ -13,6 +13,64 @@ import ( ) var ErrNoRoute = errors.New("That route doesn't exist.") +var RouteMap = map[string]interface{}{ + "routeAPI": routeAPI, + "routeOverview": routeOverview, + "routeForums": routeForums, + "routeForum": routeForum, + "routeChangeTheme": routeChangeTheme, + "routeShowAttachment": routeShowAttachment, + "routeReportSubmit": routeReportSubmit, + "routeTopicCreate": routeTopicCreate, + "routeTopics": routeTopics, + "routePanelForums": routePanelForums, + "routePanelForumsCreateSubmit": routePanelForumsCreateSubmit, + "routePanelForumsDelete": routePanelForumsDelete, + "routePanelForumsDeleteSubmit": routePanelForumsDeleteSubmit, + "routePanelForumsEdit": routePanelForumsEdit, + "routePanelForumsEditSubmit": routePanelForumsEditSubmit, + "routePanelForumsEditPermsSubmit": routePanelForumsEditPermsSubmit, + "routePanelSettings": routePanelSettings, + "routePanelSettingEdit": routePanelSettingEdit, + "routePanelSettingEditSubmit": routePanelSettingEditSubmit, + "routePanelWordFilters": routePanelWordFilters, + "routePanelWordFiltersCreate": routePanelWordFiltersCreate, + "routePanelWordFiltersEdit": routePanelWordFiltersEdit, + "routePanelWordFiltersEditSubmit": routePanelWordFiltersEditSubmit, + "routePanelWordFiltersDeleteSubmit": routePanelWordFiltersDeleteSubmit, + "routePanelThemes": routePanelThemes, + "routePanelThemesSetDefault": routePanelThemesSetDefault, + "routePanelPlugins": routePanelPlugins, + "routePanelPluginsActivate": routePanelPluginsActivate, + "routePanelPluginsDeactivate": routePanelPluginsDeactivate, + "routePanelPluginsInstall": routePanelPluginsInstall, + "routePanelUsers": routePanelUsers, + "routePanelUsersEdit": routePanelUsersEdit, + "routePanelUsersEditSubmit": routePanelUsersEditSubmit, + "routePanelGroups": routePanelGroups, + "routePanelGroupsEdit": routePanelGroupsEdit, + "routePanelGroupsEditPerms": routePanelGroupsEditPerms, + "routePanelGroupsEditSubmit": routePanelGroupsEditSubmit, + "routePanelGroupsEditPermsSubmit": routePanelGroupsEditPermsSubmit, + "routePanelGroupsCreateSubmit": routePanelGroupsCreateSubmit, + "routePanelBackups": routePanelBackups, + "routePanelLogsMod": routePanelLogsMod, + "routePanelDebug": routePanelDebug, + "routePanel": routePanel, + "routeAccountEditCritical": routeAccountEditCritical, + "routeAccountEditCriticalSubmit": routeAccountEditCriticalSubmit, + "routeAccountEditAvatar": routeAccountEditAvatar, + "routeAccountEditAvatarSubmit": routeAccountEditAvatarSubmit, + "routeAccountEditUsername": routeAccountEditUsername, + "routeAccountEditUsernameSubmit": routeAccountEditUsernameSubmit, + "routeAccountEditEmail": routeAccountEditEmail, + "routeAccountEditEmailTokenSubmit": routeAccountEditEmailTokenSubmit, + "routeProfile": routeProfile, + "routeBanSubmit": routeBanSubmit, + "routeUnban": routeUnban, + "routeActivate": routeActivate, + "routeIps": routeIps, +} type GenRouter struct { UploadHandler func(http.ResponseWriter, *http.Request) @@ -45,27 +103,22 @@ func (router *GenRouter) Handle(_ string, _ http.Handler) { func (router *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, common.User) common.RouteError) { router.Lock() + defer router.Unlock() router.extra_routes[pattern] = handle - router.Unlock() } func (router *GenRouter) RemoveFunc(pattern string) error { router.Lock() + defer router.Unlock() _, ok := router.extra_routes[pattern] if !ok { - router.Unlock() return ErrNoRoute } delete(router.extra_routes, pattern) - router.Unlock() return nil } func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { - //if req.URL.Path == "/" { - // default_route(w,req) - // return - //} if len(req.URL.Path) == 0 || req.URL.Path[0] != '/' { w.WriteHeader(405) w.Write([]byte("")) @@ -95,6 +148,9 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before PreRoute") } + + // Increment the global view counter + common.GlobalViewCounter.Bump() // Deal with the session stuff, etc. user, ok := common.PreRoute(w, req) diff --git a/main.go b/main.go index 32621db9..30465373 100644 --- a/main.go +++ b/main.go @@ -80,6 +80,10 @@ func afterDBInit() (err error) { if err != nil { return err } + common.GlobalViewCounter, err = common.NewGlobalViewCounter() + if err != nil { + return err + } return nil } @@ -163,43 +167,50 @@ func main() { defer watcher.Close() go func() { + var modifiedFileEvent = func(path string) error { + var pathBits = strings.Split(path, "\\") + if len(pathBits) == 0 { + return nil + } + if pathBits[0] == "themes" { + var themeName string + if len(pathBits) >= 2 { + themeName = pathBits[1] + } + if len(pathBits) >= 3 && pathBits[2] == "public" { + // TODO: Handle new themes freshly plopped into the folder? + theme, ok := common.Themes[themeName] + if ok { + return theme.LoadStaticFiles() + } + } + } + return nil + } + + var err error for { select { case event := <-watcher.Events: log.Println("event:", event) + // TODO: Handle file deletes (and renames more graciously by removing the old version of it) if event.Op&fsnotify.Write == fsnotify.Write { log.Println("modified file:", event.Name) - var pathBits = strings.Split(event.Name, "\\") - if len(pathBits) > 0 { - if pathBits[0] == "themes" { - var themeName string - if len(pathBits) >= 1 { - themeName = pathBits[1] - } - - if len(pathBits) >= 2 && pathBits[2] == "public" { - // TODO: Handle new themes freshly plopped into the folder? - theme, ok := common.Themes[themeName] - if ok { - err = theme.LoadStaticFiles() - if err != nil { - common.LogError(err) - } - } - } - - } - } + err = modifiedFileEvent(event.Name) } else if event.Op&fsnotify.Create == fsnotify.Create { log.Println("new file:", event.Name) + err = modifiedFileEvent(event.Name) } - case err := <-watcher.Errors: - log.Println("error:", err) + if err != nil { + common.LogError(err) + } + case err = <-watcher.Errors: + common.LogError(err) } } }() - // TODO: Keep tabs on the theme stuff, and the langpacks + // TODO: Keep tabs on the (non-resource) theme stuff, and the langpacks err = watcher.Add("./public") if err != nil { log.Fatal(err) @@ -226,6 +237,12 @@ func main() { //log.Print("Running the second ticker") // TODO: Add a plugin hook here + for _, task := range common.ScheduledSecondTasks { + if task() != nil { + common.LogError(err) + } + } + err := common.HandleExpiredScheduledGroups() if err != nil { common.LogError(err) @@ -249,6 +266,12 @@ func main() { case <-fifteenMinuteTicker.C: // TODO: Add a plugin hook here + for _, task := range common.ScheduledFifteenMinuteTasks { + if task() != nil { + common.LogError(err) + } + } + // TODO: Automatically lock topics, if they're really old, and the associated setting is enabled. // TODO: Publish scheduled posts. diff --git a/query_gen/tables.go b/query_gen/tables.go index 3f546e51..555a7237 100644 --- a/query_gen/tables.go +++ b/query_gen/tables.go @@ -354,6 +354,14 @@ func createTables(adapter qgen.Adapter) error { []qgen.DBTableKey{}, ) + qgen.Install.CreateTable("viewchunks", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"count", "int", 0, false, false, "0"}, + qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""}, + }, + []qgen.DBTableKey{}, + ) + qgen.Install.CreateTable("sync", "", "", []qgen.DBTableColumn{ qgen.DBTableColumn{"last_update", "datetime", 0, false, false, ""}, diff --git a/router_gen/main.go b/router_gen/main.go index 4b9ccb32..fb0d644d 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -1,10 +1,12 @@ /* WIP Under Construction */ package main -import "log" +import ( + "log" + "os" +) //import "strings" -import "os" var routeList []*RouteImpl var routeGroups []*RouteGroup @@ -15,32 +17,39 @@ func main() { // Load all the routes... routes() - var out string - var fileData = "// Code generated by. DO NOT EDIT.\n/* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */\n" - - for _, route := range routeList { - var end int - if route.Path[len(route.Path)-1] == '/' { - end = len(route.Path) - 1 - } else { - end = len(route.Path) - 1 + var out, routeMap string + var mapIt = func(name string) { + routeMap += "\t\"" + name + "\": " + name + ",\n" + //reverseRouteMap += "\t" + name + ": \"" + name + "\",\n" + } + var countToIndents = func(indent int) (indentor string) { + for i := 0; i < indent; i++ { + indentor += "\t" } - out += "\n\t\tcase \"" + route.Path[0:end] + "\":" - if len(route.RunBefore) > 0 { - for _, runnable := range route.RunBefore { + return indentor + } + var runBefore = func(runnables []Runnable, indent int) (out string) { + var indentor = countToIndents(indent) + if len(runnables) > 0 { + for _, runnable := range runnables { if runnable.Literal { - out += "\n\t\t\t\t\t" + runnable.Contents + out += "\n\t" + indentor + runnable.Contents } else { - out += ` - err = common.` + runnable.Contents + `(w,req,user) - if err != nil { - router.handleError(err,w,req,user) - return - } - ` + out += "\n" + indentor + "err = common." + runnable.Contents + "(w,req,user)\n" + + indentor + "if err != nil {\n" + + indentor + "\trouter.handleError(err,w,req,user)\n" + + indentor + "\treturn\n" + + indentor + "}\n" + indentor } } } + return out + } + + for _, route := range routeList { + var end = len(route.Path) - 1 + out += "\n\t\tcase \"" + route.Path[0:end] + "\":" + out += runBefore(route.RunBefore, 4) out += "\n\t\t\terr = " + route.Name + "(w,req,user" for _, item := range route.Vars { out += "," + item @@ -49,25 +58,13 @@ func main() { if err != nil { router.handleError(err,w,req,user) }` + mapIt(route.Name) } for _, group := range routeGroups { var end = len(group.Path) - 1 - out += ` - case "` + group.Path[0:end] + `":` - for _, runnable := range group.RunBefore { - if runnable.Literal { - out += "\t\t\t" + runnable.Contents - } else { - out += ` - err = common.` + runnable.Contents + `(w,req,user) - if err != nil { - router.handleError(err,w,req,user) - return - } - ` - } - } + out += "\n\t\tcase \"" + group.Path[0:end] + "\":" + out += runBefore(group.RunBefore, 3) out += "\n\t\t\tswitch(req.URL.Path) {" var defaultRoute = blankRoute() @@ -115,30 +112,18 @@ func main() { out += "," + item } out += ")" + mapIt(route.Name) } if defaultRoute.Name != "" { out += "\n\t\t\t\tdefault:" - if len(defaultRoute.RunBefore) > 0 { - for _, runnable := range defaultRoute.RunBefore { - if runnable.Literal { - out += "\n\t\t\t\t\t" + runnable.Contents - } else { - out += ` - err = common.` + runnable.Contents + `(w,req,user) - if err != nil { - router.handleError(err,w,req,user) - return - } - ` - } - } - } + out += runBefore(defaultRoute.RunBefore, 4) out += "\n\t\t\t\t\terr = " + defaultRoute.Name + "(w,req,user" for _, item := range defaultRoute.Vars { out += ", " + item } out += ")" + mapIt(defaultRoute.Name) } out += ` } @@ -147,7 +132,9 @@ func main() { }` } - fileData += `package main + var fileData = `// Code generated by. DO NOT EDIT. +/* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ +package main import ( "log" @@ -160,6 +147,8 @@ import ( ) var ErrNoRoute = errors.New("That route doesn't exist.") +var RouteMap = map[string]interface{}{ +` + routeMap + `} type GenRouter struct { UploadHandler func(http.ResponseWriter, *http.Request) @@ -192,27 +181,22 @@ func (router *GenRouter) Handle(_ string, _ http.Handler) { func (router *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, common.User) common.RouteError) { router.Lock() + defer router.Unlock() router.extra_routes[pattern] = handle - router.Unlock() } func (router *GenRouter) RemoveFunc(pattern string) error { router.Lock() + defer router.Unlock() _, ok := router.extra_routes[pattern] if !ok { - router.Unlock() return ErrNoRoute } delete(router.extra_routes, pattern) - router.Unlock() return nil } func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { - //if req.URL.Path == "/" { - // default_route(w,req) - // return - //} if len(req.URL.Path) == 0 || req.URL.Path[0] != '/' { w.WriteHeader(405) w.Write([]byte("")) @@ -242,6 +226,9 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before PreRoute") } + + // Increment the global view counter + common.GlobalViewCounter.Bump() // Deal with the session stuff, etc. user, ok := common.PreRoute(w, req) diff --git a/schema/mssql/query_viewchunks.sql b/schema/mssql/query_viewchunks.sql new file mode 100644 index 00000000..efff665a --- /dev/null +++ b/schema/mssql/query_viewchunks.sql @@ -0,0 +1,4 @@ +CREATE TABLE [viewchunks] ( + [count] int DEFAULT 0 not null, + [createdAt] datetime not null +); \ No newline at end of file diff --git a/schema/mysql/query_viewchunks.sql b/schema/mysql/query_viewchunks.sql new file mode 100644 index 00000000..803c612e --- /dev/null +++ b/schema/mysql/query_viewchunks.sql @@ -0,0 +1,4 @@ +CREATE TABLE `viewchunks` ( + `count` int DEFAULT 0 not null, + `createdAt` datetime not null +); \ No newline at end of file diff --git a/schema/pgsql/query_viewchunks.sql b/schema/pgsql/query_viewchunks.sql new file mode 100644 index 00000000..4065bcbc --- /dev/null +++ b/schema/pgsql/query_viewchunks.sql @@ -0,0 +1,4 @@ +CREATE TABLE `viewchunks` ( + `count` int DEFAULT 0 not null, + `createdAt` timestamp not null +); \ No newline at end of file diff --git a/template_forum.go b/template_forum.go index 436c2111..3b784f23 100644 --- a/template_forum.go +++ b/template_forum.go @@ -3,9 +3,9 @@ // Code generated by Gosora. More below: /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ package main +import "strconv" import "net/http" import "./common" -import "strconv" // nolint func init() { @@ -115,94 +115,82 @@ if tmpl_forum_vars.CurrentUser.ID != 0 { w.Write(forum_18) if tmpl_forum_vars.CurrentUser.Perms.CreateTopic { w.Write(forum_19) -if tmpl_forum_vars.CurrentUser.Avatar != "" { -w.Write(forum_20) w.Write([]byte(tmpl_forum_vars.CurrentUser.Avatar)) -w.Write(forum_21) -} -w.Write(forum_22) +w.Write(forum_20) w.Write([]byte(strconv.Itoa(tmpl_forum_vars.Forum.ID))) -w.Write(forum_23) +w.Write(forum_21) if tmpl_forum_vars.CurrentUser.Perms.UploadFiles { +w.Write(forum_22) +} +w.Write(forum_23) +} +} w.Write(forum_24) -} -w.Write(forum_25) -} -} -w.Write(forum_26) if len(tmpl_forum_vars.ItemList) != 0 { for _, item := range tmpl_forum_vars.ItemList { -w.Write(forum_27) +w.Write(forum_25) w.Write([]byte(strconv.Itoa(item.ID))) -w.Write(forum_28) +w.Write(forum_26) if item.Sticky { -w.Write(forum_29) +w.Write(forum_27) } else { if item.IsClosed { +w.Write(forum_28) +} +} +w.Write(forum_29) +w.Write([]byte(item.Creator.Link)) w.Write(forum_30) -} -} -w.Write(forum_31) -if item.Creator.Avatar != "" { -w.Write(forum_32) -w.Write([]byte(item.Creator.Link)) -w.Write(forum_33) w.Write([]byte(item.Creator.Avatar)) -w.Write(forum_34) -} -w.Write(forum_35) +w.Write(forum_31) w.Write([]byte(item.Link)) -w.Write(forum_36) +w.Write(forum_32) w.Write([]byte(item.Title)) -w.Write(forum_37) +w.Write(forum_33) w.Write([]byte(item.Creator.Link)) -w.Write(forum_38) +w.Write(forum_34) w.Write([]byte(item.Creator.Name)) -w.Write(forum_39) +w.Write(forum_35) if item.IsClosed { -w.Write(forum_40) +w.Write(forum_36) } if item.Sticky { +w.Write(forum_37) +} +w.Write(forum_38) +w.Write([]byte(strconv.Itoa(item.PostCount))) +w.Write(forum_39) +w.Write([]byte(strconv.Itoa(item.LikeCount))) +w.Write(forum_40) +if item.Sticky { w.Write(forum_41) -} -w.Write(forum_42) -w.Write([]byte(strconv.Itoa(item.PostCount))) -w.Write(forum_43) -w.Write([]byte(strconv.Itoa(item.LikeCount))) -w.Write(forum_44) -if item.Sticky { -w.Write(forum_45) } else { if item.IsClosed { -w.Write(forum_46) +w.Write(forum_42) } } -w.Write(forum_47) -if item.LastUser.Avatar != "" { -w.Write(forum_48) +w.Write(forum_43) w.Write([]byte(item.LastUser.Link)) -w.Write(forum_49) +w.Write(forum_44) w.Write([]byte(item.LastUser.Avatar)) -w.Write(forum_50) -} -w.Write(forum_51) +w.Write(forum_45) w.Write([]byte(item.LastUser.Link)) -w.Write(forum_52) +w.Write(forum_46) w.Write([]byte(item.LastUser.Name)) -w.Write(forum_53) +w.Write(forum_47) w.Write([]byte(item.RelativeLastReplyAt)) -w.Write(forum_54) +w.Write(forum_48) } } else { -w.Write(forum_55) +w.Write(forum_49) if tmpl_forum_vars.CurrentUser.Perms.CreateTopic { -w.Write(forum_56) +w.Write(forum_50) w.Write([]byte(strconv.Itoa(tmpl_forum_vars.Forum.ID))) -w.Write(forum_57) +w.Write(forum_51) } -w.Write(forum_58) +w.Write(forum_52) } -w.Write(forum_59) +w.Write(forum_53) w.Write(footer_0) w.Write([]byte(common.BuildWidget("footer",tmpl_forum_vars.Header))) w.Write(footer_1) diff --git a/template_list.go b/template_list.go index 2bea57a2..5f29baee 100644 --- a/template_list.go +++ b/template_list.go @@ -42,30 +42,31 @@ var menu_0 = []byte(`