gosora/common/counters/referrers.go

124 lines
3.4 KiB
Go
Raw Normal View History

package counters
import (
"database/sql"
"sync"
"sync/atomic"
c "github.com/Azareal/Gosora/common"
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
qgen "github.com/Azareal/Gosora/query_gen"
"github.com/pkg/errors"
)
var ReferrerTracker *DefaultReferrerTracker
// Add ReferrerItems here after they've had zero views for a while
var referrersToDelete = make(map[string]*ReferrerItem)
type ReferrerItem struct {
Count int64
}
// ? We'll track referrer domains here rather than the exact URL they arrived from for now, we'll think about expanding later
// ? Referrers are fluid and ever-changing so we have to use string keys rather than 'enum' ints
type DefaultReferrerTracker struct {
odd map[string]*ReferrerItem
even map[string]*ReferrerItem
oddLock sync.RWMutex
evenLock sync.RWMutex
insert *sql.Stmt
}
func NewDefaultReferrerTracker() (*DefaultReferrerTracker, error) {
acc := qgen.NewAcc()
refTracker := &DefaultReferrerTracker{
odd: make(map[string]*ReferrerItem),
even: make(map[string]*ReferrerItem),
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
insert: acc.Insert("viewchunks_referrers").Columns("count,createdAt,domain").Fields("?,UTC_TIMESTAMP(),?").Prepare(), // TODO: Do something more efficient than doing a query for each referrer
}
c.Tasks.FifteenMin.Add(refTracker.Tick)
//c.Tasks.Sec.Add(refTracker.Tick)
c.Tasks.Shutdown.Add(refTracker.Tick)
return refTracker, acc.FirstError()
}
// TODO: Move this and the other view tickers out of the main task loop to avoid blocking other tasks?
func (ref *DefaultReferrerTracker) Tick() (err error) {
for referrer, counter := range referrersToDelete {
// Handle views which squeezed through the gaps at the last moment
count := counter.Count
if count != 0 {
err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
if err != nil {
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
return errors.Wrap(errors.WithStack(err), "ref counter")
}
}
delete(referrersToDelete, referrer)
}
// Run the queries and schedule zero view refs for deletion from memory
refLoop := func(l *sync.RWMutex, m map[string]*ReferrerItem) error {
l.Lock()
defer l.Unlock()
for referrer, counter := range m {
if counter.Count == 0 {
referrersToDelete[referrer] = counter
delete(m, referrer)
}
count := atomic.SwapInt64(&counter.Count, 0)
err := ref.insertChunk(referrer, count) // TODO: Bulk insert for speed?
if err != nil {
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
return errors.Wrap(errors.WithStack(err), "ref counter")
}
}
return nil
}
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
err = refLoop(&ref.oddLock, ref.odd)
if err != nil {
return err
}
initial perf anaytics add tasks to debug page ignore .git on debug page for speed add perfchunks table Renamed phrases (changed statistics to stats): panel_menu_stats panel_menu_stats_posts panel_menu_stats_topics panel_menu_stats_forums panel_menu_stats_routes panel_menu_stats_agents panel_menu_stats_systems panel_menu_stats_languages panel_menu_stats_referrers panel_menu_stats_memory panel_menu_stats_active_memory panel_menu_stats_perf panel_stats_views_head_suffix panel_stats_user_agents_head panel_stats_forums_head panel_stats_languages_head panel_stats_post_counts_head panel_stats_referrers_head panel_stats_routes_head panel_stats_operating_systems_head panel_stats_topic_counts_head panel_stats_requests_head panel_stats_memory_head panel_stats_active_memory_head panel_stats_spam_hide panel_stats_spam_show panel_stats_memory_type_total panel_stats_memory_type_stack panel_stats_memory_type_heap panel_stats_time_range_one_year panel_stats_time_range_three_months panel_stats_time_range_one_month panel_stats_time_range_one_week panel_stats_time_range_two_days panel_stats_time_range_one_day panel_stats_time_range_twelve_hours panel_stats_time_range_six_hours panel_stats_post_counts_chart_aria panel_stats_topic_counts_chart_aria panel_stats_requests_chart_aria panel_stats_memory_chart_aria panel_stats_details_head panel_stats_post_counts_table_aria panel_stats_topic_counts_table_aria panel_stats_route_views_table_aria panel_stats_requests_table_aria panel_stats_memory_table_aria panel_stats_views_suffix panel_stats_posts_suffix panel_stats_topics_suffix panel_stats_user_agents_no_user_agents panel_stats_forums_no_forums panel_stats_languages_no_languages panel_stats_post_counts_no_post_counts panel_stats_referrers_no_referrers panel_stats_routes_no_routes panel_stats_operating_systems_no_operating_systems panel_stats_memory_no_memory Added phrases: panel_debug_tasks panel_debug_tasks_half_second panel_debug_tasks_second panel_debug_tasks_fifteen_minute panel_debug_tasks_hour panel_debug_tasks_shutdown panel_stats_perf_head panel_stats_perf_low panel_stats_perf_high panel_stats_perf_avg You will need to run the updater / patcher for this commit.
2020-02-23 09:08:47 +00:00
return refLoop(&ref.evenLock, ref.even)
}
func (ref *DefaultReferrerTracker) insertChunk(referrer string, count int64) error {
if count == 0 {
return nil
}
c.DebugDetailf("Inserting a vchunk with a count of %d for ref %s", count, referrer)
_, err := ref.insert.Exec(count, referrer)
return err
}
func (ref *DefaultReferrerTracker) Bump(referrer string) {
if referrer == "" {
return
}
var refItem *ReferrerItem
// Slightly crude and rudimentary, but it should give a basic degree of sharding
if referrer[0]%2 == 0 {
ref.evenLock.RLock()
refItem = ref.even[referrer]
ref.evenLock.RUnlock()
if refItem != nil {
atomic.AddInt64(&refItem.Count, 1)
} else {
ref.evenLock.Lock()
ref.even[referrer] = &ReferrerItem{Count: 1}
ref.evenLock.Unlock()
}
} else {
ref.oddLock.RLock()
refItem = ref.odd[referrer]
ref.oddLock.RUnlock()
if refItem != nil {
atomic.AddInt64(&refItem.Count, 1)
} else {
ref.oddLock.Lock()
ref.odd[referrer] = &ReferrerItem{Count: 1}
ref.oddLock.Unlock()
}
}
}