Merge pull request #64 in DNS/adguard-dns from remove_old_stats to master

* commit '903b20dcab855f0dec7af3b305e8362fa4255b8a':
  Remove dead code

app.go

@@ -6,10 +6,8 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"strconv"
-	"time"
 
 	"github.com/gobuffalo/packr"
 )
@@ -17,12 +15,7 @@ import (
 // VersionString will be set through ldflags, contains current version
 var VersionString = "undefined"
 
-func cleanup() {
-	writeStats()
-}
-
 func main() {
-	c := make(chan os.Signal, 1)
 	log.Printf("AdGuard DNS web interface backend, version %s\n", VersionString)
 	box := packr.NewBox("build/static")
 	{
@@ -121,31 +114,8 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = loadStats()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	signal.Notify(c, os.Interrupt)
-	go func() {
-		<-c
-		cleanup()
-		os.Exit(1)
-	}()
-
-	go func() {
-		for range time.Tick(time.Hour * 24) {
-			err := writeStats()
-			if err != nil {
-				log.Printf("Couldn't write stats: %s", err)
-				// try later on next iteration, don't abort
-			}
-		}
-	}()
-
 	address := net.JoinHostPort(config.BindHost, strconv.Itoa(config.BindPort))
 
-	runStatsCollectors()
 	runFilterRefreshers()
 
 	http.Handle("/", optionalAuthHandler(http.FileServer(box)))

control.go

@@ -35,6 +35,10 @@ var versionCheckLastTime time.Time
 const versionCheckURL = "https://adguardteam.github.io/AdguardDNS/version.json"
 const versionCheckPeriod = time.Hour * 8
 
+var client = &http.Client{
+	Timeout: time.Second * 30,
+}
+
 // -------------------
 // coredns run control
 // -------------------
@@ -360,13 +364,36 @@ func handleQueryLogDisable(w http.ResponseWriter, r *http.Request) {
 }
 
 func handleStatsReset(w http.ResponseWriter, r *http.Request) {
-	purgeStats()
+	resp, err := client.Post("http://127.0.0.1:8618/stats_reset", "text/plain", nil)
-
-	_, err := fmt.Fprintf(w, "OK\n")
 	if err != nil {
-		httpError(w, http.StatusInternalServerError, "Couldn't write body: %s", err)
+		errortext := fmt.Sprintf("Couldn't get stats_top from coredns: %T %s\n", err, err)
+		log.Println(errortext)
+		http.Error(w, errortext, http.StatusBadGateway)
 		return
 	}
+	if resp != nil && resp.Body != nil {
+		defer resp.Body.Close()
+	}
+
+	// read the body entirely
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		errortext := fmt.Sprintf("Couldn't read response body: %s", err)
+		log.Println(errortext)
+		http.Error(w, errortext, http.StatusBadGateway)
+		return
+	}
+
+	// forward body entirely with status code
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+	w.WriteHeader(resp.StatusCode)
+	_, err = w.Write(body)
+	if err != nil {
+		errortext := fmt.Sprintf("Couldn't write body: %s", err)
+		log.Println(errortext)
+		http.Error(w, errortext, http.StatusInternalServerError)
+	}
 }
 
 func handleStatsTop(w http.ResponseWriter, r *http.Request) {

helpers.go

@@ -91,46 +91,6 @@ func optionalAuthHandler(handler http.Handler) http.Handler {
 	return &authHandler{handler}
 }
 
-// --------------------------
-// helper functions for stats
-// --------------------------
-func getReversedSlice(input [statsHistoryElements]float64, start int, end int) []float64 {
-	output := make([]float64, 0)
-	for i := start; i <= end; i++ {
-		output = append([]float64{input[i]}, output...)
-	}
-	return output
-}
-
-func generateMapFromStats(stats *periodicStats, start int, end int) map[string]interface{} {
-	// clamp
-	start = clamp(start, 0, statsHistoryElements)
-	end = clamp(end, 0, statsHistoryElements)
-
-	avgProcessingTime := make([]float64, 0)
-
-	count := getReversedSlice(stats.Entries[processingTimeCount], start, end)
-	sum := getReversedSlice(stats.Entries[processingTimeSum], start, end)
-	for i := 0; i < len(count); i++ {
-		var avg float64
-		if count[i] != 0 {
-			avg = sum[i] / count[i]
-			avg *= 1000
-		}
-		avgProcessingTime = append(avgProcessingTime, avg)
-	}
-
-	result := map[string]interface{}{
-		"dns_queries":           getReversedSlice(stats.Entries[totalRequests], start, end),
-		"blocked_filtering":     getReversedSlice(stats.Entries[filteredTotal], start, end),
-		"replaced_safebrowsing": getReversedSlice(stats.Entries[filteredSafebrowsing], start, end),
-		"replaced_safesearch":   getReversedSlice(stats.Entries[filteredSafesearch], start, end),
-		"replaced_parental":     getReversedSlice(stats.Entries[filteredParental], start, end),
-		"avg_processing_time":   avgProcessingTime,
-	}
-	return result
-}
-
 // -------------------------------------------------
 // helper functions for parsing parameters from body
 // -------------------------------------------------

stats.go

@@ -1,277 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"io/ioutil"
-	"log"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-)
-
-var client = &http.Client{
-	Timeout: time.Second * 30,
-}
-
-// as seen over HTTP
-type statsEntry map[string]float64
-type statsEntries map[string][statsHistoryElements]float64
-
-const (
-	statsHistoryElements = 60 + 1 // +1 for calculating delta
-	totalRequests        = `coredns_dns_request_count_total`
-	filteredTotal        = `coredns_dnsfilter_filtered_total`
-	filteredSafebrowsing = `coredns_dnsfilter_filtered_safebrowsing_total`
-	filteredSafesearch   = `coredns_dnsfilter_safesearch_total`
-	filteredParental     = `coredns_dnsfilter_filtered_parental_total`
-	processingTimeSum    = `coredns_dns_request_duration_seconds_sum`
-	processingTimeCount  = `coredns_dns_request_duration_seconds_count`
-)
-
-var entryWhiteList = map[string]bool{
-	totalRequests:        true,
-	filteredTotal:        true,
-	filteredSafebrowsing: true,
-	filteredSafesearch:   true,
-	filteredParental:     true,
-	processingTimeSum:    true,
-	processingTimeCount:  true,
-}
-
-type periodicStats struct {
-	Entries    statsEntries
-	LastRotate time.Time // last time this data was rotated
-}
-
-type stats struct {
-	PerSecond periodicStats
-	PerMinute periodicStats
-	PerHour   periodicStats
-	PerDay    periodicStats
-
-	LastSeen statsEntry
-	sync.RWMutex
-}
-
-var statistics stats
-
-func initPeriodicStats(periodic *periodicStats) {
-	periodic.Entries = statsEntries{}
-	periodic.LastRotate = time.Time{}
-}
-
-func init() {
-	purgeStats()
-}
-
-func purgeStats() {
-	statistics.Lock()
-	initPeriodicStats(&statistics.PerSecond)
-	initPeriodicStats(&statistics.PerMinute)
-	initPeriodicStats(&statistics.PerHour)
-	initPeriodicStats(&statistics.PerDay)
-	statistics.Unlock()
-}
-
-func runStatsCollectors() {
-	go statsCollector(time.Second)
-}
-
-func statsCollector(t time.Duration) {
-	for range time.Tick(t) {
-		collectStats()
-	}
-}
-
-func isConnRefused(err error) bool {
-	if err != nil {
-		if uerr, ok := err.(*url.Error); ok {
-			if noerr, ok := uerr.Err.(*net.OpError); ok {
-				if scerr, ok := noerr.Err.(*os.SyscallError); ok {
-					if scerr.Err == syscall.ECONNREFUSED {
-						return true
-					}
-				}
-			}
-		}
-	}
-	return false
-}
-
-func statsRotate(periodic *periodicStats, now time.Time, rotations int64) {
-	if rotations > statsHistoryElements {
-		rotations = statsHistoryElements
-	}
-	// calculate how many times we should rotate
-	for r := int64(0); r < rotations; r++ {
-		for key, values := range periodic.Entries {
-			newValues := [statsHistoryElements]float64{}
-			for i := 1; i < len(values); i++ {
-				newValues[i] = values[i-1]
-			}
-			periodic.Entries[key] = newValues
-		}
-	}
-	if rotations > 0 {
-		periodic.LastRotate = now
-	}
-}
-
-// called every second, accumulates stats for each second, minute, hour and day
-func collectStats() {
-	now := time.Now()
-	statistics.Lock()
-	statsRotate(&statistics.PerSecond, now, int64(now.Sub(statistics.PerSecond.LastRotate)/time.Second))
-	statsRotate(&statistics.PerMinute, now, int64(now.Sub(statistics.PerMinute.LastRotate)/time.Minute))
-	statsRotate(&statistics.PerHour, now, int64(now.Sub(statistics.PerHour.LastRotate)/time.Hour))
-	statsRotate(&statistics.PerDay, now, int64(now.Sub(statistics.PerDay.LastRotate)/time.Hour/24))
-	statistics.Unlock()
-
-	// grab HTTP from prometheus
-	resp, err := client.Get("http://127.0.0.1:9153/metrics")
-	if resp != nil && resp.Body != nil {
-		defer resp.Body.Close()
-	}
-	if err != nil {
-		if isConnRefused(err) {
-			return
-		}
-		log.Printf("Couldn't get coredns metrics: %T %s\n", err, err)
-		return
-	}
-
-	// read the body entirely
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("Couldn't read response body:", err)
-		return
-	}
-
-	entry := statsEntry{}
-
-	// handle body
-	scanner := bufio.NewScanner(strings.NewReader(string(body)))
-	for scanner.Scan() {
-		line := scanner.Text()
-		// ignore comments
-		if line[0] == '#' {
-			continue
-		}
-		splitted := strings.Split(line, " ")
-		if len(splitted) < 2 {
-			continue
-		}
-
-		value, err := strconv.ParseFloat(splitted[1], 64)
-		if err != nil {
-			log.Printf("Failed to parse number input %s: %s", splitted[1], err)
-			continue
-		}
-
-		key := splitted[0]
-		index := strings.IndexByte(key, '{')
-		if index >= 0 {
-			key = key[:index]
-		}
-
-		// empty keys are not ok
-		if key == "" {
-			continue
-		}
-
-		// keys not in whitelist are not ok
-		if entryWhiteList[key] == false {
-			continue
-		}
-
-		got, ok := entry[key]
-		if ok {
-			value += got
-		}
-		entry[key] = value
-	}
-
-	// calculate delta
-	statistics.Lock()
-	delta := calcDelta(entry, statistics.LastSeen)
-
-	// apply delta to second/minute/hour/day
-	applyDelta(&statistics.PerSecond, delta)
-	applyDelta(&statistics.PerMinute, delta)
-	applyDelta(&statistics.PerHour, delta)
-	applyDelta(&statistics.PerDay, delta)
-
-	// save last seen
-	statistics.LastSeen = entry
-	statistics.Unlock()
-}
-
-func calcDelta(current, seen statsEntry) statsEntry {
-	delta := statsEntry{}
-	for key, currentValue := range current {
-		seenValue := seen[key]
-		deltaValue := currentValue - seenValue
-		delta[key] = deltaValue
-	}
-	return delta
-}
-
-func applyDelta(current *periodicStats, delta statsEntry) {
-	for key, deltaValue := range delta {
-		currentValues := current.Entries[key]
-		currentValues[0] += deltaValue
-		current.Entries[key] = currentValues
-	}
-}
-
-func loadStats() error {
-	statsFile := filepath.Join(config.ourBinaryDir, "stats.json")
-	if _, err := os.Stat(statsFile); os.IsNotExist(err) {
-		log.Printf("Stats JSON does not exist, skipping: %s", statsFile)
-		return nil
-	}
-	log.Printf("Loading JSON stats: %s", statsFile)
-	jsonText, err := ioutil.ReadFile(statsFile)
-	if err != nil {
-		log.Printf("Couldn't read JSON stats: %s", err)
-		return err
-	}
-	err = json.Unmarshal(jsonText, &statistics)
-	if err != nil {
-		log.Printf("Couldn't parse JSON stats: %s", err)
-		return err
-	}
-
-	return nil
-}
-
-func writeStats() error {
-	statsFile := filepath.Join(config.ourBinaryDir, "stats.json")
-	log.Printf("Writing JSON file: %s", statsFile)
-	statistics.RLock()
-	json, err := json.MarshalIndent(&statistics, "", "  ")
-	statistics.RUnlock()
-	if err != nil {
-		log.Printf("Couldn't generate JSON: %s", err)
-		return err
-	}
-	err = ioutil.WriteFile(statsFile+".tmp", json, 0644)
-	if err != nil {
-		log.Printf("Couldn't write stats in JSON: %s", err)
-		return err
-	}
-	err = os.Rename(statsFile+".tmp", statsFile)
-	if err != nil {
-		log.Printf("Couldn't rename stats JSON: %s", err)
-		return err
-	}
-	return nil
-}