nofv

2022-06-03 10:31:31 -05:00 · 2022-06-03 10:31:31 -05:00 · a7b868510f
parent f46c9f74d5
commit a7b868510f
8 changed files with 282 additions and 1 deletions
--- a/.tool-versions
+++ b/.tool-versions
@ -0,0 +1 @@
+nodejs 16.15.1
--- a/go.mod
+++ b/go.mod
@ -1,6 +1,6 @@
 module github.com/AdguardTeam/AdGuardHome

-go 1.17
+go 1.18

 require (
 	github.com/AdguardTeam/dnsproxy v0.43.0
@ -42,6 +42,7 @@ require (
 	github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect
 	github.com/ameshkov/dnsstamps v1.0.3 // indirect
 	github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 // indirect
+	github.com/bits-and-blooms/bloom v2.0.3+incompatible // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@ -56,8 +57,10 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 // indirect
+	github.com/willf/bitset v1.1.11 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/text v0.3.7 // indirect
--- a/go.sum
+++ b/go.sum
@ -40,6 +40,8 @@ github.com/beefsack/go-rate v0.0.0-20200827232406-6cde80facd47/go.mod h1:6YNgTHL
 github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 h1:0b2vaepXIfMsG++IsjHiI2p4bxALD1Y2nQKGMR5zDQM=
 github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/bits-and-blooms/bloom v2.0.3+incompatible h1:3ONZFjJoMyfHDil5iCcNkcPJ//PNNo+55RHvPrfUGnY=
+github.com/bits-and-blooms/bloom v2.0.3+incompatible/go.mod h1:nEmPH2pqJb3sCXfd7cyDSKC4iPfCAt312JHgNrtnnDE=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
@ -240,6 +242,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -262,6 +266,8 @@ github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 h1:hl6sK6aFgTLISijk6xIz
 github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
+github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
--- a/internal/storage/bolt/bolt.go
+++ b/internal/storage/bolt/bolt.go
@ -0,0 +1,68 @@
+package bolt
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"go.etcd.io/bbolt"
+)
+
+type DB struct {
+	bolt *bbolt.DB
+}
+
+func Open(filename string) (o *DB, err error) {
+	o = &DB{}
+	o.bolt, err = bbolt.Open(filename, 0o644, nil)
+	return o, err
+}
+
+func (d *DB) Close() (err error) {
+	if d.bolt != nil {
+		return d.bolt.Close()
+	}
+	return
+}
+
+func (d *DB) Store(root []byte, key []byte, val interface{}) error {
+	return d.bolt.Update(func(t *bbolt.Tx) error {
+		bkt, err := t.CreateBucketIfNotExists(root)
+		if err != nil {
+			return err
+		}
+		var buf bytes.Buffer
+		err = gob.NewEncoder(&buf).Encode(val)
+		if err != nil {
+			return err
+		}
+		err = bkt.Put(key, buf.Bytes())
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func (d *DB) Load(root []byte, key []byte, val interface{}) error {
+	return d.bolt.View(func(t *bbolt.Tx) error {
+		bkt, err := t.CreateBucketIfNotExists(root)
+		if err != nil {
+			return err
+		}
+		err = gob.NewDecoder(bytes.NewBuffer(bkt.Get(key))).Decode(val)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func (d *DB) Delete(root []byte, key []byte) error {
+	return d.bolt.Update(func(t *bbolt.Tx) error {
+		bkt, err := t.CreateBucketIfNotExists(root)
+		if err != nil {
+			return err
+		}
+		return bkt.Delete(key)
+	})
+}
--- a/internal/storage/logs/logfile.go
+++ b/internal/storage/logs/logfile.go
@ -0,0 +1,153 @@
+package logfile
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
+	"github.com/AdguardTeam/AdGuardHome/internal/querylog"
+	"github.com/bits-and-blooms/bloom"
+)
+
+type LogStorage struct {
+}
+
+type LogBlock struct {
+}
+
+type ShapeStorage struct {
+	blocks []ShapeBlock
+
+	shapeCache map[string][8]byte
+}
+
+func (s *ShapeStorage) FindOrAdd(k []byte) [8]byte {
+	if len(k) <= 8 {
+		o := [8]byte{}
+		for i, v := range k {
+			o[i] = v
+		}
+		return o
+	}
+	return [8]byte{}
+}
+
+type ShapeBlock struct {
+	filter *bloom.BloomFilter
+	fp     *os.File
+}
+
+func (b *ShapeBlock) Open(name string) (err error) {
+	_, err = b.fp.Stat()
+	if os.IsNotExist(err) {
+		b.fp, err = os.Create(name)
+		if err != nil {
+			return err
+		}
+	} else if err == nil {
+		b.fp, err = os.OpenFile(name, os.O_RDWR, 0o644)
+		if err != nil {
+			return err
+		}
+	} else {
+		return err
+	}
+	return
+}
+
+var magic = [8]byte{0xfe, 0xed, 0xbe, 0xef, 0x69, 0x00, 0x00, 0x00}
+
+const BLOCK_SIZE = 1600
+const BLOOM_SIZE_BITS = 500 * 64
+const BLOOM_SIZE_BYTES = BLOOM_SIZE_BITS / 8
+const BLOOM_SIZE_TOTAL = 8 + 8 + 8 + BLOOM_SIZE_BYTES
+
+func (b *ShapeBlock) WriteTo() (err error) {
+	if b.filter == nil {
+		b.filter = bloom.New(BLOOM_SIZE_BITS, 14)
+	}
+	_, err = b.fp.WriteAt(magic[:], 0)
+	bf := new(bytes.Buffer)
+	b.filter.WriteTo(bf)
+	n, err := b.fp.WriteAt(bf.Bytes(), 8)
+	if err != nil {
+		return err
+	}
+	if n != BLOOM_SIZE_TOTAL {
+		panic("invalid bloom header size")
+	}
+	return err
+}
+
+func (b *ShapeBlock) ReadHeader() (err error) {
+	prefix := make([]byte, 8)
+	_, err = b.fp.ReadAt(prefix, 0)
+	if err != nil {
+		return err
+	}
+	bloom := make([]byte, BLOOM_SIZE_TOTAL)
+	_, err = b.fp.ReadAt(bloom, 8)
+	if err != nil {
+		return err
+	}
+	_, err = b.filter.ReadFrom(bytes.NewBuffer(bloom))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *ShapeBlock) Add(k []byte) int {
+	if uint(len(b.keys)+1) >= b.filter.Cap() {
+		return -1
+	}
+	if b.Find(k) == -1 {
+		b.filter.Add(k)
+		return len(b.keys)
+	}
+	return -1
+}
+
+func (b *ShapeBlock) Find(k []byte) [8]byte {
+	if !b.filter.Test(k) {
+		return -1
+	}
+	for idx, v := range b.keys {
+		if len(k) != len(v) {
+			continue
+		}
+		for i := range v {
+			if v[i] != k[i] {
+				continue
+			}
+		}
+		return idx
+	}
+	return -1
+}
+
+type StorageEntry struct {
+	Block int64
+}
+
+// represents a single log entry in storage
+type LogEntry struct {
+	IP   net.IP    `json:"IP"` // Client IP
+	Time time.Time `json:"T"`
+
+	QHost  string `json:"QH"`
+	QType  string `json:"QT"`
+	QClass string `json:"QC"`
+
+	ClientID    string               `json:"CID,omitempty"`
+	ClientProto querylog.ClientProto `json:"CP"`
+
+	Answer     []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net
+	OrigAnswer []byte `json:",omitempty"`
+
+	Result   filtering.Result
+	Elapsed  time.Duration
+	Upstream string `json:",omitempty"` // if empty, means it was cached
+}
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@ -0,0 +1,38 @@
+package storage
+
+type DB interface {
+	Store(Datapoint) error
+	Load(root []byte, key []byte, val interface{}) error
+	Delete(root []byte, key []byte) error
+	Close() error
+}
+
+type Datapoint interface {
+	Bucket() []byte
+	Key() []byte
+	Value() any
+}
+
+func RawData[T any](root []byte, key []byte, val T) Datapoint {
+	return &rawData[T]{
+		b: root,
+		k: key,
+		v: val,
+	}
+}
+
+type rawData[T any] struct {
+	b []byte
+	k []byte
+	v T
+}
+
+func (d *rawData[T]) Bucket() []byte {
+	return d.b
+}
+func (d *rawData[T]) Key() []byte {
+	return d.k
+}
+func (d *rawData[T]) Value() any {
+	return d.v
+}
--- a/internal/storage/table.go
+++ b/internal/storage/table.go
@ -0,0 +1,8 @@
+package storage
+
+type Table struct {
+	columns map[string]Column
+}
+
+type Column struct {
+}
--- a/yarn.lock
+++ b/yarn.lock
@ -0,0 +1,4 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+