diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..243b5dfc --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +nodejs 16.15.1 diff --git a/go.mod b/go.mod index d3c3e1de..29a04b60 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/AdguardTeam/AdGuardHome -go 1.17 +go 1.18 require ( github.com/AdguardTeam/dnsproxy v0.43.0 @@ -42,6 +42,7 @@ require ( github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect github.com/ameshkov/dnsstamps v1.0.3 // indirect github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 // indirect + github.com/bits-and-blooms/bloom v2.0.3+incompatible // indirect github.com/cheekybits/genny v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect @@ -56,8 +57,10 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/objx v0.1.1 // indirect github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 // indirect + github.com/willf/bitset v1.1.11 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index ae65fdab..5ec0df94 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/beefsack/go-rate v0.0.0-20200827232406-6cde80facd47/go.mod h1:6YNgTHL github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 h1:0b2vaepXIfMsG++IsjHiI2p4bxALD1Y2nQKGMR5zDQM= github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bits-and-blooms/bloom v2.0.3+incompatible h1:3ONZFjJoMyfHDil5iCcNkcPJ//PNNo+55RHvPrfUGnY= +github.com/bits-and-blooms/bloom v2.0.3+incompatible/go.mod h1:nEmPH2pqJb3sCXfd7cyDSKC4iPfCAt312JHgNrtnnDE= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= @@ -240,6 +242,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -262,6 +266,8 @@ github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 h1:hl6sK6aFgTLISijk6xIz github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/internal/storage/bolt/bolt.go b/internal/storage/bolt/bolt.go new file mode 100644 index 00000000..91d570a6 --- /dev/null +++ b/internal/storage/bolt/bolt.go @@ -0,0 +1,68 @@ +package bolt + +import ( + "bytes" + "encoding/gob" + + "go.etcd.io/bbolt" +) + +type DB struct { + bolt *bbolt.DB +} + +func Open(filename string) (o *DB, err error) { + o = &DB{} + o.bolt, err = bbolt.Open(filename, 0o644, nil) + return o, err +} + +func (d *DB) Close() (err error) { + if d.bolt != nil { + return d.bolt.Close() + } + return +} + +func (d *DB) Store(root []byte, key []byte, val interface{}) error { + return d.bolt.Update(func(t *bbolt.Tx) error { + bkt, err := t.CreateBucketIfNotExists(root) + if err != nil { + return err + } + var buf bytes.Buffer + err = gob.NewEncoder(&buf).Encode(val) + if err != nil { + return err + } + err = bkt.Put(key, buf.Bytes()) + if err != nil { + return err + } + return nil + }) +} + +func (d *DB) Load(root []byte, key []byte, val interface{}) error { + return d.bolt.View(func(t *bbolt.Tx) error { + bkt, err := t.CreateBucketIfNotExists(root) + if err != nil { + return err + } + err = gob.NewDecoder(bytes.NewBuffer(bkt.Get(key))).Decode(val) + if err != nil { + return err + } + return nil + }) +} + +func (d *DB) Delete(root []byte, key []byte) error { + return d.bolt.Update(func(t *bbolt.Tx) error { + bkt, err := t.CreateBucketIfNotExists(root) + if err != nil { + return err + } + return bkt.Delete(key) + }) +} diff --git a/internal/storage/logs/logfile.go b/internal/storage/logs/logfile.go new file mode 100644 index 00000000..67c448f6 --- /dev/null +++ b/internal/storage/logs/logfile.go @@ -0,0 +1,153 @@ +package logfile + +import ( + "bytes" + "net" + "os" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/AdGuardHome/internal/querylog" + "github.com/bits-and-blooms/bloom" +) + +type LogStorage struct { +} + +type LogBlock struct { +} + +type ShapeStorage struct { + blocks []ShapeBlock + + shapeCache map[string][8]byte +} + +func (s *ShapeStorage) FindOrAdd(k []byte) [8]byte { + if len(k) <= 8 { + o := [8]byte{} + for i, v := range k { + o[i] = v + } + return o + } + return [8]byte{} +} + +type ShapeBlock struct { + filter *bloom.BloomFilter + fp *os.File +} + +func (b *ShapeBlock) Open(name string) (err error) { + _, err = b.fp.Stat() + if os.IsNotExist(err) { + b.fp, err = os.Create(name) + if err != nil { + return err + } + } else if err == nil { + b.fp, err = os.OpenFile(name, os.O_RDWR, 0o644) + if err != nil { + return err + } + } else { + return err + } + return +} + +var magic = [8]byte{0xfe, 0xed, 0xbe, 0xef, 0x69, 0x00, 0x00, 0x00} + +const BLOCK_SIZE = 1600 +const BLOOM_SIZE_BITS = 500 * 64 +const BLOOM_SIZE_BYTES = BLOOM_SIZE_BITS / 8 +const BLOOM_SIZE_TOTAL = 8 + 8 + 8 + BLOOM_SIZE_BYTES + +func (b *ShapeBlock) WriteTo() (err error) { + if b.filter == nil { + b.filter = bloom.New(BLOOM_SIZE_BITS, 14) + } + _, err = b.fp.WriteAt(magic[:], 0) + bf := new(bytes.Buffer) + b.filter.WriteTo(bf) + n, err := b.fp.WriteAt(bf.Bytes(), 8) + if err != nil { + return err + } + if n != BLOOM_SIZE_TOTAL { + panic("invalid bloom header size") + } + return err +} + +func (b *ShapeBlock) ReadHeader() (err error) { + prefix := make([]byte, 8) + _, err = b.fp.ReadAt(prefix, 0) + if err != nil { + return err + } + bloom := make([]byte, BLOOM_SIZE_TOTAL) + _, err = b.fp.ReadAt(bloom, 8) + if err != nil { + return err + } + _, err = b.filter.ReadFrom(bytes.NewBuffer(bloom)) + if err != nil { + return err + } + return nil +} + +func (b *ShapeBlock) Add(k []byte) int { + if uint(len(b.keys)+1) >= b.filter.Cap() { + return -1 + } + if b.Find(k) == -1 { + b.filter.Add(k) + return len(b.keys) + } + return -1 +} + +func (b *ShapeBlock) Find(k []byte) [8]byte { + if !b.filter.Test(k) { + return -1 + } + for idx, v := range b.keys { + if len(k) != len(v) { + continue + } + for i := range v { + if v[i] != k[i] { + continue + } + } + return idx + } + return -1 +} + +type StorageEntry struct { + Block int64 +} + +// represents a single log entry in storage +type LogEntry struct { + IP net.IP `json:"IP"` // Client IP + Time time.Time `json:"T"` + + QHost string `json:"QH"` + QType string `json:"QT"` + QClass string `json:"QC"` + + ClientID string `json:"CID,omitempty"` + ClientProto querylog.ClientProto `json:"CP"` + + Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net + OrigAnswer []byte `json:",omitempty"` + + Result filtering.Result + Elapsed time.Duration + Upstream string `json:",omitempty"` // if empty, means it was cached +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 00000000..a2f21061 --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,38 @@ +package storage + +type DB interface { + Store(Datapoint) error + Load(root []byte, key []byte, val interface{}) error + Delete(root []byte, key []byte) error + Close() error +} + +type Datapoint interface { + Bucket() []byte + Key() []byte + Value() any +} + +func RawData[T any](root []byte, key []byte, val T) Datapoint { + return &rawData[T]{ + b: root, + k: key, + v: val, + } +} + +type rawData[T any] struct { + b []byte + k []byte + v T +} + +func (d *rawData[T]) Bucket() []byte { + return d.b +} +func (d *rawData[T]) Key() []byte { + return d.k +} +func (d *rawData[T]) Value() any { + return d.v +} diff --git a/internal/storage/table.go b/internal/storage/table.go new file mode 100644 index 00000000..17c87f0a --- /dev/null +++ b/internal/storage/table.go @@ -0,0 +1,8 @@ +package storage + +type Table struct { + columns map[string]Column +} + +type Column struct { +} diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..fb57ccd1 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,4 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +