53 lines
872 B
Go
53 lines
872 B
Go
package logfile
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"log"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
func TestBigFils(t *testing.T) {
|
|
file, err := os.Open("./querylog.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
json_logs := make([]*LogEntry, 0, 1000)
|
|
scn := bufio.NewScanner(file)
|
|
idx := 0
|
|
for scn.Scan() {
|
|
idx++
|
|
var entry LogEntry
|
|
err = json.Unmarshal(scn.Bytes(), &entry)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
json_logs = append(json_logs, &entry)
|
|
}
|
|
file.Close()
|
|
ls, err := New("./logdb/")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = ls
|
|
|
|
func() {
|
|
defer func() {
|
|
ls.db.Close()
|
|
ls.shapes.db.Close()
|
|
}()
|
|
for i, v := range json_logs {
|
|
if i%10000 == 0 {
|
|
log.Printf("%d / %d", i, len(json_logs))
|
|
}
|
|
err = ls.Store(v)
|
|
if err != nil {
|
|
log.Println(err)
|
|
continue
|
|
}
|
|
}
|
|
}()
|
|
|
|
}
|