agola/internal/datamanager/changes.go

355 lines
8.9 KiB
Go
Raw Normal View History

2019-02-22 07:45:59 +00:00
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package datamanager
2019-02-22 07:45:59 +00:00
import (
"context"
"encoding/json"
"fmt"
"io"
"path"
"strings"
"sync"
"time"
2019-07-01 09:40:20 +00:00
"agola.io/agola/internal/etcd"
2019-02-22 07:45:59 +00:00
etcdclientv3rpc "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
"go.etcd.io/etcd/mvcc/mvccpb"
errors "golang.org/x/xerrors"
2019-02-22 07:45:59 +00:00
)
// TODO(sgotti) rewrite this to use a sqlite local cache
2019-02-22 07:45:59 +00:00
type WalChanges struct {
actions map[string][]*Action
puts map[string]map[string]string // map[dataType]map[id]
deletes map[string]map[string]string
2019-02-22 07:45:59 +00:00
walSeq string
revision int64
changeGroupsRevisions changeGroupsRevisions
initialized bool
sync.Mutex
}
func NewWalChanges(dataTypes []string) *WalChanges {
changes := &WalChanges{
2019-02-22 07:45:59 +00:00
actions: make(map[string][]*Action),
puts: make(map[string]map[string]string),
deletes: make(map[string]map[string]string),
2019-02-22 07:45:59 +00:00
changeGroupsRevisions: make(changeGroupsRevisions),
}
for _, dataType := range dataTypes {
changes.puts[dataType] = make(map[string]string)
changes.deletes[dataType] = make(map[string]string)
}
return changes
2019-02-22 07:45:59 +00:00
}
func (c *WalChanges) String() string {
return fmt.Sprintf("puts: %s, deletes: %s, walSeq: %s, revision: %d, initialized: %t", c.puts, c.deletes, c.walSeq, c.revision, c.initialized)
}
func (c *WalChanges) curRevision() int64 {
return c.revision
}
func (c *WalChanges) putRevision(revision int64) {
c.revision = revision
}
func (c *WalChanges) getPut(dataType, id string) (string, bool) {
walseq, ok := c.puts[dataType][id]
2019-02-22 07:45:59 +00:00
return walseq, ok
}
func (c *WalChanges) addPut(dataType, id, walseq string, revision int64) {
delete(c.deletes[dataType], id)
c.puts[dataType][id] = walseq
2019-02-22 07:45:59 +00:00
c.walSeq = walseq
c.revision = revision
}
func (c *WalChanges) removePut(dataType, id string, revision int64) {
delete(c.puts[dataType], id)
2019-02-22 07:45:59 +00:00
c.revision = revision
}
func (c *WalChanges) addDelete(dataType, id, walseq string, revision int64) {
delete(c.puts[dataType], id)
c.deletes[dataType][id] = walseq
2019-02-22 07:45:59 +00:00
c.walSeq = walseq
c.revision = revision
}
func (c *WalChanges) removeDelete(dataType, id string, revision int64) {
delete(c.deletes[dataType], id)
2019-02-22 07:45:59 +00:00
c.revision = revision
}
func (c *WalChanges) getChangeGroups(cgNames []string) changeGroupsRevisions {
cgr := map[string]int64{}
for _, cgName := range cgNames {
if rev, ok := c.changeGroupsRevisions[cgName]; ok {
cgr[cgName] = rev
} else {
// for non existing changegroups use a changegroup with revision = 0
cgr[cgName] = 0
}
}
return cgr
}
func (c *WalChanges) putChangeGroup(cgName string, cgRev int64) {
c.changeGroupsRevisions[cgName] = cgRev
}
func (c *WalChanges) removeChangeGroup(cgName string) {
delete(c.changeGroupsRevisions, cgName)
}
func (d *DataManager) applyWalChanges(ctx context.Context, walData *WalData, revision int64) error {
walDataFilePath := d.storageWalDataFile(walData.WalDataFileID)
2019-02-22 07:45:59 +00:00
walDataFile, err := d.ost.ReadObject(walDataFilePath)
2019-02-22 07:45:59 +00:00
if err != nil {
return errors.Errorf("failed to read waldata %q: %w", walDataFilePath, err)
2019-02-22 07:45:59 +00:00
}
defer walDataFile.Close()
dec := json.NewDecoder(walDataFile)
d.changes.Lock()
defer d.changes.Unlock()
2019-02-22 07:45:59 +00:00
for {
var action *Action
err := dec.Decode(&action)
if err == io.EOF {
// all done
break
}
if err != nil {
return errors.Errorf("failed to decode wal file: %w", err)
2019-02-22 07:45:59 +00:00
}
d.applyWalChangesAction(ctx, action, walData.WalSequence, revision)
2019-02-22 07:45:59 +00:00
}
return nil
}
func (d *DataManager) applyWalChangesAction(ctx context.Context, action *Action, walSequence string, revision int64) {
2019-02-22 07:45:59 +00:00
switch action.ActionType {
case ActionTypePut:
d.changes.addPut(action.DataType, action.ID, walSequence, revision)
2019-02-22 07:45:59 +00:00
case ActionTypeDelete:
d.changes.addDelete(action.DataType, action.ID, walSequence, revision)
2019-02-22 07:45:59 +00:00
}
if d.changes.actions[walSequence] == nil {
d.changes.actions[walSequence] = []*Action{}
2019-02-22 07:45:59 +00:00
}
d.changes.actions[walSequence] = append(d.changes.actions[walSequence], action)
2019-02-22 07:45:59 +00:00
}
func (d *DataManager) watcherLoop(ctx context.Context) {
2019-02-22 07:45:59 +00:00
for {
initialized := d.changes.initialized
2019-02-22 07:45:59 +00:00
if !initialized {
if err := d.initializeChanges(ctx); err != nil {
d.log.Errorf("watcher err: %+v", err)
2019-02-22 07:45:59 +00:00
}
} else {
if err := d.watcher(ctx); err != nil {
d.log.Errorf("watcher err: %+v", err)
2019-02-22 07:45:59 +00:00
}
}
select {
case <-ctx.Done():
d.log.Infof("watcher exiting")
return
2019-02-22 07:45:59 +00:00
default:
}
time.Sleep(1 * time.Second)
}
}
func (d *DataManager) initializeChanges(ctx context.Context) error {
2019-02-22 07:45:59 +00:00
var revision int64
var continuation *etcd.ListPagedContinuation
for {
listResp, err := d.e.ListPaged(ctx, etcdWalsDir+"/", 0, 10, continuation)
2019-02-22 07:45:59 +00:00
if err != nil {
return err
}
resp := listResp.Resp
continuation = listResp.Continuation
revision = resp.Header.Revision
for _, kv := range resp.Kvs {
var walData *WalData
if err := json.Unmarshal(kv.Value, &walData); err != nil {
return err
}
if err := d.applyWalChanges(ctx, walData, revision); err != nil {
2019-02-22 07:45:59 +00:00
return err
}
}
if !listResp.HasMore {
break
}
}
continuation = nil
// use the same revision
for {
listResp, err := d.e.ListPaged(ctx, etcdChangeGroupsDir+"/", 0, 10, continuation)
2019-02-22 07:45:59 +00:00
if err != nil {
return err
}
resp := listResp.Resp
continuation = listResp.Continuation
for _, kv := range resp.Kvs {
d.changes.Lock()
2019-02-22 07:45:59 +00:00
changeGroup := path.Base(string(kv.Key))
d.changes.putChangeGroup(changeGroup, kv.ModRevision)
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
}
if !listResp.HasMore {
break
}
}
d.changes.Lock()
d.changes.revision = revision
d.changes.initialized = true
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
return nil
}
func (d *DataManager) watcher(ctx context.Context) error {
d.changes.Lock()
revision := d.changes.curRevision()
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
wctx, cancel := context.WithCancel(ctx)
defer cancel()
wch := d.e.Watch(wctx, etcdWalBaseDir+"/", revision+1)
2019-02-22 07:45:59 +00:00
for wresp := range wch {
if wresp.Canceled {
err := wresp.Err()
if err == etcdclientv3rpc.ErrCompacted {
d.log.Errorf("required events already compacted, reinitializing watcher changes")
d.changes.Lock()
d.changes.initialized = false
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
}
return errors.Errorf("watch error: %w", err)
2019-02-22 07:45:59 +00:00
}
revision := wresp.Header.Revision
for _, ev := range wresp.Events {
key := string(ev.Kv.Key)
switch {
case strings.HasPrefix(key, etcdWalsDir+"/"):
switch ev.Type {
case mvccpb.PUT:
var walData *WalData
if err := json.Unmarshal(ev.Kv.Value, &walData); err != nil {
return err
}
if walData.WalStatus != WalStatusCommitted {
continue
}
if err := d.applyWalChanges(ctx, walData, revision); err != nil {
2019-02-22 07:45:59 +00:00
return err
}
case mvccpb.DELETE:
walseq := path.Base(string(key))
d.changes.Lock()
putsToDelete := map[string][]string{}
deletesToDelete := map[string][]string{}
for _, dataType := range d.dataTypes {
putsToDelete[dataType] = []string{}
deletesToDelete[dataType] = []string{}
}
for _, dataType := range d.dataTypes {
for p, pwalseq := range d.changes.puts[dataType] {
if pwalseq == walseq {
putsToDelete[dataType] = append(putsToDelete[dataType], p)
}
2019-02-22 07:45:59 +00:00
}
}
for _, dataType := range d.dataTypes {
for id, pwalseq := range d.changes.deletes[dataType] {
if pwalseq == walseq {
deletesToDelete[dataType] = append(deletesToDelete[dataType], id)
}
2019-02-22 07:45:59 +00:00
}
}
for dataType, ids := range putsToDelete {
for _, id := range ids {
d.changes.removePut(dataType, id, revision)
}
2019-02-22 07:45:59 +00:00
}
for dataType, ids := range putsToDelete {
for _, id := range ids {
d.changes.removeDelete(dataType, id, revision)
}
2019-02-22 07:45:59 +00:00
}
delete(d.changes.actions, walseq)
2019-02-22 07:45:59 +00:00
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
}
case strings.HasPrefix(key, etcdChangeGroupsDir+"/"):
switch ev.Type {
case mvccpb.PUT:
d.changes.Lock()
changeGroup := strings.TrimPrefix(string(ev.Kv.Key), etcdChangeGroupsDir+"/")
d.changes.putChangeGroup(changeGroup, ev.Kv.ModRevision)
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
case mvccpb.DELETE:
d.changes.Lock()
changeGroup := strings.TrimPrefix(string(ev.Kv.Key), etcdChangeGroupsDir+"/")
d.changes.removeChangeGroup(changeGroup)
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
}
case key == etcdPingKey:
d.changes.Lock()
d.changes.putRevision(wresp.Header.Revision)
d.changes.Unlock()
2019-02-22 07:45:59 +00:00
}
}
}
return nil
}