2019-02-22 07:45:59 +00:00
|
|
|
// Copyright 2019 Sorint.lab
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
package datamanager
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"path"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2019-07-01 09:40:20 +00:00
|
|
|
"agola.io/agola/internal/etcd"
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
etcdclientv3rpc "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
|
|
|
|
"go.etcd.io/etcd/mvcc/mvccpb"
|
2019-05-23 09:23:14 +00:00
|
|
|
errors "golang.org/x/xerrors"
|
2019-02-22 07:45:59 +00:00
|
|
|
)
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
// TODO(sgotti) rewrite this to use a sqlite local cache
|
|
|
|
|
2019-02-22 07:45:59 +00:00
|
|
|
type WalChanges struct {
|
|
|
|
actions map[string][]*Action
|
2019-04-26 14:00:03 +00:00
|
|
|
puts map[string]map[string]string // map[dataType]map[id]
|
|
|
|
deletes map[string]map[string]string
|
2019-02-22 07:45:59 +00:00
|
|
|
walSeq string
|
|
|
|
revision int64
|
|
|
|
changeGroupsRevisions changeGroupsRevisions
|
|
|
|
initialized bool
|
|
|
|
sync.Mutex
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func NewWalChanges(dataTypes []string) *WalChanges {
|
|
|
|
changes := &WalChanges{
|
2019-02-22 07:45:59 +00:00
|
|
|
actions: make(map[string][]*Action),
|
2019-04-26 14:00:03 +00:00
|
|
|
puts: make(map[string]map[string]string),
|
|
|
|
deletes: make(map[string]map[string]string),
|
2019-02-22 07:45:59 +00:00
|
|
|
changeGroupsRevisions: make(changeGroupsRevisions),
|
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
|
|
|
|
for _, dataType := range dataTypes {
|
|
|
|
changes.puts[dataType] = make(map[string]string)
|
|
|
|
changes.deletes[dataType] = make(map[string]string)
|
|
|
|
}
|
|
|
|
|
|
|
|
return changes
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) String() string {
|
|
|
|
return fmt.Sprintf("puts: %s, deletes: %s, walSeq: %s, revision: %d, initialized: %t", c.puts, c.deletes, c.walSeq, c.revision, c.initialized)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) curRevision() int64 {
|
|
|
|
return c.revision
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) putRevision(revision int64) {
|
|
|
|
c.revision = revision
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (c *WalChanges) getPut(dataType, id string) (string, bool) {
|
|
|
|
walseq, ok := c.puts[dataType][id]
|
2019-02-22 07:45:59 +00:00
|
|
|
return walseq, ok
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (c *WalChanges) addPut(dataType, id, walseq string, revision int64) {
|
|
|
|
delete(c.deletes[dataType], id)
|
|
|
|
c.puts[dataType][id] = walseq
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
c.walSeq = walseq
|
|
|
|
c.revision = revision
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (c *WalChanges) removePut(dataType, id string, revision int64) {
|
|
|
|
delete(c.puts[dataType], id)
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
c.revision = revision
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (c *WalChanges) addDelete(dataType, id, walseq string, revision int64) {
|
|
|
|
delete(c.puts[dataType], id)
|
|
|
|
c.deletes[dataType][id] = walseq
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
c.walSeq = walseq
|
|
|
|
c.revision = revision
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (c *WalChanges) removeDelete(dataType, id string, revision int64) {
|
|
|
|
delete(c.deletes[dataType], id)
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
c.revision = revision
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) getChangeGroups(cgNames []string) changeGroupsRevisions {
|
|
|
|
cgr := map[string]int64{}
|
|
|
|
for _, cgName := range cgNames {
|
|
|
|
if rev, ok := c.changeGroupsRevisions[cgName]; ok {
|
|
|
|
cgr[cgName] = rev
|
|
|
|
} else {
|
|
|
|
// for non existing changegroups use a changegroup with revision = 0
|
|
|
|
cgr[cgName] = 0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return cgr
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) putChangeGroup(cgName string, cgRev int64) {
|
|
|
|
c.changeGroupsRevisions[cgName] = cgRev
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *WalChanges) removeChangeGroup(cgName string) {
|
|
|
|
delete(c.changeGroupsRevisions, cgName)
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (d *DataManager) applyWalChanges(ctx context.Context, walData *WalData, revision int64) error {
|
|
|
|
walDataFilePath := d.storageWalDataFile(walData.WalDataFileID)
|
2019-02-22 07:45:59 +00:00
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
walDataFile, err := d.ost.ReadObject(walDataFilePath)
|
2019-02-22 07:45:59 +00:00
|
|
|
if err != nil {
|
2019-05-23 09:23:14 +00:00
|
|
|
return errors.Errorf("failed to read waldata %q: %w", walDataFilePath, err)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
defer walDataFile.Close()
|
|
|
|
dec := json.NewDecoder(walDataFile)
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
|
|
|
defer d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
for {
|
|
|
|
var action *Action
|
|
|
|
|
|
|
|
err := dec.Decode(&action)
|
2022-02-24 11:56:52 +00:00
|
|
|
if errors.Is(err, io.EOF) {
|
2019-02-22 07:45:59 +00:00
|
|
|
// all done
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if err != nil {
|
2019-05-23 09:23:14 +00:00
|
|
|
return errors.Errorf("failed to decode wal file: %w", err)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
d.applyWalChangesAction(ctx, action, walData.WalSequence, revision)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (d *DataManager) applyWalChangesAction(ctx context.Context, action *Action, walSequence string, revision int64) {
|
2019-02-22 07:45:59 +00:00
|
|
|
switch action.ActionType {
|
|
|
|
case ActionTypePut:
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.addPut(action.DataType, action.ID, walSequence, revision)
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
case ActionTypeDelete:
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.addDelete(action.DataType, action.ID, walSequence, revision)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
if d.changes.actions[walSequence] == nil {
|
|
|
|
d.changes.actions[walSequence] = []*Action{}
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.actions[walSequence] = append(d.changes.actions[walSequence], action)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
2019-07-02 12:27:51 +00:00
|
|
|
func (d *DataManager) watcherLoop(ctx context.Context) {
|
2019-02-22 07:45:59 +00:00
|
|
|
for {
|
2019-04-26 14:00:03 +00:00
|
|
|
initialized := d.changes.initialized
|
2019-02-22 07:45:59 +00:00
|
|
|
if !initialized {
|
2019-04-26 14:00:03 +00:00
|
|
|
if err := d.initializeChanges(ctx); err != nil {
|
|
|
|
d.log.Errorf("watcher err: %+v", err)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
} else {
|
2019-04-26 14:00:03 +00:00
|
|
|
if err := d.watcher(ctx); err != nil {
|
|
|
|
d.log.Errorf("watcher err: %+v", err)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-25 13:53:26 +00:00
|
|
|
sleepCh := time.NewTimer(1 * time.Second).C
|
2019-02-22 07:45:59 +00:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
2019-04-26 14:00:03 +00:00
|
|
|
d.log.Infof("watcher exiting")
|
2019-07-02 12:27:51 +00:00
|
|
|
return
|
2019-07-25 13:53:26 +00:00
|
|
|
case <-sleepCh:
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (d *DataManager) initializeChanges(ctx context.Context) error {
|
2019-02-22 07:45:59 +00:00
|
|
|
var revision int64
|
|
|
|
var continuation *etcd.ListPagedContinuation
|
|
|
|
for {
|
2019-04-26 14:00:03 +00:00
|
|
|
listResp, err := d.e.ListPaged(ctx, etcdWalsDir+"/", 0, 10, continuation)
|
2019-02-22 07:45:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
resp := listResp.Resp
|
|
|
|
continuation = listResp.Continuation
|
|
|
|
|
|
|
|
revision = resp.Header.Revision
|
|
|
|
|
|
|
|
for _, kv := range resp.Kvs {
|
|
|
|
var walData *WalData
|
|
|
|
if err := json.Unmarshal(kv.Value, &walData); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
if err := d.applyWalChanges(ctx, walData, revision); err != nil {
|
2019-02-22 07:45:59 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !listResp.HasMore {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
continuation = nil
|
|
|
|
// use the same revision
|
|
|
|
for {
|
2019-04-26 14:00:03 +00:00
|
|
|
listResp, err := d.e.ListPaged(ctx, etcdChangeGroupsDir+"/", 0, 10, continuation)
|
2019-02-22 07:45:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
resp := listResp.Resp
|
|
|
|
continuation = listResp.Continuation
|
|
|
|
|
|
|
|
for _, kv := range resp.Kvs {
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
2019-02-22 07:45:59 +00:00
|
|
|
changeGroup := path.Base(string(kv.Key))
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.putChangeGroup(changeGroup, kv.ModRevision)
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
if !listResp.HasMore {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
|
|
|
d.changes.revision = revision
|
|
|
|
d.changes.initialized = true
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
func (d *DataManager) watcher(ctx context.Context) error {
|
|
|
|
d.changes.Lock()
|
|
|
|
revision := d.changes.curRevision()
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
|
|
|
|
wctx, cancel := context.WithCancel(ctx)
|
|
|
|
defer cancel()
|
2019-04-26 14:00:03 +00:00
|
|
|
wch := d.e.Watch(wctx, etcdWalBaseDir+"/", revision+1)
|
2019-02-22 07:45:59 +00:00
|
|
|
for wresp := range wch {
|
|
|
|
if wresp.Canceled {
|
|
|
|
err := wresp.Err()
|
2022-02-24 11:56:52 +00:00
|
|
|
if errors.Is(err, etcdclientv3rpc.ErrCompacted) {
|
2019-04-26 14:00:03 +00:00
|
|
|
d.log.Errorf("required events already compacted, reinitializing watcher changes")
|
|
|
|
d.changes.Lock()
|
|
|
|
d.changes.initialized = false
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
2019-05-23 09:23:14 +00:00
|
|
|
return errors.Errorf("watch error: %w", err)
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
revision := wresp.Header.Revision
|
|
|
|
|
|
|
|
for _, ev := range wresp.Events {
|
|
|
|
key := string(ev.Kv.Key)
|
|
|
|
|
|
|
|
switch {
|
|
|
|
case strings.HasPrefix(key, etcdWalsDir+"/"):
|
|
|
|
switch ev.Type {
|
|
|
|
case mvccpb.PUT:
|
|
|
|
var walData *WalData
|
|
|
|
if err := json.Unmarshal(ev.Kv.Value, &walData); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if walData.WalStatus != WalStatusCommitted {
|
|
|
|
continue
|
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
if err := d.applyWalChanges(ctx, walData, revision); err != nil {
|
2019-02-22 07:45:59 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
case mvccpb.DELETE:
|
|
|
|
walseq := path.Base(string(key))
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
|
|
|
putsToDelete := map[string][]string{}
|
|
|
|
deletesToDelete := map[string][]string{}
|
|
|
|
for _, dataType := range d.dataTypes {
|
|
|
|
putsToDelete[dataType] = []string{}
|
|
|
|
deletesToDelete[dataType] = []string{}
|
|
|
|
}
|
|
|
|
for _, dataType := range d.dataTypes {
|
|
|
|
for p, pwalseq := range d.changes.puts[dataType] {
|
|
|
|
if pwalseq == walseq {
|
|
|
|
putsToDelete[dataType] = append(putsToDelete[dataType], p)
|
|
|
|
}
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
for _, dataType := range d.dataTypes {
|
|
|
|
for id, pwalseq := range d.changes.deletes[dataType] {
|
|
|
|
if pwalseq == walseq {
|
|
|
|
deletesToDelete[dataType] = append(deletesToDelete[dataType], id)
|
|
|
|
}
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
for dataType, ids := range putsToDelete {
|
|
|
|
for _, id := range ids {
|
|
|
|
d.changes.removePut(dataType, id, revision)
|
|
|
|
}
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
2019-04-26 14:00:03 +00:00
|
|
|
for dataType, ids := range putsToDelete {
|
|
|
|
for _, id := range ids {
|
|
|
|
d.changes.removeDelete(dataType, id, revision)
|
|
|
|
}
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
delete(d.changes.actions, walseq)
|
2019-02-22 07:45:59 +00:00
|
|
|
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
case strings.HasPrefix(key, etcdChangeGroupsDir+"/"):
|
|
|
|
switch ev.Type {
|
|
|
|
case mvccpb.PUT:
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
2019-03-28 14:42:45 +00:00
|
|
|
changeGroup := strings.TrimPrefix(string(ev.Kv.Key), etcdChangeGroupsDir+"/")
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.putChangeGroup(changeGroup, ev.Kv.ModRevision)
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
case mvccpb.DELETE:
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
2019-03-28 14:42:45 +00:00
|
|
|
changeGroup := strings.TrimPrefix(string(ev.Kv.Key), etcdChangeGroupsDir+"/")
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.removeChangeGroup(changeGroup)
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
case key == etcdPingKey:
|
2019-04-26 14:00:03 +00:00
|
|
|
d.changes.Lock()
|
|
|
|
d.changes.putRevision(wresp.Header.Revision)
|
|
|
|
d.changes.Unlock()
|
2019-02-22 07:45:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|