runservice: merge RunConfig and RunData

* Use just RunConfig
* Use StaticEnvironment vs Environment in RunConfig to distinguish between env
that won't change at run recreation from env that could change at every
recreation
* The RunCreate api will just receive the runtasks instead of a runconfig (more
right)
This commit is contained in:
Simone Gotti 2019-04-09 18:11:00 +02:00
parent 3642be6f21
commit 671b89d391
9 changed files with 152 additions and 223 deletions

View File

@ -143,16 +143,12 @@ fi
} }
} }
// GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks // GenRunConfigTasks generates a run config tasks from a pipeline in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits) // this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string, env, variables map[string]string, branch, tag, ref string) *rstypes.RunConfig { func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, pipelineName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask {
cp := c.Pipeline(pipelineName) cp := c.Pipeline(pipelineName)
rc := &rstypes.RunConfig{ rcts := map[string]*rstypes.RunConfigTask{}
Name: cp.Name,
Tasks: make(map[string]*rstypes.RunConfigTask),
Environment: env,
}
for _, cpe := range cp.Elements { for _, cpe := range cp.Elements {
include := types.MatchWhen(cpe.When, branch, tag, ref) include := types.MatchWhen(cpe.When, branch, tag, ref)
@ -185,11 +181,11 @@ func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string
NeedsApproval: cpe.Approval, NeedsApproval: cpe.Approval,
} }
rc.Tasks[t.ID] = t rcts[t.ID] = t
} }
// populate depends, needs to be done after having created all the tasks so we can resolve their id // populate depends, needs to be done after having created all the tasks so we can resolve their id
for _, rct := range rc.Tasks { for _, rct := range rcts {
cpe := cp.Elements[rct.Name] cpe := cp.Elements[rct.Name]
depends := make([]*rstypes.RunConfigTaskDepend, len(cpe.Depends)) depends := make([]*rstypes.RunConfigTaskDepend, len(cpe.Depends))
@ -211,7 +207,7 @@ func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string
} }
} }
drct := getRunConfigTaskByName(rc, d.ElementName) drct := getRunConfigTaskByName(rcts, d.ElementName)
depends[id] = &rstypes.RunConfigTaskDepend{ depends[id] = &rstypes.RunConfigTaskDepend{
TaskID: drct.ID, TaskID: drct.ID,
Conditions: conditions, Conditions: conditions,
@ -221,11 +217,11 @@ func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string
rct.Depends = depends rct.Depends = depends
} }
return rc return rcts
} }
func getRunConfigTaskByName(rc *rstypes.RunConfig, name string) *rstypes.RunConfigTask { func getRunConfigTaskByName(rcts map[string]*rstypes.RunConfigTask, name string) *rstypes.RunConfigTask {
for _, rct := range rc.Tasks { for _, rct := range rcts {
if rct.Name == name { if rct.Name == name {
return rct return rct
} }
@ -233,17 +229,17 @@ func getRunConfigTaskByName(rc *rstypes.RunConfig, name string) *rstypes.RunConf
return nil return nil
} }
func CheckRunConfig(rc *rstypes.RunConfig) error { func CheckRunConfigTasks(rcts map[string]*rstypes.RunConfigTask) error {
// check circular dependencies // check circular dependencies
cerrs := &util.Errors{} cerrs := &util.Errors{}
for _, t := range rc.Tasks { for _, t := range rcts {
allParents := GetAllParents(rc, t) allParents := GetAllParents(rcts, t)
for _, parent := range allParents { for _, parent := range allParents {
if parent.ID == t.ID { if parent.ID == t.ID {
// TODO(sgotti) get the parent that depends on task to report it // TODO(sgotti) get the parent that depends on task to report it
dep := []string{} dep := []string{}
for _, parent := range allParents { for _, parent := range allParents {
pparents := GetParents(rc, parent) pparents := GetParents(rcts, parent)
for _, pparent := range pparents { for _, pparent := range pparents {
if pparent.ID == t.ID { if pparent.ID == t.ID {
dep = append(dep, fmt.Sprintf("%q", parent.Name)) dep = append(dep, fmt.Sprintf("%q", parent.Name))
@ -259,11 +255,11 @@ func CheckRunConfig(rc *rstypes.RunConfig) error {
} }
// check that the task and its parent don't have a common dependency // check that the task and its parent don't have a common dependency
for _, t := range rc.Tasks { for _, t := range rcts {
parents := GetParents(rc, t) parents := GetParents(rcts, t)
for _, parent := range parents { for _, parent := range parents {
allParents := GetAllParents(rc, t) allParents := GetAllParents(rcts, t)
allParentParents := GetAllParents(rc, parent) allParentParents := GetAllParents(rcts, parent)
for _, p := range allParents { for _, p := range allParents {
for _, pp := range allParentParents { for _, pp := range allParentParents {
if p.ID == pp.ID { if p.ID == pp.ID {
@ -277,22 +273,22 @@ func CheckRunConfig(rc *rstypes.RunConfig) error {
return nil return nil
} }
func GenTasksLevels(rc *rstypes.RunConfig) error { func GenTasksLevels(rcts map[string]*rstypes.RunConfigTask) error {
// reset all task level // reset all task level
for _, t := range rc.Tasks { for _, t := range rcts {
t.Level = -1 t.Level = -1
} }
level := 0 level := 0
for { for {
c := 0 c := 0
for _, t := range rc.Tasks { for _, t := range rcts {
// skip tasks with the level already set // skip tasks with the level already set
if t.Level != -1 { if t.Level != -1 {
continue continue
} }
parents := GetParents(rc, t) parents := GetParents(rcts, t)
ok := true ok := true
for _, p := range parents { for _, p := range parents {
// * skip if the parent doesn't have a level yet // * skip if the parent doesn't have a level yet
@ -314,7 +310,7 @@ func GenTasksLevels(rc *rstypes.RunConfig) error {
} }
level++ level++
} }
for _, t := range rc.Tasks { for _, t := range rcts {
if t.Level == -1 { if t.Level == -1 {
return errors.Errorf("circular dependency detected") return errors.Errorf("circular dependency detected")
} }
@ -323,9 +319,9 @@ func GenTasksLevels(rc *rstypes.RunConfig) error {
} }
// GetParents returns direct parents of task. // GetParents returns direct parents of task.
func GetParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask { func GetParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
parents := []*rstypes.RunConfigTask{} parents := []*rstypes.RunConfigTask{}
for _, t := range rc.Tasks { for _, t := range rcts {
isParent := false isParent := false
for _, d := range task.Depends { for _, d := range task.Depends {
if d.TaskID == t.ID { if d.TaskID == t.ID {
@ -342,9 +338,9 @@ func GetParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.R
// GetAllParents returns all the parents (both direct and ancestors) of task. // GetAllParents returns all the parents (both direct and ancestors) of task.
// In case of circular dependency it won't loop forever but will also return // In case of circular dependency it won't loop forever but will also return
// task as parent of itself // task as parent of itself
func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask { func GetAllParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
pMap := map[string]*rstypes.RunConfigTask{} pMap := map[string]*rstypes.RunConfigTask{}
nextParents := GetParents(rc, task) nextParents := GetParents(rcts, task)
for len(nextParents) > 0 { for len(nextParents) > 0 {
parents := nextParents parents := nextParents
@ -354,7 +350,7 @@ func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstype
continue continue
} }
pMap[parent.ID] = parent pMap[parent.ID] = parent
nextParents = append(nextParents, GetParents(rc, parent)...) nextParents = append(nextParents, GetParents(rcts, parent)...)
} }
} }

View File

@ -205,25 +205,25 @@ func TestGenTasksLevels(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} inRcts := map[string]*rstypes.RunConfigTask{}
for _, t := range tt.in { for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ inRcts[t.ID] = &rstypes.RunConfigTask{
ID: t.ID, ID: t.ID,
Level: t.Level, Level: t.Level,
Depends: t.Depends, Depends: t.Depends,
} }
} }
outRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} outRcts := map[string]*rstypes.RunConfigTask{}
for _, t := range tt.out { for _, t := range tt.out {
outRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ outRcts[t.ID] = &rstypes.RunConfigTask{
ID: t.ID, ID: t.ID,
Level: t.Level, Level: t.Level,
Depends: t.Depends, Depends: t.Depends,
} }
} }
if err := GenTasksLevels(inRunConfig); err != nil { if err := GenTasksLevels(inRcts); err != nil {
if err.Error() != tt.err.Error() { if err.Error() != tt.err.Error() {
t.Fatalf("got error: %v, want error: %v", err, tt.err) t.Fatalf("got error: %v, want error: %v", err, tt.err)
} }
@ -232,8 +232,8 @@ func TestGenTasksLevels(t *testing.T) {
if tt.err != nil { if tt.err != nil {
t.Fatalf("got nil error, want error: %v", tt.err) t.Fatalf("got nil error, want error: %v", tt.err)
} }
if !reflect.DeepEqual(inRunConfig.Tasks, outRunConfig.Tasks) { if !reflect.DeepEqual(inRcts, outRcts) {
t.Fatalf("got %s, expected %s", util.Dump(inRunConfig), util.Dump(outRunConfig)) t.Fatalf("got %s, expected %s", util.Dump(inRcts), util.Dump(outRcts))
} }
}) })
} }
@ -473,9 +473,9 @@ func TestGetAllParents(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} inRcts := map[string]*rstypes.RunConfigTask{}
for _, t := range tt.in { for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ inRcts[t.ID] = &rstypes.RunConfigTask{
ID: t.ID, ID: t.ID,
Level: t.Level, Level: t.Level,
Depends: t.Depends, Depends: t.Depends,
@ -483,8 +483,8 @@ func TestGetAllParents(t *testing.T) {
} }
for _, task := range inRunConfig.Tasks { for _, task := range inRcts {
allParents := GetAllParents(inRunConfig, task) allParents := GetAllParents(inRcts, task)
allParentsList := []string{} allParentsList := []string{}
for _, p := range allParents { for _, p := range allParents {
@ -658,9 +658,9 @@ func TestCheckRunConfig(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} inRcts := map[string]*rstypes.RunConfigTask{}
for _, t := range tt.in { for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ inRcts[t.ID] = &rstypes.RunConfigTask{
Name: fmt.Sprintf("task%s", t.ID), Name: fmt.Sprintf("task%s", t.ID),
ID: t.ID, ID: t.ID,
Level: t.Level, Level: t.Level,
@ -669,7 +669,7 @@ func TestCheckRunConfig(t *testing.T) {
} }
if err := CheckRunConfig(inRunConfig); err != nil { if err := CheckRunConfigTasks(inRcts); err != nil {
if errs, ok := err.(*util.Errors); ok { if errs, ok := err.(*util.Errors); ok {
if !errs.Equal(tt.err) { if !errs.Equal(tt.err) {
t.Fatalf("got error: %v, want error: %v", err, tt.err) t.Fatalf("got error: %v, want error: %v", err, tt.err)
@ -692,9 +692,8 @@ func TestGenRunConfig(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
in *config.Config in *config.Config
env map[string]string
variables map[string]string variables map[string]string
out *rstypes.RunConfig out map[string]*rstypes.RunConfigTask
}{ }{
{ {
name: "test runconfig generation", name: "test runconfig generation",
@ -779,43 +778,34 @@ func TestGenRunConfig(t *testing.T) {
}, },
}, },
}, },
env: map[string]string{
"ENV01": "ENVVALUE01",
},
variables: map[string]string{ variables: map[string]string{
"variable01": "VARVALUE01", "variable01": "VARVALUE01",
}, },
out: &rstypes.RunConfig{ out: map[string]*rstypes.RunConfigTask{
Name: "pipeline01", uuid.New("element01").String(): &rstypes.RunConfigTask{
Environment: map[string]string{ ID: uuid.New("element01").String(),
"ENV01": "ENVVALUE01", Name: "element01", Depends: []*rstypes.RunConfigTaskDepend{},
}, Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
Tasks: map[string]*rstypes.RunConfigTask{ Containers: []*rstypes.Container{
uuid.New("element01").String(): &rstypes.RunConfigTask{ {
ID: uuid.New("element01").String(), Image: "image01",
Name: "element01", Depends: []*rstypes.RunConfigTaskDepend{}, Environment: map[string]string{
Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"), "ENV01": "ENV01",
Containers: []*rstypes.Container{ "ENVFROMVARIABLE01": "VARVALUE01",
{
Image: "image01",
Environment: map[string]string{
"ENV01": "ENV01",
"ENVFROMVARIABLE01": "VARVALUE01",
},
}, },
}, },
}, },
Environment: map[string]string{
"ENV01": "ENV01",
"ENVFROMVARIABLE01": "VARVALUE01",
},
Steps: []interface{}{
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command01"}, Command: "command01", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "name different than command"}, Command: "command02", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command03"}, Command: "command03", Environment: map[string]string{"ENV01": "ENV01", "ENVFROMVARIABLE01": "VARVALUE01"}},
},
Skip: true,
}, },
Environment: map[string]string{
"ENV01": "ENV01",
"ENVFROMVARIABLE01": "VARVALUE01",
},
Steps: []interface{}{
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command01"}, Command: "command01", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "name different than command"}, Command: "command02", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command03"}, Command: "command03", Environment: map[string]string{"ENV01": "ENV01", "ENVFROMVARIABLE01": "VARVALUE01"}},
},
Skip: true,
}, },
}, },
}, },
@ -823,7 +813,7 @@ func TestGenRunConfig(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
out := GenRunConfig(uuid, tt.in, "pipeline01", tt.env, tt.variables, "", "", "") out := GenRunConfigTasks(uuid, tt.in, "pipeline01", tt.variables, "", "", "")
//if err != nil { //if err != nil {
// t.Fatalf("unexpected error: %v", err) // t.Fatalf("unexpected error: %v", err)

View File

@ -338,7 +338,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
return 0, "", nil return 0, "", nil
} }
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env, variables map[string]string, webhookData *types.WebhookData) error { func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
config, err := config.ParseConfig([]byte(configData)) config, err := config.ParseConfig([]byte(configData))
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to parse config") return errors.Wrapf(err, "failed to parse config")
@ -347,14 +347,16 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName) //h.log.Debugf("pipeline: %s", createRunOpts.PipelineName)
for _, pipeline := range config.Pipelines { for _, pipeline := range config.Pipelines {
rc := runconfig.GenRunConfig(util.DefaultUUIDGenerator{}, config, pipeline.Name, env, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref) rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, pipeline.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
h.log.Debugf("rc: %s", util.Dump(rc)) h.log.Debugf("rcts: %s", util.Dump(rcts))
h.log.Infof("group: %s", group) h.log.Infof("group: %s", group)
createRunReq := &rsapi.RunCreateRequest{ createRunReq := &rsapi.RunCreateRequest{
RunConfig: rc, RunConfigTasks: rcts,
Group: group, Group: group,
Annotations: annotations, Name: pipeline.Name,
StaticEnvironment: staticEnv,
Annotations: annotations,
} }
if _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil { if _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil {

View File

@ -480,19 +480,22 @@ func (h *RunsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
type RunCreateRequest struct { type RunCreateRequest struct {
// new run // new run fields
RunConfig *types.RunConfig `json:"run_config"` RunConfigTasks map[string]*types.RunConfigTask `json:"run_config_tasks"`
Name string `json:"name"`
Group string `json:"group"`
StaticEnvironment map[string]string `json:"static_environment"`
// existing run // existing run fields
RunID string `json:"run_id"` RunID string `json:"run_id"`
RunConfigID string `json:"run_config_id"` FromStart bool `json:"from_start"`
FromStart bool `json:"from_start"` ResetTasks []string `json:"reset_tasks"`
ResetTasks []string `json:"reset_tasks"`
Group string `json:"group"` // common fields
Environment map[string]string `json:"environment"` Environment map[string]string `json:"environment"`
Annotations map[string]string `json:"annotations"` Annotations map[string]string `json:"annotations"`
ChangeGroupsUpdateToken string `json:"changeup_update_tokens"`
ChangeGroupsUpdateToken string `json:"changeup_update_tokens"`
} }
type RunCreateHandler struct { type RunCreateHandler struct {
@ -518,12 +521,15 @@ func (h *RunCreateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
creq := &command.RunCreateRequest{ creq := &command.RunCreateRequest{
RunConfig: req.RunConfig, RunConfigTasks: req.RunConfigTasks,
RunID: req.RunID, Name: req.Name,
RunConfigID: req.RunConfigID, Group: req.Group,
FromStart: req.FromStart, StaticEnvironment: req.StaticEnvironment,
ResetTasks: req.ResetTasks,
Group: req.Group, RunID: req.RunID,
FromStart: req.FromStart,
ResetTasks: req.ResetTasks,
Environment: req.Environment, Environment: req.Environment,
Annotations: req.Annotations, Annotations: req.Annotations,
ChangeGroupsUpdateToken: req.ChangeGroupsUpdateToken, ChangeGroupsUpdateToken: req.ChangeGroupsUpdateToken,

View File

@ -117,14 +117,20 @@ func (s *CommandHandler) StopRun(ctx context.Context, req *RunStopRequest) error
} }
type RunCreateRequest struct { type RunCreateRequest struct {
RunConfig *types.RunConfig RunConfigTasks map[string]*types.RunConfigTask
RunID string Name string
RunConfigID string Group string
FromStart bool StaticEnvironment map[string]string
ResetTasks []string
Group string // existing run fields
Environment map[string]string RunID string
Annotations map[string]string FromStart bool
ResetTasks []string
// common fields
Environment map[string]string
Annotations map[string]string
ChangeGroupsUpdateToken string ChangeGroupsUpdateToken string
} }
@ -148,8 +154,7 @@ func (s *CommandHandler) CreateRun(ctx context.Context, req *RunCreateRequest) (
} }
func (s *CommandHandler) newRun(ctx context.Context, req *RunCreateRequest) (*types.RunBundle, error) { func (s *CommandHandler) newRun(ctx context.Context, req *RunCreateRequest) (*types.RunBundle, error) {
rc := req.RunConfig rcts := req.RunConfigTasks
var run *types.Run
if req.Group == "" { if req.Group == "" {
return nil, util.NewErrBadRequest(errors.Errorf("run group is empty")) return nil, util.NewErrBadRequest(errors.Errorf("run group is empty"))
@ -158,53 +163,50 @@ func (s *CommandHandler) newRun(ctx context.Context, req *RunCreateRequest) (*ty
return nil, util.NewErrBadRequest(errors.Errorf("run group %q must be an absolute path", req.Group)) return nil, util.NewErrBadRequest(errors.Errorf("run group %q must be an absolute path", req.Group))
} }
// generate a new run sequence that will be the same for the run, runconfig and rundata // generate a new run sequence that will be the same for the run and runconfig
seq, err := sequence.IncSequence(ctx, s.e, common.EtcdRunSequenceKey) seq, err := sequence.IncSequence(ctx, s.e, common.EtcdRunSequenceKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
id := seq.String() id := seq.String()
if err := runconfig.CheckRunConfig(rc); err != nil { if err := runconfig.CheckRunConfigTasks(rcts); err != nil {
return nil, util.NewErrBadRequest(err) return nil, util.NewErrBadRequest(err)
} }
// set the run config ID
rc.ID = id
// generate tasks levels // generate tasks levels
if err := runconfig.GenTasksLevels(rc); err != nil { if err := runconfig.GenTasksLevels(rcts); err != nil {
return nil, util.NewErrBadRequest(err) return nil, util.NewErrBadRequest(err)
} }
rd := &types.RunData{ rc := &types.RunConfig{
ID: id, ID: id,
Group: req.Group, Name: req.Name,
Environment: req.Environment, Group: req.Group,
Annotations: req.Annotations, Tasks: rcts,
StaticEnvironment: req.StaticEnvironment,
Environment: req.Environment,
Annotations: req.Annotations,
} }
run, err = s.genRun(ctx, rc, rd) run := s.genRun(ctx, rc)
if err != nil {
return nil, util.NewErrBadRequest(err)
}
s.log.Debugf("created run: %s", util.Dump(run)) s.log.Debugf("created run: %s", util.Dump(run))
return &types.RunBundle{ return &types.RunBundle{
Run: run, Run: run,
Rc: rc, Rc: rc,
Rd: rd,
}, nil }, nil
} }
func (s *CommandHandler) recreateRun(ctx context.Context, req *RunCreateRequest) (*types.RunBundle, error) { func (s *CommandHandler) recreateRun(ctx context.Context, req *RunCreateRequest) (*types.RunBundle, error) {
// generate a new run sequence that will be the same for the run, runconfig and rundata // generate a new run sequence that will be the same for the run and runconfig
seq, err := sequence.IncSequence(ctx, s.e, common.EtcdRunSequenceKey) seq, err := sequence.IncSequence(ctx, s.e, common.EtcdRunSequenceKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
id := seq.String() id := seq.String()
// fetch the existing runconfig, rundata and run // fetch the existing runconfig and run
s.log.Infof("creating run from existing run") s.log.Infof("creating run from existing run")
rc, err := store.LTSGetRunConfig(s.wal, req.RunID) rc, err := store.LTSGetRunConfig(s.wal, req.RunID)
if err != nil { if err != nil {
@ -212,13 +214,8 @@ func (s *CommandHandler) recreateRun(ctx context.Context, req *RunCreateRequest)
} }
// update the run config ID // update the run config ID
rc.ID = id rc.ID = id
// update the run config Environment
rd, err := store.LTSGetRunData(s.wal, req.RunID) rc.Environment = req.Environment
if err != nil {
return nil, util.NewErrBadRequest(errors.Wrapf(err, "rundata %q doens't exist", req.RunID))
}
// update the run data ID
rd.ID = id
run, err := store.GetRunEtcdOrLTS(ctx, s.e, s.wal, req.RunID) run, err := store.GetRunEtcdOrLTS(ctx, s.e, s.wal, req.RunID)
if err != nil { if err != nil {
@ -288,14 +285,12 @@ func (s *CommandHandler) recreateRun(ctx context.Context, req *RunCreateRequest)
return &types.RunBundle{ return &types.RunBundle{
Run: run, Run: run,
Rc: rc, Rc: rc,
Rd: rd,
}, nil }, nil
} }
func (s *CommandHandler) saveRun(ctx context.Context, rb *types.RunBundle, runcgt *types.ChangeGroupsUpdateToken) error { func (s *CommandHandler) saveRun(ctx context.Context, rb *types.RunBundle, runcgt *types.ChangeGroupsUpdateToken) error {
run := rb.Run run := rb.Run
rc := rb.Rc rc := rb.Rc
rd := rb.Rd
c, cgt, err := s.getRunCounter(run.Group) c, cgt, err := s.getRunCounter(run.Group)
s.log.Infof("c: %d, cgt: %s", c, util.Dump(cgt)) s.log.Infof("c: %d, cgt: %s", c, util.Dump(cgt))
@ -321,13 +316,6 @@ func (s *CommandHandler) saveRun(ctx context.Context, rb *types.RunBundle, runcg
} }
actions = append(actions, rca) actions = append(actions, rca)
// persist run data
rda, err := store.LTSSaveRunDataAction(rd)
if err != nil {
return err
}
actions = append(actions, rda)
if _, err = s.wal.WriteWal(ctx, actions, cgt); err != nil { if _, err = s.wal.WriteWal(ctx, actions, cgt); err != nil {
return err return err
} }
@ -374,12 +362,12 @@ func (s *CommandHandler) genRunTask(ctx context.Context, rct *types.RunConfigTas
return rt return rt
} }
func (s *CommandHandler) genRun(ctx context.Context, rc *types.RunConfig, rd *types.RunData) (*types.Run, error) { func (s *CommandHandler) genRun(ctx context.Context, rc *types.RunConfig) *types.Run {
r := &types.Run{ r := &types.Run{
ID: rc.ID, ID: rc.ID,
Name: rc.Name, Name: rc.Name,
Group: rd.Group, Group: rc.Group,
Annotations: rd.Annotations, Annotations: rc.Annotations,
Phase: types.RunPhaseQueued, Phase: types.RunPhaseQueued,
Result: types.RunResultUnknown, Result: types.RunResultUnknown,
RunTasks: make(map[string]*types.RunTask), RunTasks: make(map[string]*types.RunTask),
@ -391,7 +379,7 @@ func (s *CommandHandler) genRun(ctx context.Context, rc *types.RunConfig, rd *ty
r.RunTasks[rt.ID] = rt r.RunTasks[rt.ID] = rt
} }
return r, nil return r
} }
type RunTaskApproveRequest struct { type RunTaskApproveRequest struct {

View File

@ -55,7 +55,6 @@ const (
var ( var (
StorageDataDir = "" StorageDataDir = ""
StorageRunsDir = path.Join(StorageDataDir, "runs") StorageRunsDir = path.Join(StorageDataDir, "runs")
StorageRunsDataDir = path.Join(StorageDataDir, "runsdata")
StorageRunsConfigDir = path.Join(StorageDataDir, "runsconfig") StorageRunsConfigDir = path.Join(StorageDataDir, "runsconfig")
StorageRunsIndexesDir = path.Join(StorageDataDir, "runsindexes") StorageRunsIndexesDir = path.Join(StorageDataDir, "runsindexes")
StorageCountersDir = path.Join(StorageDataDir, "counters") StorageCountersDir = path.Join(StorageDataDir, "counters")
@ -69,10 +68,6 @@ func StorageRunFile(runID string) string {
return path.Join(StorageRunsDir, runID) return path.Join(StorageRunsDir, runID)
} }
func StorageRunDataFile(runID string) string {
return path.Join(StorageRunsDataDir, runID)
}
func StorageRunConfigFile(runID string) string { func StorageRunConfigFile(runID string) string {
return path.Join(StorageRunsConfigDir, runID) return path.Join(StorageRunsConfigDir, runID)
} }
@ -85,7 +80,6 @@ type DataType string
const ( const (
DataTypeRun DataType = "run" DataTypeRun DataType = "run"
DataTypeRunData DataType = "rundata"
DataTypeRunConfig DataType = "runconfig" DataTypeRunConfig DataType = "runconfig"
DataTypeRunCounter DataType = "runcounter" DataTypeRunCounter DataType = "runcounter"
) )
@ -94,8 +88,6 @@ func DataToPathFunc(dataType string, id string) string {
switch DataType(dataType) { switch DataType(dataType) {
case DataTypeRun: case DataTypeRun:
return StorageRunFile(id) return StorageRunFile(id)
case DataTypeRunData:
return StorageRunDataFile(id)
case DataTypeRunConfig: case DataTypeRunConfig:
return StorageRunConfigFile(id) return StorageRunConfigFile(id)
case DataTypeRunCounter: case DataTypeRunCounter:

View File

@ -88,11 +88,6 @@ func (s *Scheduler) advanceRunTasks(ctx context.Context, r *types.Run) error {
return errors.Wrapf(err, "cannot get run config %q", r.ID) return errors.Wrapf(err, "cannot get run config %q", r.ID)
} }
log.Debugf("rc: %s", util.Dump(rc)) log.Debugf("rc: %s", util.Dump(rc))
rd, err := store.LTSGetRunData(s.wal, r.ID)
if err != nil {
return errors.Wrapf(err, "cannot get run data %q", r.ID)
}
log.Debugf("rd: %s", util.Dump(rd))
tasksToRun := []*types.RunTask{} tasksToRun := []*types.RunTask{}
// get tasks that can be executed // get tasks that can be executed
@ -106,7 +101,7 @@ func (s *Scheduler) advanceRunTasks(ctx context.Context, r *types.Run) error {
} }
rct := rc.Tasks[rt.ID] rct := rc.Tasks[rt.ID]
parents := runconfig.GetParents(rc, rct) parents := runconfig.GetParents(rc.Tasks, rct)
canRun := true canRun := true
for _, p := range parents { for _, p := range parents {
rp := r.RunTasks[p.ID] rp := r.RunTasks[p.ID]
@ -136,7 +131,7 @@ func (s *Scheduler) advanceRunTasks(ctx context.Context, r *types.Run) error {
log.Debugf("tasksToRun: %s", util.Dump(tasksToRun)) log.Debugf("tasksToRun: %s", util.Dump(tasksToRun))
for _, rt := range tasksToRun { for _, rt := range tasksToRun {
et, err := s.genExecutorTask(ctx, r, rt, rc, rd) et, err := s.genExecutorTask(ctx, r, rt, rc)
if err != nil { if err != nil {
return err return err
} }
@ -178,7 +173,7 @@ func (s *Scheduler) chooseExecutor(ctx context.Context) (*types.Executor, error)
return nil, nil return nil, nil
} }
func (s *Scheduler) genExecutorTask(ctx context.Context, r *types.Run, rt *types.RunTask, rc *types.RunConfig, rd *types.RunData) (*types.ExecutorTask, error) { func (s *Scheduler) genExecutorTask(ctx context.Context, r *types.Run, rt *types.RunTask, rc *types.RunConfig) (*types.ExecutorTask, error) {
executor, err := s.chooseExecutor(ctx) executor, err := s.chooseExecutor(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
@ -193,9 +188,9 @@ func (s *Scheduler) genExecutorTask(ctx context.Context, r *types.Run, rt *types
if rct.Environment != nil { if rct.Environment != nil {
environment = rct.Environment environment = rct.Environment
} }
mergeEnv(environment, rc.StaticEnvironment)
// run config Environment variables ovverride every other environment variable
mergeEnv(environment, rc.Environment) mergeEnv(environment, rc.Environment)
// run data Environment variables ovverride every other environment variable
mergeEnv(environment, rd.Environment)
et := &types.ExecutorTask{ et := &types.ExecutorTask{
// The executorTask ID must be the same as the runTask ID so we can detect if // The executorTask ID must be the same as the runTask ID so we can detect if
@ -225,7 +220,7 @@ func (s *Scheduler) genExecutorTask(ctx context.Context, r *types.Run, rt *types
// calculate workspace layers // calculate workspace layers
ws := make(types.Workspace, rct.Level+1) ws := make(types.Workspace, rct.Level+1)
rctAllParents := runconfig.GetAllParents(rc, rct) rctAllParents := runconfig.GetAllParents(rc.Tasks, rct)
log.Debugf("rctAllParents: %s", util.Dump(rctAllParents)) log.Debugf("rctAllParents: %s", util.Dump(rctAllParents))
for _, rctParent := range rctAllParents { for _, rctParent := range rctAllParents {
log.Debugf("rctParent: %s", util.Dump(rctParent)) log.Debugf("rctParent: %s", util.Dump(rctParent))

View File

@ -163,38 +163,6 @@ func LTSSaveRunConfigAction(rc *types.RunConfig) (*wal.Action, error) {
return action, nil return action, nil
} }
func LTSGetRunData(wal *wal.WalManager, runDataID string) (*types.RunData, error) {
runDataPath := common.StorageRunDataFile(runDataID)
rdf, _, err := wal.ReadObject(runDataPath, nil)
if err != nil {
return nil, err
}
defer rdf.Close()
d := json.NewDecoder(rdf)
var rd *types.RunData
if err := d.Decode(&rd); err != nil {
return nil, err
}
return rd, nil
}
func LTSSaveRunDataAction(rd *types.RunData) (*wal.Action, error) {
rdj, err := json.Marshal(rd)
if err != nil {
return nil, err
}
action := &wal.Action{
ActionType: wal.ActionTypePut,
DataType: string(common.DataTypeRunData),
ID: rd.ID,
Data: rdj,
}
return action, nil
}
func LTSGetRun(wal *wal.WalManager, runID string) (*types.Run, error) { func LTSGetRun(wal *wal.WalManager, runID string) (*types.Run, error) {
runPath := common.StorageRunFile(runID) runPath := common.StorageRunFile(runID)
rf, _, err := wal.ReadObject(runPath, nil) rf, _, err := wal.ReadObject(runPath, nil)

View File

@ -33,7 +33,6 @@ const (
type RunBundle struct { type RunBundle struct {
Run *Run Run *Run
Rc *RunConfig Rc *RunConfig
Rd *RunData
} }
type RunCounter struct { type RunCounter struct {
@ -251,13 +250,15 @@ type RunTaskStep struct {
EndTime *time.Time `json:"end_time,omitempty"` EndTime *time.Time `json:"end_time,omitempty"`
} }
// RunData // RunConfig
// RunData is the data for a RUN. It contains everything that isn't a state // RunConfig is the run configuration.
// (it's contained in a Run) and that may use a lot of space. It lives in the // It contains everything that isn't a state (that is contained in a Run) and
// storage. There is a RunData for every Run. // that may use a lot of space. It lives in the storage. There is a RunConfig
type RunData struct { // for every Run.
ID string `json:"id,omitempty"` type RunConfig struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
// Group is the run group of the run. Every run is assigned to a specific group // Group is the run group of the run. Every run is assigned to a specific group
// i.e. project/$projectid/$branch // i.e. project/$projectid/$branch
@ -266,27 +267,18 @@ type RunData struct {
// also needed to fetch them when they aren't indexed in the readdb. // also needed to fetch them when they aren't indexed in the readdb.
Group string `json:"group,omitempty"` Group string `json:"group,omitempty"`
// Environment contains all environment variables that are different between runs also if using the same RunConfig
// (like secrets that may change or user provided enviroment specific to this run)
Environment map[string]string `json:"environment,omitempty"`
// Annotations contain custom run properties // Annotations contain custom run properties
// Note: Annotations are currently both saved in a Run and in RunData to easily return them without loading RunData from the lts // Note: Annotations are currently both saved in a Run and in RunConfig to
// easily return them without loading RunConfig from the lts
Annotations map[string]string `json:"annotations,omitempty"` Annotations map[string]string `json:"annotations,omitempty"`
}
// RunConfig // StaticEnvironment contains all environment variables that won't change when
// RunConfig is the run configuration. It lives in the storage. It can be
// copied (i.e when we create a new run from an previous run).
// It could also be shared but to simplify the run delete logic we will just
// copy it when creating a new run as a modified previous run.
type RunConfig struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
// Environment contains all environment variables that won't change when
// generating a new run (like COMMIT_SHA, BRANCH, REPOSITORY_URL etc...) // generating a new run (like COMMIT_SHA, BRANCH, REPOSITORY_URL etc...)
StaticEnvironment map[string]string `json:"static_environment,omitempty"`
// Environment contains all environment variables that are different between
// runs recreations (like secrets that may change or user provided enviroment
// specific to this run)
Environment map[string]string `json:"environment,omitempty"` Environment map[string]string `json:"environment,omitempty"`
Tasks map[string]*RunConfigTask `json:"tasks,omitempty"` Tasks map[string]*RunConfigTask `json:"tasks,omitempty"`