a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

runservice: rename config pipeline(s) to run(s) 

No need for different names. We are always talking about a run. In the config
it's the run definition, in the service is the run execution.
This commit is contained in:
Simone Gotti 2019-04-15 23:42:10 +02:00
parent 06374e14fd
commit 6066221136
6 changed files with 98 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.agola/config.yml
View File

@ -61,7 +61,7 @@ tasks:
- run: sleep 5
- run: ./bin/docker-tests -test.parallel 1 -test.v
pipelines:
runs:
agola build/test:
elements:
build go1.12:

88
internal/config/config.go
View File

@ -28,9 +28,9 @@ import (
)
const (
maxPipelineNameLength = 100
maxTaskNameLength = 100
maxStepNameLength = 100
maxRunNameLength = 100
maxTaskNameLength = 100
maxStepNameLength = 100
)
var (
@ -38,9 +38,9 @@ var (
)
type Config struct {
Runtimes map[string]*Runtime `yaml:"runtimes"`
Tasks map[string]*Task `yaml:"tasks"`
Pipelines map[string]*Pipeline `yaml:"pipelines"`
Runtimes map[string]*Runtime `yaml:"runtimes"`
Tasks map[string]*Task `yaml:"tasks"`
Runs map[string]*Run `yaml:"runs"`
}
type Task struct {
@ -90,7 +90,7 @@ type Container struct {
Entrypoint string `yaml:"entrypoint"`
}
type Pipeline struct {
type Run struct {
Name string `yaml:"name"`
Elements map[string]*Element `yaml:"elements"`
}
@ -532,13 +532,13 @@ func (c *Config) Task(taskName string) *Task {
panic(fmt.Sprintf("task %q doesn't exists", taskName))
}
func (c *Config) Pipeline(pipelineName string) *Pipeline {
for n, p := range c.Pipelines {
if n == pipelineName {
func (c *Config) Run(runName string) *Run {
for n, p := range c.Runs {
if n == runName {
return p
}
}
panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName))
panic(fmt.Sprintf("run %q doesn't exists", runName))
}
var DefaultConfig = Config{}
@ -549,8 +549,8 @@ func ParseConfig(configData []byte) (*Config, error) {
return nil, errors.Wrapf(err, "failed to unmarshal config")
}
if len(config.Pipelines) == 0 {
return nil, errors.Errorf("no pipelines defined")
if len(config.Runs) == 0 {
return nil, errors.Errorf("no runs defined")
}
// Set names from maps keys
@ -568,17 +568,17 @@ func ParseConfig(configData []byte) (*Config, error) {
task.Name = n
}
for n, pipeline := range config.Pipelines {
if pipeline == nil {
return nil, errors.Errorf("pipeline %q is empty", n)
for n, run := range config.Runs {
if run == nil {
return nil, errors.Errorf("run %q is empty", n)
}
pipeline.Name = n
run.Name = n
}
for _, pipeline := range config.Pipelines {
for n, element := range pipeline.Elements {
for _, run := range config.Runs {
for n, element := range run.Elements {
if element == nil {
return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n)
return nil, errors.Errorf("run %q: element %q is empty", run.Name, n)
}
element.Name = n
}
@ -624,33 +624,33 @@ func ParseConfig(configData []byte) (*Config, error) {
func checkConfig(config *Config) error {
// check broken dependencies
for _, pipeline := range config.Pipelines {
for _, run := range config.Runs {
// collect all task names
allElements := map[string]struct{}{}
for _, element := range pipeline.Elements {
for _, element := range run.Elements {
allElements[element.Name] = struct{}{}
}
for _, element := range pipeline.Elements {
for _, element := range run.Elements {
for _, dep := range element.Depends {
if _, ok := allElements[dep.ElementName]; !ok {
return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
return errors.Errorf("run element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
}
}
}
}
// check circular dependencies
for _, pipeline := range config.Pipelines {
for _, run := range config.Runs {
cerrs := &util.Errors{}
for _, element := range pipeline.Elements {
allParents := getAllElementParents(pipeline, element)
for _, element := range run.Elements {
allParents := getAllElementParents(run, element)
for _, parent := range allParents {
if parent.Name == element.Name {
// TODO(sgotti) get the parent that depends on task to report it
dep := []string{}
for _, parent := range allParents {
pparents := getElementParents(pipeline, parent)
pparents := getElementParents(run, parent)
for _, pparent := range pparents {
if pparent.Name == element.Name {
dep = append(dep, fmt.Sprintf("%q", parent.Name))
@ -667,12 +667,12 @@ func checkConfig(config *Config) error {
}
// check that the task and its parent don't have a common dependency
for _, pipeline := range config.Pipelines {
for _, element := range pipeline.Elements {
parents := getElementParents(pipeline, element)
for _, run := range config.Runs {
for _, element := range run.Elements {
parents := getElementParents(run, element)
for _, parent := range parents {
allParents := getAllElementParents(pipeline, element)
allParentParents := getAllElementParents(pipeline, parent)
allParents := getAllElementParents(run, element)
allParentParents := getAllElementParents(run, parent)
for _, p := range allParents {
for _, pp := range allParentParents {
if p.Name == pp.Name {
@ -725,17 +725,17 @@ func checkConfig(config *Config) error {
}
}
for _, pipeline := range config.Pipelines {
if len(pipeline.Name) > maxPipelineNameLength {
return errors.Errorf("pipeline name %q too long", pipeline.Name)
for _, run := range config.Runs {
if len(run.Name) > maxRunNameLength {
return errors.Errorf("run name %q too long", run.Name)
}
for _, element := range pipeline.Elements {
for _, element := range run.Elements {
// check missing tasks reference
if element.Task == "" {
return errors.Errorf("no task defined for pipeline element %q", element.Name)
return errors.Errorf("no task defined for run element %q", element.Name)
}
if _, ok := config.Tasks[element.Task]; !ok {
return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name)
return errors.Errorf("task %q needed by run element %q doesn't exist", element.Task, element.Name)
}
// check duplicate dependencies in task
seenDependencies := map[string]struct{}{}
@ -752,9 +752,9 @@ func checkConfig(config *Config) error {
}
// getElementParents returns direct parents of element.
func getElementParents(pipeline *Pipeline, element *Element) []*Element {
func getElementParents(run *Run, element *Element) []*Element {
parents := []*Element{}
for _, el := range pipeline.Elements {
for _, el := range run.Elements {
isParent := false
for _, d := range element.Depends {
if d.ElementName == el.Name {
@ -771,9 +771,9 @@ func getElementParents(pipeline *Pipeline, element *Element) []*Element {
// getAllElementParents returns all the parents (both direct and ancestors) of an element.
// In case of circular dependency it won't loop forever but will also return
// the element as parent of itself
func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
func getAllElementParents(run *Run, element *Element) []*Element {
pMap := map[string]*Element{}
nextParents := getElementParents(pipeline, element)
nextParents := getElementParents(run, element)
for len(nextParents) > 0 {
parents := nextParents
@ -783,7 +783,7 @@ func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
continue
}
pMap[parent.Name] = parent
nextParents = append(nextParents, getElementParents(pipeline, parent)...)
nextParents = append(nextParents, getElementParents(run, parent)...)
}
}

38
internal/config/config_test.go
View File

@ -32,24 +32,24 @@ func TestParseConfig(t *testing.T) {
err error
}{
{
name: "test no pipelines 1",
name: "test no runs 1",
in: ``,
err: fmt.Errorf(`no pipelines defined`),
err: fmt.Errorf(`no runs defined`),
},
{
name: "test no pipelines 2",
name: "test no runs 2",
in: `
pipelines:
runs:
`,
err: fmt.Errorf(`no pipelines defined`),
err: fmt.Errorf(`no runs defined`),
},
{
name: "test empty pipeline",
name: "test empty run",
in: `
pipelines:
pipeline01:
runs:
run01:
`,
err: fmt.Errorf(`pipeline "pipeline01" is empty`),
err: fmt.Errorf(`run "run01" is empty`),
},
{
name: "test missing element dependency",
@ -59,15 +59,15 @@ func TestParseConfig(t *testing.T) {
environment:
ENV01: ENV01
pipelines:
pipeline01:
runs:
run01:
elements:
element01:
task: task01
depends:
- element02
`,
err: fmt.Errorf(`pipeline element "element02" needed by element "element01" doesn't exist`),
err: fmt.Errorf(`run element "element02" needed by element "element01" doesn't exist`),
},
{
name: "test circular dependency between 2 elements a -> b -> a",
@ -77,8 +77,8 @@ func TestParseConfig(t *testing.T) {
environment:
ENV01: ENV01
pipelines:
pipeline01:
runs:
run01:
elements:
element01:
task: task01
@ -172,8 +172,8 @@ func TestParseOutput(t *testing.T) {
contents:
- source_dir: /go/pkg/mod/cache
pipelines:
pipeline01:
runs:
run01:
elements:
element01:
task: task01
@ -259,9 +259,9 @@ func TestParseOutput(t *testing.T) {
},
},
},
Pipelines: map[string]*Pipeline{
"pipeline01": &Pipeline{
Name: "pipeline01",
Runs: map[string]*Run{
"run01": &Run{
Name: "run01",
Elements: map[string]*Element{
"element01": &Element{
Name: "element01",

40
internal/runconfig/runconfig.go
View File

@ -182,39 +182,39 @@ fi
}
}
// GenRunConfigTasks generates a run config tasks from a pipeline in the config, expanding all the references to tasks
// GenRunConfigTasks generates a run config tasks from a run in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, pipelineName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask {
cp := c.Pipeline(pipelineName)
func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask {
cr := c.Run(runName)
rcts := map[string]*rstypes.RunConfigTask{}
for _, cpe := range cp.Elements {
include := types.MatchWhen(cpe.When, branch, tag, ref)
for _, cre := range cr.Elements {
include := types.MatchWhen(cre.When, branch, tag, ref)
// resolve referenced task
cpt := c.Task(cpe.Task)
ct := c.Task(cre.Task)
steps := make([]interface{}, len(cpt.Steps))
for i, cpts := range cpt.Steps {
steps := make([]interface{}, len(ct.Steps))
for i, cpts := range ct.Steps {
steps[i] = stepFromConfigStep(cpts, variables)
}
tEnv := genEnv(cpt.Environment, variables)
tEnv := genEnv(ct.Environment, variables)
t := &rstypes.RunConfigTask{
ID: uuid.New(cpe.Name).String(),
ID: uuid.New(cre.Name).String(),
// use the element name from the config as the task name
Name: cpe.Name,
Runtime: genRuntime(c, cpt.Runtime, variables),
Name: cre.Name,
Runtime: genRuntime(c, ct.Runtime, variables),
Environment: tEnv,
WorkingDir: cpt.WorkingDir,
Shell: cpt.Shell,
User: cpt.User,
WorkingDir: ct.WorkingDir,
Shell: ct.Shell,
User: ct.User,
Steps: steps,
IgnoreFailure: cpe.IgnoreFailure,
IgnoreFailure: cre.IgnoreFailure,
Skip: !include,
NeedsApproval: cpe.Approval,
NeedsApproval: cre.Approval,
}
rcts[t.ID] = t
@ -222,10 +222,10 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, pipelineName s
// populate depends, needs to be done after having created all the tasks so we can resolve their id
for _, rct := range rcts {
cpe := cp.Elements[rct.Name]
cre := cr.Elements[rct.Name]
depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cpe.Depends))
for _, d := range cpe.Depends {
depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cre.Depends))
for _, d := range cre.Depends {
conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
// when no conditions are defined default to on_success
if len(d.Conditions) == 0 {

20
internal/runconfig/runconfig_test.go
View File

@ -699,9 +699,9 @@ func TestGenRunConfig(t *testing.T) {
},
},
},
Pipelines: map[string]*config.Pipeline{
"pipeline01": &config.Pipeline{
Name: "pipeline01",
Runs: map[string]*config.Run{
"run01": &config.Run{
Name: "run01",
Elements: map[string]*config.Element{
"element01": &config.Element{
Name: "element01",
@ -794,9 +794,9 @@ func TestGenRunConfig(t *testing.T) {
},
},
},
Pipelines: map[string]*config.Pipeline{
"pipeline01": &config.Pipeline{
Name: "pipeline01",
Runs: map[string]*config.Run{
"run01": &config.Run{
Name: "run01",
Elements: map[string]*config.Element{
"element01": &config.Element{
Name: "element01",
@ -874,9 +874,9 @@ func TestGenRunConfig(t *testing.T) {
},
},
},
Pipelines: map[string]*config.Pipeline{
"pipeline01": &config.Pipeline{
Name: "pipeline01",
Runs: map[string]*config.Run{
"run01": &config.Run{
Name: "run01",
Elements: map[string]*config.Element{
"element01": &config.Element{
Name: "element01",
@ -918,7 +918,7 @@ func TestGenRunConfig(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out := GenRunConfigTasks(uuid, tt.in, "pipeline01", tt.variables, "", "", "")
out := GenRunConfigTasks(uuid, tt.in, "run01", tt.variables, "", "", "")
//if err != nil {
// t.Fatalf("unexpected error: %v", err)

10
internal/services/gateway/webhook.go
View File

@ -348,7 +348,7 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
log.Errorf("failed to parse config: %+v", err)
// create a run (per config file) with a generic error since we cannot parse
// it and know how many pipelines are defined
// it and know how many runs are defined
setupErrors = append(setupErrors, err.Error())
createRunReq := &rsapi.RunCreateRequest{
RunConfigTasks: nil,
@ -365,11 +365,9 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
}
return nil
}
//h.log.Debugf("config: %v", util.Dump(config))
//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName)
for _, pipeline := range config.Pipelines {
rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, pipeline.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
for _, run := range config.Runs {
rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, run.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
h.log.Debugf("rcts: %s", util.Dump(rcts))
h.log.Infof("group: %s", group)
@ -377,7 +375,7 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
RunConfigTasks: rcts,
Group: group,
SetupErrors: setupErrors,
Name: pipeline.Name,
Name: run.Name,
StaticEnvironment: staticEnv,
Annotations: annotations,
}