a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

runservice: rename config pipeline(s) to run(s) 

No need for different names. We are always talking about a run. In the config
it's the run definition, in the service is the run execution.
This commit is contained in:
Simone Gotti 2019-04-15 23:42:10 +02:00
parent 06374e14fd
commit 6066221136
6 changed files with 98 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.agola/config.yml
View File

@ -61,7 +61,7 @@ tasks:
- run: sleep 5 - run: sleep 5
- run: ./bin/docker-tests -test.parallel 1 -test.v - run: ./bin/docker-tests -test.parallel 1 -test.v
pipelines: runs:
agola build/test: agola build/test:
elements: elements:
build go1.12: build go1.12:

88
internal/config/config.go
View File

@ -28,9 +28,9 @@ import (
) )
const ( const (
maxPipelineNameLength = 100 maxRunNameLength = 100
maxTaskNameLength = 100 maxTaskNameLength = 100
maxStepNameLength = 100 maxStepNameLength = 100
) )
var ( var (
@ -38,9 +38,9 @@ var (
) )
type Config struct { type Config struct {
Runtimes map[string]*Runtime `yaml:"runtimes"` Runtimes map[string]*Runtime `yaml:"runtimes"`
Tasks map[string]*Task `yaml:"tasks"` Tasks map[string]*Task `yaml:"tasks"`
Pipelines map[string]*Pipeline `yaml:"pipelines"` Runs map[string]*Run `yaml:"runs"`
} }
type Task struct { type Task struct {
@ -90,7 +90,7 @@ type Container struct {
Entrypoint string `yaml:"entrypoint"` Entrypoint string `yaml:"entrypoint"`
} }
type Pipeline struct { type Run struct {
Name string `yaml:"name"` Name string `yaml:"name"`
Elements map[string]*Element `yaml:"elements"` Elements map[string]*Element `yaml:"elements"`
} }
@ -532,13 +532,13 @@ func (c *Config) Task(taskName string) *Task {
panic(fmt.Sprintf("task %q doesn't exists", taskName)) panic(fmt.Sprintf("task %q doesn't exists", taskName))
} }
func (c *Config) Pipeline(pipelineName string) *Pipeline { func (c *Config) Run(runName string) *Run {
for n, p := range c.Pipelines { for n, p := range c.Runs {
if n == pipelineName { if n == runName {
return p return p
} }
} }
panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName)) panic(fmt.Sprintf("run %q doesn't exists", runName))
} }
var DefaultConfig = Config{} var DefaultConfig = Config{}
@ -549,8 +549,8 @@ func ParseConfig(configData []byte) (*Config, error) {
return nil, errors.Wrapf(err, "failed to unmarshal config") return nil, errors.Wrapf(err, "failed to unmarshal config")
} }
if len(config.Pipelines) == 0 { if len(config.Runs) == 0 {
return nil, errors.Errorf("no pipelines defined") return nil, errors.Errorf("no runs defined")
} }
// Set names from maps keys // Set names from maps keys
@ -568,17 +568,17 @@ func ParseConfig(configData []byte) (*Config, error) {
task.Name = n task.Name = n
} }
for n, pipeline := range config.Pipelines { for n, run := range config.Runs {
if pipeline == nil { if run == nil {
return nil, errors.Errorf("pipeline %q is empty", n) return nil, errors.Errorf("run %q is empty", n)
} }
pipeline.Name = n run.Name = n
} }
for _, pipeline := range config.Pipelines { for _, run := range config.Runs {
for n, element := range pipeline.Elements { for n, element := range run.Elements {
if element == nil { if element == nil {
return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n) return nil, errors.Errorf("run %q: element %q is empty", run.Name, n)
} }
element.Name = n element.Name = n
} }
@ -624,33 +624,33 @@ func ParseConfig(configData []byte) (*Config, error) {
func checkConfig(config *Config) error { func checkConfig(config *Config) error {
// check broken dependencies // check broken dependencies
for _, pipeline := range config.Pipelines { for _, run := range config.Runs {
// collect all task names // collect all task names
allElements := map[string]struct{}{} allElements := map[string]struct{}{}
for _, element := range pipeline.Elements { for _, element := range run.Elements {
allElements[element.Name] = struct{}{} allElements[element.Name] = struct{}{}
} }
for _, element := range pipeline.Elements { for _, element := range run.Elements {
for _, dep := range element.Depends { for _, dep := range element.Depends {
if _, ok := allElements[dep.ElementName]; !ok { if _, ok := allElements[dep.ElementName]; !ok {
return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name) return errors.Errorf("run element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
} }
} }
} }
} }
// check circular dependencies // check circular dependencies
for _, pipeline := range config.Pipelines { for _, run := range config.Runs {
cerrs := &util.Errors{} cerrs := &util.Errors{}
for _, element := range pipeline.Elements { for _, element := range run.Elements {
allParents := getAllElementParents(pipeline, element) allParents := getAllElementParents(run, element)
for _, parent := range allParents { for _, parent := range allParents {
if parent.Name == element.Name { if parent.Name == element.Name {
// TODO(sgotti) get the parent that depends on task to report it // TODO(sgotti) get the parent that depends on task to report it
dep := []string{} dep := []string{}
for _, parent := range allParents { for _, parent := range allParents {
pparents := getElementParents(pipeline, parent) pparents := getElementParents(run, parent)
for _, pparent := range pparents { for _, pparent := range pparents {
if pparent.Name == element.Name { if pparent.Name == element.Name {
dep = append(dep, fmt.Sprintf("%q", parent.Name)) dep = append(dep, fmt.Sprintf("%q", parent.Name))
@ -667,12 +667,12 @@ func checkConfig(config *Config) error {
} }
// check that the task and its parent don't have a common dependency // check that the task and its parent don't have a common dependency
for _, pipeline := range config.Pipelines { for _, run := range config.Runs {
for _, element := range pipeline.Elements { for _, element := range run.Elements {
parents := getElementParents(pipeline, element) parents := getElementParents(run, element)
for _, parent := range parents { for _, parent := range parents {
allParents := getAllElementParents(pipeline, element) allParents := getAllElementParents(run, element)
allParentParents := getAllElementParents(pipeline, parent) allParentParents := getAllElementParents(run, parent)
for _, p := range allParents { for _, p := range allParents {
for _, pp := range allParentParents { for _, pp := range allParentParents {
if p.Name == pp.Name { if p.Name == pp.Name {
@ -725,17 +725,17 @@ func checkConfig(config *Config) error {
} }
} }
for _, pipeline := range config.Pipelines { for _, run := range config.Runs {
if len(pipeline.Name) > maxPipelineNameLength { if len(run.Name) > maxRunNameLength {
return errors.Errorf("pipeline name %q too long", pipeline.Name) return errors.Errorf("run name %q too long", run.Name)
} }
for _, element := range pipeline.Elements { for _, element := range run.Elements {
// check missing tasks reference // check missing tasks reference
if element.Task == "" { if element.Task == "" {
return errors.Errorf("no task defined for pipeline element %q", element.Name) return errors.Errorf("no task defined for run element %q", element.Name)
} }
if _, ok := config.Tasks[element.Task]; !ok { if _, ok := config.Tasks[element.Task]; !ok {
return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name) return errors.Errorf("task %q needed by run element %q doesn't exist", element.Task, element.Name)
} }
// check duplicate dependencies in task // check duplicate dependencies in task
seenDependencies := map[string]struct{}{} seenDependencies := map[string]struct{}{}
@ -752,9 +752,9 @@ func checkConfig(config *Config) error {
} }
// getElementParents returns direct parents of element. // getElementParents returns direct parents of element.
func getElementParents(pipeline *Pipeline, element *Element) []*Element { func getElementParents(run *Run, element *Element) []*Element {
parents := []*Element{} parents := []*Element{}
for _, el := range pipeline.Elements { for _, el := range run.Elements {
isParent := false isParent := false
for _, d := range element.Depends { for _, d := range element.Depends {
if d.ElementName == el.Name { if d.ElementName == el.Name {
@ -771,9 +771,9 @@ func getElementParents(pipeline *Pipeline, element *Element) []*Element {
// getAllElementParents returns all the parents (both direct and ancestors) of an element. // getAllElementParents returns all the parents (both direct and ancestors) of an element.
// In case of circular dependency it won't loop forever but will also return // In case of circular dependency it won't loop forever but will also return
// the element as parent of itself // the element as parent of itself
func getAllElementParents(pipeline *Pipeline, element *Element) []*Element { func getAllElementParents(run *Run, element *Element) []*Element {
pMap := map[string]*Element{} pMap := map[string]*Element{}
nextParents := getElementParents(pipeline, element) nextParents := getElementParents(run, element)
for len(nextParents) > 0 { for len(nextParents) > 0 {
parents := nextParents parents := nextParents
@ -783,7 +783,7 @@ func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
continue continue
} }
pMap[parent.Name] = parent pMap[parent.Name] = parent
nextParents = append(nextParents, getElementParents(pipeline, parent)...) nextParents = append(nextParents, getElementParents(run, parent)...)
} }
} }

38
internal/config/config_test.go
View File

@ -32,24 +32,24 @@ func TestParseConfig(t *testing.T) {
err error err error
}{ }{
{ {
name: "test no pipelines 1", name: "test no runs 1",
in: ``, in: ``,
err: fmt.Errorf(`no pipelines defined`), err: fmt.Errorf(`no runs defined`),
}, },
{ {
name: "test no pipelines 2", name: "test no runs 2",
in: ` in: `
pipelines: runs:
`, `,
err: fmt.Errorf(`no pipelines defined`), err: fmt.Errorf(`no runs defined`),
}, },
{ {
name: "test empty pipeline", name: "test empty run",
in: ` in: `
pipelines: runs:
pipeline01: run01:
`, `,
err: fmt.Errorf(`pipeline "pipeline01" is empty`), err: fmt.Errorf(`run "run01" is empty`),
}, },
{ {
name: "test missing element dependency", name: "test missing element dependency",
@ -59,15 +59,15 @@ func TestParseConfig(t *testing.T) {
environment: environment:
ENV01: ENV01 ENV01: ENV01
pipelines: runs:
pipeline01: run01:
elements: elements:
element01: element01:
task: task01 task: task01
depends: depends:
- element02 - element02
`, `,
err: fmt.Errorf(`pipeline element "element02" needed by element "element01" doesn't exist`), err: fmt.Errorf(`run element "element02" needed by element "element01" doesn't exist`),
}, },
{ {
name: "test circular dependency between 2 elements a -> b -> a", name: "test circular dependency between 2 elements a -> b -> a",
@ -77,8 +77,8 @@ func TestParseConfig(t *testing.T) {
environment: environment:
ENV01: ENV01 ENV01: ENV01
pipelines: runs:
pipeline01: run01:
elements: elements:
element01: element01:
task: task01 task: task01
@ -172,8 +172,8 @@ func TestParseOutput(t *testing.T) {
contents: contents:
- source_dir: /go/pkg/mod/cache - source_dir: /go/pkg/mod/cache
pipelines: runs:
pipeline01: run01:
elements: elements:
element01: element01:
task: task01 task: task01
@ -259,9 +259,9 @@ func TestParseOutput(t *testing.T) {
}, },
}, },
}, },
Pipelines: map[string]*Pipeline{ Runs: map[string]*Run{
"pipeline01": &Pipeline{ "run01": &Run{
Name: "pipeline01", Name: "run01",
Elements: map[string]*Element{ Elements: map[string]*Element{
"element01": &Element{ "element01": &Element{
Name: "element01", Name: "element01",

40
internal/runconfig/runconfig.go
View File

@ -182,39 +182,39 @@ fi
} }
} }
// GenRunConfigTasks generates a run config tasks from a pipeline in the config, expanding all the references to tasks // GenRunConfigTasks generates a run config tasks from a run in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits) // this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, pipelineName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask { func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask {
cp := c.Pipeline(pipelineName) cr := c.Run(runName)
rcts := map[string]*rstypes.RunConfigTask{} rcts := map[string]*rstypes.RunConfigTask{}
for _, cpe := range cp.Elements { for _, cre := range cr.Elements {
include := types.MatchWhen(cpe.When, branch, tag, ref) include := types.MatchWhen(cre.When, branch, tag, ref)
// resolve referenced task // resolve referenced task
cpt := c.Task(cpe.Task) ct := c.Task(cre.Task)
steps := make([]interface{}, len(cpt.Steps)) steps := make([]interface{}, len(ct.Steps))
for i, cpts := range cpt.Steps { for i, cpts := range ct.Steps {
steps[i] = stepFromConfigStep(cpts, variables) steps[i] = stepFromConfigStep(cpts, variables)
} }
tEnv := genEnv(cpt.Environment, variables) tEnv := genEnv(ct.Environment, variables)
t := &rstypes.RunConfigTask{ t := &rstypes.RunConfigTask{
ID: uuid.New(cpe.Name).String(), ID: uuid.New(cre.Name).String(),
// use the element name from the config as the task name // use the element name from the config as the task name
Name: cpe.Name, Name: cre.Name,
Runtime: genRuntime(c, cpt.Runtime, variables), Runtime: genRuntime(c, ct.Runtime, variables),
Environment: tEnv, Environment: tEnv,
WorkingDir: cpt.WorkingDir, WorkingDir: ct.WorkingDir,
Shell: cpt.Shell, Shell: ct.Shell,
User: cpt.User, User: ct.User,
Steps: steps, Steps: steps,
IgnoreFailure: cpe.IgnoreFailure, IgnoreFailure: cre.IgnoreFailure,
Skip: !include, Skip: !include,
NeedsApproval: cpe.Approval, NeedsApproval: cre.Approval,
} }
rcts[t.ID] = t rcts[t.ID] = t
@ -222,10 +222,10 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, pipelineName s
// populate depends, needs to be done after having created all the tasks so we can resolve their id // populate depends, needs to be done after having created all the tasks so we can resolve their id
for _, rct := range rcts { for _, rct := range rcts {
cpe := cp.Elements[rct.Name] cre := cr.Elements[rct.Name]
depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cpe.Depends)) depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cre.Depends))
for _, d := range cpe.Depends { for _, d := range cre.Depends {
conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions)) conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
// when no conditions are defined default to on_success // when no conditions are defined default to on_success
if len(d.Conditions) == 0 { if len(d.Conditions) == 0 {

20
internal/runconfig/runconfig_test.go
View File

@ -699,9 +699,9 @@ func TestGenRunConfig(t *testing.T) {
}, },
}, },
}, },
Pipelines: map[string]*config.Pipeline{ Runs: map[string]*config.Run{
"pipeline01": &config.Pipeline{ "run01": &config.Run{
Name: "pipeline01", Name: "run01",
Elements: map[string]*config.Element{ Elements: map[string]*config.Element{
"element01": &config.Element{ "element01": &config.Element{
Name: "element01", Name: "element01",
@ -794,9 +794,9 @@ func TestGenRunConfig(t *testing.T) {
}, },
}, },
}, },
Pipelines: map[string]*config.Pipeline{ Runs: map[string]*config.Run{
"pipeline01": &config.Pipeline{ "run01": &config.Run{
Name: "pipeline01", Name: "run01",
Elements: map[string]*config.Element{ Elements: map[string]*config.Element{
"element01": &config.Element{ "element01": &config.Element{
Name: "element01", Name: "element01",
@ -874,9 +874,9 @@ func TestGenRunConfig(t *testing.T) {
}, },
}, },
}, },
Pipelines: map[string]*config.Pipeline{ Runs: map[string]*config.Run{
"pipeline01": &config.Pipeline{ "run01": &config.Run{
Name: "pipeline01", Name: "run01",
Elements: map[string]*config.Element{ Elements: map[string]*config.Element{
"element01": &config.Element{ "element01": &config.Element{
Name: "element01", Name: "element01",
@ -918,7 +918,7 @@ func TestGenRunConfig(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
out := GenRunConfigTasks(uuid, tt.in, "pipeline01", tt.variables, "", "", "") out := GenRunConfigTasks(uuid, tt.in, "run01", tt.variables, "", "", "")
//if err != nil { //if err != nil {
// t.Fatalf("unexpected error: %v", err) // t.Fatalf("unexpected error: %v", err)

10
internal/services/gateway/webhook.go
View File

@ -348,7 +348,7 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
log.Errorf("failed to parse config: %+v", err) log.Errorf("failed to parse config: %+v", err)
// create a run (per config file) with a generic error since we cannot parse // create a run (per config file) with a generic error since we cannot parse
// it and know how many pipelines are defined // it and know how many runs are defined
setupErrors = append(setupErrors, err.Error()) setupErrors = append(setupErrors, err.Error())
createRunReq := &rsapi.RunCreateRequest{ createRunReq := &rsapi.RunCreateRequest{
RunConfigTasks: nil, RunConfigTasks: nil,
@ -365,11 +365,9 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
} }
return nil return nil
} }
//h.log.Debugf("config: %v", util.Dump(config))
//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName) for _, run := range config.Runs {
for _, pipeline := range config.Pipelines { rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, run.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, pipeline.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
h.log.Debugf("rcts: %s", util.Dump(rcts)) h.log.Debugf("rcts: %s", util.Dump(rcts))
h.log.Infof("group: %s", group) h.log.Infof("group: %s", group)
@ -377,7 +375,7 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
RunConfigTasks: rcts, RunConfigTasks: rcts,
Group: group, Group: group,
SetupErrors: setupErrors, SetupErrors: setupErrors,
Name: pipeline.Name, Name: run.Name,
StaticEnvironment: staticEnv, StaticEnvironment: staticEnv,
Annotations: annotations, Annotations: annotations,
} }