runservice: rework config format

The current config format was thought for future extensions for reusing runtimes
and job definitions adding some parameters.

After a lot of thoughts this looks like a complex approach: the final result
will be a sort of templating without a lot of powers.

Other approach like external templating should be an alternative but I really
don't think templating yaml is the way to go.

A much better approach will to just use jsonnet when we need to create matrix
runs and a lot of other use cases.

So just make the config a simple yaml/json. User can generate their config using
any preferred tool and in future we'll leverage jsonnet automated parsing and
provide a lot of jsonnet based examples for most use cases.

Main changes:

* Runs are now an array and not a map. The run name is in the Name field
* Tasks are now an array and not a map. The task name is in the Name field
* Use https://github.com/ghodss/yaml so we'll use json struct tags and unmarshall functions

.agola/config.yml

@@ -1,74 +1,60 @@
-runtimes:
-  go1.12:
-    type: pod
-    arch: amd64
-    containers:
-      - image: golang:1.12-stretch
-        environment:
-          ENV01: envvalue01
-  debian:
-    type: pod
-    arch: amd64
-    containers:
-      - image: debian:stretch
-  dind:
-    type: pod
-    arch: amd64
-    containers:
-      - image: docker:stable-dind
-        privileged: true
-        entrypoint: dockerd
-
-tasks:
-  build-go1.12:
-    runtime: go1.12
-    working_dir: /go/src/github.com/sorintlab/agola
-    environment:
-      GO111MODULE: "on"
-      VAR01:
-        from_variable: var01
-    steps:
-      - run: env
-      - clone:
-      - run: SKIP_DOCKER_TESTS=1 go test -v -count 1 ./...
-
-  build-docker-tests-go-1.12:
-    runtime: go1.12
-    working_dir: /go/src/github.com/sorintlab/agola
-    environment:
-      GO111MODULE: "on"
-    steps:
-      - run: env
-      - clone:
-      - run:
-          name: build docker tests binary
-          command: CGO_ENABLED=0 go test -c ./internal/services/runservice/executor/driver -o ./bin/docker-tests
-          environment:
-            ENV01: envvalue01
-      - save_to_workspace:
-          contents:
-            - source_dir: ./bin
-              dest_dir: /bin/
-              paths:
-                - "*"
-
-  test-docker-driver:
-    runtime: dind
-    steps:
-      - run: env
-      - restore_workspace:
-          dest_dir: .
-      - run: sleep 5
-      - run: ./bin/docker-tests -test.parallel 1 -test.v
-
 runs:
-  agola build/test:
+  - name: agola build/test
-    elements:
+    tasks:
-      build go1.12:
+      - name: build go1.12
-        task: build-go1.12
+        runtime:
-      build docker tests go1.12:
+          type: pod
-        task: build-docker-tests-go-1.12
+          arch: amd64
-      test docker driver:
+          containers:
-        task: test-docker-driver
+            - image: golang:1.12-stretch
+              environment:
+                ENV01: envvalue01
+        working_dir: /go/src/github.com/sorintlab/agola
+        environment:
+          GO111MODULE: "on"
+          VAR01:
+            from_variable: var01
+        steps:
+          - run: env
+          - clone:
+          - run: SKIP_DOCKER_TESTS=1 go test -v -count 1 ./...
+      - name: build docker tests go1.12
+        runtime:
+          type: pod
+          arch: amd64
+          containers:
+            - image: golang:1.12-stretch
+              environment:
+                ENV01: envvalue01
+        working_dir: /go/src/github.com/sorintlab/agola
+        environment:
+          GO111MODULE: "on"
+        steps:
+          - run: env
+          - clone:
+          - run:
+              name: build docker tests binary
+              command: CGO_ENABLED=0 go test -c ./internal/services/runservice/executor/driver -o ./bin/docker-tests
+              environment:
+                ENV01: envvalue01
+          - save_to_workspace:
+              contents:
+                - source_dir: ./bin
+                  dest_dir: /bin/
+                  paths:
+                    - "*"
+      - name: test docker driver
+        runtime:
+          type: pod
+          arch: amd64
+          containers:
+            - image: docker:stable-dind
+              privileged: true
+              entrypoint: dockerd
+        steps:
+          - run: env
+          - restore_workspace:
+              dest_dir: .
+          - run: ./bin/docker-tests -test.parallel 1 -test.v
         depends:
           - build docker tests go1.12

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.3.3 // indirect
 	github.com/elazarl/go-bindata-assetfs v1.0.0
+	github.com/ghodss/yaml v1.0.0
 	github.com/go-bindata/go-bindata v1.0.0
 	github.com/go-ini/ini v1.42.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect

internal/config/config.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strings"
@@ -23,8 +24,8 @@ import (
 	"github.com/sorintlab/agola/internal/services/types"
 	"github.com/sorintlab/agola/internal/util"
 
+	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
-	yaml "gopkg.in/yaml.v2"
 )
 
 const (
@@ -38,19 +39,7 @@ var (
 )
 
 type Config struct {
-	Runtimes map[string]*Runtime `yaml:"runtimes"`
+	Runs []*Run `json:"runs"`
-	Tasks    map[string]*Task    `yaml:"tasks"`
-	Runs     map[string]*Run     `yaml:"runs"`
-}
-
-type Task struct {
-	Name        string           `yaml:"name"`
-	Runtime     string           `yaml:"runtime"`
-	Environment map[string]Value `yaml:"environment,omitempty"`
-	WorkingDir  string           `yaml:"working_dir"`
-	Shell       string           `yaml:"shell"`
-	User        string           `yaml:"user"`
-	Steps       []interface{}    `yaml:"steps"`
 }
 
 type RuntimeType string
@@ -66,42 +55,46 @@ const (
 )
 
 type RegistryAuth struct {
-	Type RegistryAuthType `yaml:"type"`
+	Type RegistryAuthType `json:"type"`
 
 	// default auth
-	Username Value `yaml:"username"`
+	Username Value `json:"username"`
-	Password Value `yaml:"password"`
+	Password Value `json:"password"`
 }
 
 type Runtime struct {
-	Name       string        `yaml:"name"`
+	Type       RuntimeType   `json:"type,omitempty"`
-	Type       RuntimeType   `yaml:"type,omitempty"`
+	Auth       *RegistryAuth `json:"auth"`
-	Auth       *RegistryAuth `yaml:"auth"`
+	Arch       common.Arch   `json:"arch,omitempty"`
-	Arch       common.Arch   `yaml:"arch,omitempty"`
+	Containers []*Container  `json:"containers,omitempty"`
-	Containers []*Container  `yaml:"containers,omitempty"`
 }
 
 type Container struct {
-	Image       string           `yaml:"image,omitempty"`
+	Image       string           `json:"image,omitempty"`
-	Auth        *RegistryAuth    `yaml:"auth"`
+	Auth        *RegistryAuth    `json:"auth"`
-	Environment map[string]Value `yaml:"environment,omitempty"`
+	Environment map[string]Value `json:"environment,omitempty"`
-	User        string           `yaml:"user"`
+	User        string           `json:"user"`
-	Privileged  bool             `yaml:"privileged"`
+	Privileged  bool             `json:"privileged"`
-	Entrypoint  string           `yaml:"entrypoint"`
+	Entrypoint  string           `json:"entrypoint"`
 }
 
 type Run struct {
-	Name     string              `yaml:"name"`
+	Name  string  `json:"name"`
-	Elements map[string]*Element `yaml:"elements"`
+	Tasks []*Task `json:"tasks"`
 }
 
-type Element struct {
+type Task struct {
-	Name          string      `yaml:"name"`
+	Name          string           `json:"name"`
-	Task          string      `yaml:"task"`
+	Runtime       *Runtime         `json:"runtime"`
-	Depends       []*Depend   `yaml:"depends"`
+	Environment   map[string]Value `json:"environment,omitempty"`
-	IgnoreFailure bool        `yaml:"ignore_failure"`
+	WorkingDir    string           `json:"working_dir"`
-	Approval      bool        `yaml:"approval"`
+	Shell         string           `json:"shell"`
-	When          *types.When `yaml:"when"`
+	User          string           `json:"user"`
+	Steps         []interface{}    `json:"steps"`
+	Depends       []*Depend        `json:"depends"`
+	IgnoreFailure bool             `json:"ignore_failure"`
+	Approval      bool             `json:"approval"`
+	When          *types.When      `json:"when"`
 }
 
 type DependCondition string
@@ -113,26 +106,26 @@ const (
 )
 
 type Depend struct {
-	ElementName string            `yaml:"name"`
+	TaskName   string            `json:"task"`
-	Conditions  []DependCondition `yaml:"conditions"`
+	Conditions []DependCondition `json:"conditions"`
 }
 
 type Step struct {
-	Type string `yaml:"type"`
+	Type string `json:"type"`
-	Name string `yaml:"name"`
+	Name string `json:"name"`
 }
 
 type CloneStep struct {
-	Step `yaml:",inline"`
+	Step `json:",inline"`
 }
 
 type RunStep struct {
-	Step        `yaml:",inline"`
+	Step        `json:",inline"`
-	Command     string           `yaml:"command"`
+	Command     string           `json:"command"`
-	Environment map[string]Value `yaml:"environment,omitempty"`
+	Environment map[string]Value `json:"environment,omitempty"`
-	WorkingDir  string           `yaml:"working_dir"`
+	WorkingDir  string           `json:"working_dir"`
-	Shell       string           `yaml:"shell"`
+	Shell       string           `json:"shell"`
-	User        string           `yaml:"user"`
+	User        string           `json:"user"`
 }
 
 type ValueType int
@@ -148,63 +141,78 @@ type Value struct {
 }
 
 type SaveContent struct {
-	SourceDir string   `yaml:"source_dir"`
+	SourceDir string   `json:"source_dir"`
-	DestDir   string   `yaml:"dest_dir"`
+	DestDir   string   `json:"dest_dir"`
-	Paths     []string `yaml:"paths"`
+	Paths     []string `json:"paths"`
 }
 
 type SaveToWorkspaceStep struct {
-	Step     `yaml:",inline"`
+	Step     `json:",inline"`
-	Contents []*SaveContent `yaml:"contents"`
+	Contents []*SaveContent `json:"contents"`
 }
 
 type RestoreWorkspaceStep struct {
-	Step    `yaml:",inline"`
+	Step    `json:",inline"`
-	DestDir string `yaml:"dest_dir"`
+	DestDir string `json:"dest_dir"`
 }
 
 type SaveCacheStep struct {
-	Step     `yaml:",inline"`
+	Step     `json:",inline"`
-	Key      string         `yaml:"key"`
+	Key      string         `json:"key"`
-	Contents []*SaveContent `yaml:"contents"`
+	Contents []*SaveContent `json:"contents"`
 }
 
 type RestoreCacheStep struct {
-	Step    `yaml:",inline"`
+	Step    `json:",inline"`
-	Keys    []string `yaml:"keys"`
+	Keys    []string `json:"keys"`
-	DestDir string   `yaml:"dest_dir"`
+	DestDir string   `json:"dest_dir"`
 }
 
-func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (t *Task) UnmarshalJSON(b []byte) error {
-	type task struct {
+	type when struct {
-		Name        string                   `yaml:"name"`
+		Branch interface{} `json:"branch"`
-		Runtime     string                   `yaml:"runtime"`
+		Tag    interface{} `json:"tag"`
-		Environment map[string]Value         `yaml:"environment,omitempty"`
+		Ref    interface{} `json:"ref"`
-		WorkingDir  string                   `yaml:"working_dir"`
-		Shell       string                   `yaml:"shell"`
-		User        string                   `yaml:"user"`
-		Steps       []map[string]interface{} `yaml:"steps"`
 	}
 
-	var tt *task
+	type runtask struct {
-	if err := unmarshal(&tt); err != nil {
+		Name          string                   `json:"name"`
+		Runtime       *Runtime                 `json:"runtime"`
+		Environment   map[string]Value         `json:"environment,omitempty"`
+		WorkingDir    string                   `json:"working_dir"`
+		Shell         string                   `json:"shell"`
+		User          string                   `json:"user"`
+		Steps         []map[string]interface{} `json:"steps"`
+		Depends       []interface{}            `json:"depends"`
+		IgnoreFailure bool                     `json:"ignore_failure"`
+		Approval      bool                     `json:"approval"`
+		When          *when                    `json:"when"`
+	}
+
+	var tr *runtask
+
+	if err := json.Unmarshal(b, &tr); err != nil {
 		return err
 	}
 
-	t.Name = tt.Name
+	t.Name = tr.Name
-	t.Runtime = tt.Runtime
+	t.Runtime = tr.Runtime
-	t.Environment = tt.Environment
+	t.Environment = tr.Environment
-	t.WorkingDir = tt.WorkingDir
+	t.WorkingDir = tr.WorkingDir
-	t.Shell = tt.Shell
+	t.Shell = tr.Shell
-	t.User = tt.User
+	t.User = tr.User
+	t.IgnoreFailure = tr.IgnoreFailure
+	t.Approval = tr.Approval
 
-	steps := make([]interface{}, len(tt.Steps))
+	steps := make([]interface{}, len(tr.Steps))
-	for i, stepEntry := range tt.Steps {
+	for i, stepEntry := range tr.Steps {
-		if len(stepEntry) > 1 {
+		if _, ok := stepEntry["type"]; ok {
-			return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
+			// handle default step definition using format { type: "steptype", other steps fields }
-		}
+			stepType, ok := stepEntry["type"].(string)
-		for stepType, stepSpec := range stepEntry {
+			if !ok {
-			o, err := yaml.Marshal(stepSpec)
+				return errors.Errorf("step type at index %d must be a string", i)
+			}
+			o, err := json.Marshal(stepEntry)
 			if err != nil {
 				return err
 			}
@@ -216,20 +224,15 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 			case "run":
 				var s RunStep
-				switch stepSpec.(type) {
+				if err := json.Unmarshal(o, &s); err != nil {
-				case string:
+					return err
-					s.Command = stepSpec.(string)
-				default:
-					if err := yaml.Unmarshal(o, &s); err != nil {
-						return err
-					}
 				}
 				s.Type = stepType
 				steps[i] = &s
 
 			case "save_to_workspace":
 				var s SaveToWorkspaceStep
-				if err := yaml.Unmarshal(o, &s); err != nil {
+				if err := json.Unmarshal(o, &s); err != nil {
 					return err
 				}
 				s.Type = stepType
@@ -237,7 +240,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 			case "restore_workspace":
 				var s RestoreWorkspaceStep
-				if err := yaml.Unmarshal(o, &s); err != nil {
+				if err := json.Unmarshal(o, &s); err != nil {
 					return err
 				}
 				s.Type = stepType
@@ -245,7 +248,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 			case "save_cache":
 				var s SaveCacheStep
-				if err := yaml.Unmarshal(o, &s); err != nil {
+				if err := json.Unmarshal(o, &s); err != nil {
 					return err
 				}
 				s.Type = stepType
@@ -253,7 +256,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 			case "restore_cache":
 				var s RestoreCacheStep
-				if err := yaml.Unmarshal(o, &s); err != nil {
+				if err := json.Unmarshal(o, &s); err != nil {
 					return err
 				}
 				s.Type = stepType
@@ -261,119 +264,181 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			default:
 				return errors.Errorf("unknown step type: %s", stepType)
 			}
+		} else {
+			// handle simpler (for yaml not for json) steps definition using format "steptype": { stepSpecification }
+			if len(stepEntry) > 1 {
+				return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
+			}
+			for stepType, stepSpec := range stepEntry {
+				o, err := json.Marshal(stepSpec)
+				if err != nil {
+					return err
+				}
+				switch stepType {
+				case "clone":
+					var s CloneStep
+					s.Type = stepType
+					steps[i] = &s
+
+				case "run":
+					var s RunStep
+					switch stepSpec.(type) {
+					case string:
+						s.Command = stepSpec.(string)
+					default:
+						if err := json.Unmarshal(o, &s); err != nil {
+							return err
+						}
+					}
+					s.Type = stepType
+					steps[i] = &s
+
+				case "save_to_workspace":
+					var s SaveToWorkspaceStep
+					if err := json.Unmarshal(o, &s); err != nil {
+						return err
+					}
+					s.Type = stepType
+					steps[i] = &s
+
+				case "restore_workspace":
+					var s RestoreWorkspaceStep
+					if err := json.Unmarshal(o, &s); err != nil {
+						return err
+					}
+					s.Type = stepType
+					steps[i] = &s
+
+				case "save_cache":
+					var s SaveCacheStep
+					if err := json.Unmarshal(o, &s); err != nil {
+						return err
+					}
+					s.Type = stepType
+					steps[i] = &s
+
+				case "restore_cache":
+					var s RestoreCacheStep
+					if err := json.Unmarshal(o, &s); err != nil {
+						return err
+					}
+					s.Type = stepType
+					steps[i] = &s
+				default:
+					return errors.Errorf("unknown step type: %s", stepType)
+				}
+			}
 		}
 	}
 
 	t.Steps = steps
 
-	return nil
+	depends := make([]*Depend, len(tr.Depends))
-}
+	for i, dependEntry := range tr.Depends {
-
-func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	type when struct {
-		Branch interface{} `yaml:"branch"`
-		Tag    interface{} `yaml:"tag"`
-		Ref    interface{} `yaml:"ref"`
-	}
-
-	type element struct {
-		Name          string        `yaml:"name"`
-		Task          string        `yaml:"task"`
-		Depends       []interface{} `yaml:"depends"`
-		IgnoreFailure bool          `yaml:"ignore_failure"`
-		Approval      bool          `yaml:"approval"`
-		When          *when         `yaml:"when"`
-	}
-
-	var te *element
-
-	if err := unmarshal(&te); err != nil {
-		return err
-	}
-
-	e.Name = te.Name
-	e.Task = te.Task
-	e.IgnoreFailure = te.IgnoreFailure
-	e.Approval = te.Approval
-
-	depends := make([]*Depend, len(te.Depends))
-	for i, dependEntry := range te.Depends {
 		var depend *Depend
-		switch dependEntry.(type) {
+		isSimpler := false
+		switch de := dependEntry.(type) {
+		// handle simpler (for yaml) depends definition using format "taskname":
 		case string:
 			depend = &Depend{
-				ElementName: dependEntry.(string),
+				TaskName: dependEntry.(string),
 			}
-		case map[interface{}]interface{}:
+		case map[string]interface{}:
-			type deplist map[string][]DependCondition
+			if len(de) == 1 {
-			var dl deplist
+				for _, v := range de {
-			o, err := yaml.Marshal(dependEntry)
+					switch v.(type) {
-			if err != nil {
+					case []interface{}:
-				return err
+						isSimpler = true
+					case string:
+					default:
+						return errors.Errorf("unsupported depend entry format")
+					}
+				}
 			}
-			if err := yaml.Unmarshal(o, &dl); err != nil {
+			if !isSimpler {
-				return err
+				// handle default depends definition using format "task": "taskname", conditions: [ list of conditions ]
-			}
+				o, err := json.Marshal(dependEntry)
-			if len(dl) != 1 {
+				if err != nil {
-				return errors.Errorf("unsupported depend format. Must be a string or a list")
+					return err
-			}
+				}
-			for k, v := range dl {
+				if err := json.Unmarshal(o, &depend); err != nil {
-				depend = &Depend{
+					return err
-					ElementName: k,
+				}
-					Conditions:  v,
+			} else {
+				// handle simpler (for yaml) depends definition using format "taskname": [ list of conditions ]
+				if len(de) != 1 {
+					return errors.Errorf("unsupported depend entry format")
+				}
+				type deplist map[string][]DependCondition
+				var dl deplist
+				o, err := json.Marshal(dependEntry)
+				if err != nil {
+					return err
+				}
+				if err := json.Unmarshal(o, &dl); err != nil {
+					return err
+				}
+				if len(dl) != 1 {
+					return errors.Errorf("unsupported depend entry format")
+				}
+				for k, v := range dl {
+					depend = &Depend{
+						TaskName:   k,
+						Conditions: v,
+					}
 				}
 			}
 
 		default:
-			return errors.Errorf("unsupported depend format. Must be a string or a list")
+			return errors.Errorf("unsupported depend entry format")
 		}
 		depends[i] = depend
 	}
 
-	e.Depends = depends
+	t.Depends = depends
 
-	if te.When != nil {
+	if tr.When != nil {
 		w := &types.When{}
 
 		var err error
 
-		if te.When.Branch != nil {
+		if tr.When.Branch != nil {
-			w.Branch, err = parseWhenConditions(te.When.Branch)
+			w.Branch, err = parseWhenConditions(tr.When.Branch)
 			if err != nil {
 				return err
 			}
 		}
 
-		if te.When.Tag != nil {
+		if tr.When.Tag != nil {
-			w.Tag, err = parseWhenConditions(te.When.Tag)
+			w.Tag, err = parseWhenConditions(tr.When.Tag)
 			if err != nil {
 				return err
 			}
 		}
 
-		if te.When.Ref != nil {
+		if tr.When.Ref != nil {
-			w.Ref, err = parseWhenConditions(te.When.Ref)
+			w.Ref, err = parseWhenConditions(tr.When.Ref)
 			if err != nil {
 				return err
 			}
 		}
 
-		e.When = w
+		t.When = w
 	}
 
 	return nil
 }
 
-func (val *Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (val *Value) UnmarshalJSON(b []byte) error {
 	var ival interface{}
-	if err := unmarshal(&ival); err != nil {
+	if err := json.Unmarshal(b, &ival); err != nil {
 		return err
 	}
 	switch valValue := ival.(type) {
 	case string:
 		val.Type = ValueTypeString
 		val.Value = valValue
-	case map[interface{}]interface{}:
+	case map[string]interface{}:
 		for k, v := range valValue {
 			if k == "from_variable" {
 				switch v.(type) {
@@ -407,13 +472,9 @@ func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
 			return nil, err
 		}
 		include = ss
-	case map[interface{}]interface{}:
+	case map[string]interface{}:
 		for k, v := range c {
-			ks, ok := k.(string)
+			switch k {
-			if !ok {
-				return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
-			}
-			switch ks {
 			case "include":
 				include, err = parseStringOrSlice(v)
 				if err != nil {
@@ -425,7 +486,7 @@ func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
 					return nil, err
 				}
 			default:
-				return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
+				return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, k)
 			}
 		}
 	default:
@@ -514,31 +575,22 @@ func parseSliceString(si []interface{}) ([]string, error) {
 	return ss, nil
 }
 
-func (c *Config) Runtime(runtimeName string) *Runtime {
+func (c *Config) Run(runName string) *Run {
-	for n, r := range c.Runtimes {
+	for _, r := range c.Runs {
-		if n == runtimeName {
+		if r.Name == runName {
 			return r
 		}
 	}
-	panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName))
+	panic(fmt.Sprintf("run %q doesn't exists", runName))
 }
 
-func (c *Config) Task(taskName string) *Task {
+func (r *Run) Task(taskName string) *Task {
-	for n, t := range c.Tasks {
+	for _, t := range r.Tasks {
-		if n == taskName {
+		if t.Name == taskName {
 			return t
 		}
 	}
-	panic(fmt.Sprintf("task %q doesn't exists", taskName))
+	panic(fmt.Sprintf("task %q for run %q doesn't exists", taskName, r.Name))
-}
-
-func (c *Config) Run(runName string) *Run {
-	for n, p := range c.Runs {
-		if n == runName {
-			return p
-		}
-	}
-	panic(fmt.Sprintf("run %q doesn't exists", runName))
 }
 
 var DefaultConfig = Config{}
@@ -549,92 +601,79 @@ func ParseConfig(configData []byte) (*Config, error) {
 		return nil, errors.Wrapf(err, "failed to unmarshal config")
 	}
 
-	if len(config.Runs) == 0 {
-		return nil, errors.Errorf("no runs defined")
-	}
-
-	// Set names from maps keys
-	for n, runtime := range config.Runtimes {
-		if runtime == nil {
-			return nil, errors.Errorf("runtime %q is empty", n)
-		}
-		runtime.Name = n
-	}
-
-	for n, task := range config.Tasks {
-		if task == nil {
-			return nil, errors.Errorf("task %q is empty", n)
-		}
-		task.Name = n
-	}
-
-	for n, run := range config.Runs {
-		if run == nil {
-			return nil, errors.Errorf("run %q is empty", n)
-		}
-		run.Name = n
-	}
-
-	for _, run := range config.Runs {
-		for n, element := range run.Elements {
-			if element == nil {
-				return nil, errors.Errorf("run %q: element %q is empty", run.Name, n)
-			}
-			element.Name = n
-		}
-	}
-
-	// Set auth type to default if not specified
-	for _, runtime := range config.Runtimes {
-		if runtime.Auth != nil {
-			if runtime.Auth.Type == "" {
-				runtime.Auth.Type = RegistryAuthTypeDefault
-			}
-		}
-		for _, container := range runtime.Containers {
-			if container.Auth != nil {
-				if container.Auth.Type == "" {
-					container.Auth.Type = RegistryAuthTypeDefault
-				}
-			}
-		}
-	}
-
-	// set steps defaults
-	for _, t := range config.Tasks {
-		for _, s := range t.Steps {
-			switch step := s.(type) {
-			// TODO(sgotti) we could use the run step command as step name but when the
-			// command is very long or multi line it doesn't makes sense and will
-			// probably be quite unuseful/confusing from an UI point of view
-			case *SaveCacheStep:
-				for _, content := range step.Contents {
-					if len(content.Paths) == 0 {
-						// default to all files inside the sourceDir
-						content.Paths = []string{"**"}
-					}
-				}
-				log.Infof("s: %s", util.Dump(s))
-			}
-		}
-	}
-
 	return &config, checkConfig(&config)
 }
 
 func checkConfig(config *Config) error {
+	if len(config.Runs) == 0 {
+		return errors.Errorf("no runs defined")
+	}
+
+	seenRuns := map[string]struct{}{}
+	for ri, run := range config.Runs {
+		if run == nil {
+			return errors.Errorf("run at index %d is empty", ri)
+		}
+
+		if run.Name == "" {
+			return errors.Errorf("run at index %d has empty name", ri)
+		}
+
+		if len(run.Name) > maxRunNameLength {
+			return errors.Errorf("run name %q too long", run.Name)
+		}
+
+		if _, ok := seenRuns[run.Name]; ok {
+			return errors.Errorf("duplicate run name: %s", run.Name)
+		}
+		seenRuns[run.Name] = struct{}{}
+
+		seenTasks := map[string]struct{}{}
+		for ti, task := range run.Tasks {
+			if task == nil {
+				return errors.Errorf("run %q: task at index %d is empty", run.Name, ti)
+			}
+
+			if task.Name == "" {
+				return errors.Errorf("run %q: task at index %d has empty name", run.Name, ti)
+			}
+
+			if len(task.Name) > maxTaskNameLength {
+				return errors.Errorf("task name %q too long", task.Name)
+			}
+
+			if _, ok := seenTasks[task.Name]; ok {
+				return errors.Errorf("duplicate task name: %s", task.Name)
+			}
+			seenTasks[task.Name] = struct{}{}
+
+			// check tasks runtime
+			if task.Runtime == nil {
+				return errors.Errorf("task %q: runtime is not defined", task.Name)
+			}
+
+			r := task.Runtime
+			if r.Type != RuntimeTypePod {
+				return errors.Errorf("task %q runtime: wrong type %q", task.Name, r.Type)
+			}
+			if len(r.Containers) == 0 {
+				return errors.Errorf("task %q runtime: at least one container must be defined", task.Name)
+			}
+		}
+	}
+
 	// check broken dependencies
 	for _, run := range config.Runs {
 		// collect all task names
-		allElements := map[string]struct{}{}
+		allTasks := map[string]struct{}{}
-		for _, element := range run.Elements {
+		for _, task := range run.Tasks {
-			allElements[element.Name] = struct{}{}
+			allTasks[task.Name] = struct{}{}
 		}
 
-		for _, element := range run.Elements {
+		for _, task := range run.Tasks {
-			for _, dep := range element.Depends {
+			for _, dep := range task.Depends {
-				if _, ok := allElements[dep.ElementName]; !ok {
+				if _, ok := allTasks[dep.TaskName]; !ok {
-					return errors.Errorf("run element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
+					return errors.Errorf("run task %q needed by task %q doesn't exist", dep.TaskName, task.Name)
 				}
 			}
 		}
@@ -643,21 +682,21 @@ func checkConfig(config *Config) error {
 	// check circular dependencies
 	for _, run := range config.Runs {
 		cerrs := &util.Errors{}
-		for _, element := range run.Elements {
+		for _, task := range run.Tasks {
-			allParents := getAllElementParents(run, element)
+			allParents := getAllTaskParents(run, task)
 			for _, parent := range allParents {
-				if parent.Name == element.Name {
+				if parent.Name == task.Name {
 					// TODO(sgotti) get the parent that depends on task to report it
 					dep := []string{}
 					for _, parent := range allParents {
-						pparents := getElementParents(run, parent)
+						pparents := getTaskParents(run, parent)
 						for _, pparent := range pparents {
-							if pparent.Name == element.Name {
+							if pparent.Name == task.Name {
 								dep = append(dep, fmt.Sprintf("%q", parent.Name))
 							}
 						}
 					}
-					cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " ")))
+					cerrs.Append(errors.Errorf("circular dependency between task %q and tasks %s", task.Name, strings.Join(dep, " ")))
 				}
 			}
 		}
@@ -668,15 +707,15 @@ func checkConfig(config *Config) error {
 
 	// check that the task and its parent don't have a common dependency
 	for _, run := range config.Runs {
-		for _, element := range run.Elements {
+		for _, task := range run.Tasks {
-			parents := getElementParents(run, element)
+			parents := getTaskParents(run, task)
 			for _, parent := range parents {
-				allParents := getAllElementParents(run, element)
+				allParents := getAllTaskParents(run, task)
-				allParentParents := getAllElementParents(run, parent)
+				allParentParents := getAllTaskParents(run, parent)
 				for _, p := range allParents {
 					for _, pp := range allParentParents {
 						if p.Name == pp.Name {
-							return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name)
+							return errors.Errorf("task %s and its dependency %s have both a dependency on task %s", task.Name, parent.Name, p.Name)
 						}
 					}
 				}
@@ -684,66 +723,66 @@ func checkConfig(config *Config) error {
 		}
 	}
 
-	for _, r := range config.Runtimes {
+	// check duplicate task dependencies
-		if r.Type != RuntimeTypePod {
-			return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type)
-		}
-		if len(r.Containers) == 0 {
-			return errors.Errorf("runtime %q: at least one container must be defined", r.Name)
-		}
-	}
-
-	for _, t := range config.Tasks {
-		if len(t.Name) > maxTaskNameLength {
-			return errors.Errorf("task name %q too long", t.Name)
-		}
-		if t.Runtime == "" {
-			return errors.Errorf("task %q: undefined runtime", t.Name)
-		}
-		if _, ok := config.Runtimes[t.Runtime]; !ok {
-			return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name)
-		}
-		for i, s := range t.Steps {
-			switch step := s.(type) {
-			// TODO(sgotti) we could use the run step command as step name but when the
-			// command is very long or multi line it doesn't makes sense and will
-			// probably be quite unuseful/confusing from an UI point of view
-			case *RunStep:
-				if step.Name == "" {
-					lines, err := util.CountLines(step.Command)
-					// if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred
-					if err != nil || lines > 1 {
-						return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name)
-					}
-					len := len(step.Command)
-					if len > maxStepNameLength {
-						len = maxStepNameLength
-					}
-					step.Name = step.Command[:len]
-				}
-			}
-		}
-	}
-
 	for _, run := range config.Runs {
-		if len(run.Name) > maxRunNameLength {
+		for _, task := range run.Tasks {
-			return errors.Errorf("run name %q too long", run.Name)
-		}
-		for _, element := range run.Elements {
-			// check missing tasks reference
-			if element.Task == "" {
-				return errors.Errorf("no task defined for run element %q", element.Name)
-			}
-			if _, ok := config.Tasks[element.Task]; !ok {
-				return errors.Errorf("task %q needed by run element %q doesn't exist", element.Task, element.Name)
-			}
 			// check duplicate dependencies in task
 			seenDependencies := map[string]struct{}{}
-			for _, dep := range element.Depends {
+			for _, dep := range task.Depends {
-				if _, ok := seenDependencies[dep.ElementName]; ok {
+				if _, ok := seenDependencies[dep.TaskName]; ok {
-					return errors.Errorf("duplicate task dependency: %s", element.Name)
+					return errors.Errorf("duplicate task dependency: %s", task.Name)
+				}
+				seenDependencies[dep.TaskName] = struct{}{}
+			}
+		}
+	}
+
+	// Set defaults
+	for _, run := range config.Runs {
+		for _, task := range run.Tasks {
+			// Set auth type to default if not specified
+			runtime := task.Runtime
+			if runtime.Auth != nil {
+				if runtime.Auth.Type == "" {
+					runtime.Auth.Type = RegistryAuthTypeDefault
+				}
+			}
+			for _, container := range runtime.Containers {
+				if container.Auth != nil {
+					if container.Auth.Type == "" {
+						container.Auth.Type = RegistryAuthTypeDefault
+					}
+				}
+			}
+
+			// set steps defaults
+			for i, s := range task.Steps {
+				switch step := s.(type) {
+				// TODO(sgotti) we could use the run step command as step name but when the
+				// command is very long or multi line it doesn't makes sense and will
+				// probably be quite unuseful/confusing from an UI point of view
+				case *RunStep:
+					if step.Name == "" {
+						lines, err := util.CountLines(step.Command)
+						// if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred
+						if err != nil || lines > 1 {
+							return errors.Errorf("missing step name for step %d (run) in task %q, required since command is more than one line", i, task.Name)
+						}
+						len := len(step.Command)
+						if len > maxStepNameLength {
+							len = maxStepNameLength
+						}
+						step.Name = step.Command[:len]
+					}
+
+				case *SaveCacheStep:
+					for _, content := range step.Contents {
+						if len(content.Paths) == 0 {
+							// default to all files inside the sourceDir
+							content.Paths = []string{"**"}
+						}
+					}
 				}
-				seenDependencies[dep.ElementName] = struct{}{}
 			}
 		}
 	}
@@ -751,13 +790,13 @@ func checkConfig(config *Config) error {
 	return nil
 }
 
-// getElementParents returns direct parents of element.
+// getTaskParents returns direct parents of task.
-func getElementParents(run *Run, element *Element) []*Element {
+func getTaskParents(run *Run, task *Task) []*Task {
-	parents := []*Element{}
+	parents := []*Task{}
-	for _, el := range run.Elements {
+	for _, el := range run.Tasks {
 		isParent := false
-		for _, d := range element.Depends {
+		for _, d := range task.Depends {
-			if d.ElementName == el.Name {
+			if d.TaskName == el.Name {
 				isParent = true
 			}
 		}
@@ -768,26 +807,26 @@ func getElementParents(run *Run, element *Element) []*Element {
 	return parents
 }
 
-// getAllElementParents returns all the parents (both direct and ancestors) of an element.
+// getAllTaskParents returns all the parents (both direct and ancestors) of a task.
 // In case of circular dependency it won't loop forever but will also return
-// the element as parent of itself
+// the task as parent of itself
-func getAllElementParents(run *Run, element *Element) []*Element {
+func getAllTaskParents(run *Run, task *Task) []*Task {
-	pMap := map[string]*Element{}
+	pMap := map[string]*Task{}
-	nextParents := getElementParents(run, element)
+	nextParents := getTaskParents(run, task)
 
 	for len(nextParents) > 0 {
 		parents := nextParents
-		nextParents = []*Element{}
+		nextParents = []*Task{}
 		for _, parent := range parents {
 			if _, ok := pMap[parent.Name]; ok {
 				continue
 			}
 			pMap[parent.Name] = parent
-			nextParents = append(nextParents, getElementParents(run, parent)...)
+			nextParents = append(nextParents, getTaskParents(run, parent)...)
 		}
 	}
 
-	parents := make([]*Element, 0, len(pMap))
+	parents := make([]*Task, 0, len(pMap))
 	for _, v := range pMap {
 		parents = append(parents, v)
 	}

internal/config/config_test.go

@@ -47,52 +47,61 @@ func TestParseConfig(t *testing.T) {
 			name: "test empty run",
 			in: `
                 runs:
-                  run01:
+                  -
                 `,
-			err: fmt.Errorf(`run "run01" is empty`),
+			err: fmt.Errorf(`run at index 0 is empty`),
 		},
 		{
-			name: "test missing element dependency",
+			name: "test empty task",
 			in: `
-                tasks:
-                  task0k1:
-                    environment:
-                      ENV01: ENV01
-
                 runs:
-                  run01:
+                  - name: run01
-                    elements:
+                    tasks:
-                      element01:
+                      - 
-                        task: task01
-                        depends:
-                          - element02
                 `,
-			err: fmt.Errorf(`run element "element02" needed by element "element01" doesn't exist`),
+			err: fmt.Errorf(`run "run01": task at index 0 is empty`),
 		},
 		{
-			name: "test circular dependency between 2 elements a -> b -> a",
+			name: "test missing task dependency",
 			in: `
-                tasks:
-                  task01:
-                    environment:
-                      ENV01: ENV01
-
                 runs:
-                  run01:
+                  - name: run01
-                    elements:
+                    tasks:
-                      element01:
+                      - name: task01
-                        task: task01
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: busybox
                         depends:
-                          - element02
+                          - task02
-                      element02:
+                `,
-                        task: task01
+			err: fmt.Errorf(`run task "task02" needed by task "task01" doesn't exist`),
+		},
+		{
+			name: "test circular dependency between 2 tasks a -> b -> a",
+			in: `
+                runs:
+                  - name: run01
+                    tasks:
+                      - name: task01
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: busybox
                         depends:
-                          - element01
+                          - task02
+                      - name: task02
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: busybox
+                        depends:
+                          - task01
                 `,
 			err: &util.Errors{
 				Errs: []error{
-					errors.Errorf("circular dependency between element %q and elements %q", "element01", "element02"),
+					errors.Errorf("circular dependency between task %q and tasks %q", "task01", "task02"),
-					errors.Errorf("circular dependency between element %q and elements %q", "element02", "element01"),
+					errors.Errorf("circular dependency between task %q and tasks %q", "task02", "task01"),
 				},
 			},
 		},
@@ -129,54 +138,67 @@ func TestParseOutput(t *testing.T) {
 		out  *Config
 	}{
 		{
-			name: "test element when conditions",
+			name: "test task all options",
 			in: `
-                runtimes:
+                runs:
-                  runtime01:
+                  - name: run01
-                    type: pod
+                    tasks:
-                    auth:
+                      - name: task01
-                      username: username
+                        runtime:
-                      password:
+                          type: pod
-                        from_variable: password
+                          auth:
-                    containers:
+                            username: username
-                      - image: image01
+                            password:
-                        auth:
+                              from_variable: password
-                          username:
+                          containers:
-                            from_variable: username2
+                            - image: image01
-                          password: password2
+                              auth:
+                                username:
+                                  from_variable: username2
+                                password: password2
+                              environment:
+                                ENV01: ENV01
+                                ENVFROMVARIABLE01:
+                                  from_variable: variable01
                         environment:
                           ENV01: ENV01
                           ENVFROMVARIABLE01:
                             from_variable: variable01
+                        steps:
+                          # normal step definition
+                          - type: clone
+                          - type: run
+                            command: command01
+                          - type: run
+                            name: name different than command
+                            command: command02
+                          - type: run
+                            command: command03
+                            environment:
+                              ENV01: ENV01
+                              ENVFROMVARIABLE01:
+                                from_variable: variable01
+                          - type: save_cache
+                            key: cache-{{ arch }}
+                            contents:
+                              - source_dir: /go/pkg/mod/cache
 
-                tasks:
+                          # simpler (for yaml not for json) steps definition
-                  task01:
+                          - clone:
-                    runtime: runtime01
+                          - run: command01
-                    environment:
+                          - run:
-                      ENV01: ENV01
+                              name: name different than command
-                      ENVFROMVARIABLE01:
+                              command: command02
-                        from_variable: variable01
+                          - run:
-                    steps:
+                              command: command03
-                      - run: command01
+                              environment:
-                      - run:
+                                ENV01: ENV01
-                          name: name different than command
+                                ENVFROMVARIABLE01:
-                          command: command02
+                                  from_variable: variable01
-                      - run:
+                          - save_cache:
-                          command: command03
+                              key: cache-{{ arch }}
-                          environment:
+                              contents:
-                            ENV01: ENV01
+                                - source_dir: /go/pkg/mod/cache
-                            ENVFROMVARIABLE01:
-                              from_variable: variable01
-                      - save_cache:
-                          key: cache-{{ arch }}
-                          contents:
-                            - source_dir: /go/pkg/mod/cache
-
-                runs:
-                  run01:
-                    elements:
-                      element01:
-                        task: task01
                         when:
                           branch: master
                           tag:
@@ -185,88 +207,132 @@ func TestParseOutput(t *testing.T) {
                           ref:
                             include: master
                             exclude: [ /branch01/ , branch02 ]
+                        depends:
+                          - task: task02
+                            conditions:
+                              - on_success
+                              - on_failure
+                          - task03
+                          - task04:
+                            - on_success
+                      - name: task02
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: image01
+                      - name: task03
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: image01
+                      - name: task04
+                        runtime:
+                          type: pod
+                          containers:
+                            - image: image01
           `,
 			out: &Config{
-				Runtimes: map[string]*Runtime{
+				Runs: []*Run{
-					"runtime01": &Runtime{
+					&Run{
-						Name: "runtime01",
-						Type: "pod",
-						Auth: &RegistryAuth{
-							Type:     RegistryAuthTypeDefault,
-							Username: Value{Type: ValueTypeString, Value: "username"},
-							Password: Value{Type: ValueTypeFromVariable, Value: "password"},
-						},
-						Arch: "",
-						Containers: []*Container{
-							&Container{
-								Image: "image01",
-								Auth: &RegistryAuth{
-									Type:     RegistryAuthTypeDefault,
-									Username: Value{Type: ValueTypeFromVariable, Value: "username2"},
-									Password: Value{Type: ValueTypeString, Value: "password2"},
-								},
-								Environment: map[string]Value{
-									"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
-									"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
-								},
-								User: "",
-							},
-						},
-					},
-				},
-				Tasks: map[string]*Task{
-					"task01": &Task{
-						Name:    "task01",
-						Runtime: "runtime01",
-						Environment: map[string]Value{
-							"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
-							"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
-						},
-						WorkingDir: "",
-						Shell:      "",
-						User:       "",
-						Steps: []interface{}{
-							&RunStep{
-								Step: Step{
-									Type: "run",
-									Name: "command01",
-								},
-								Command: "command01",
-							},
-							&RunStep{
-								Step: Step{
-									Type: "run",
-									Name: "name different than command",
-								},
-								Command: "command02",
-							},
-							&RunStep{
-								Step: Step{
-									Type: "run",
-									Name: "command03",
-								},
-								Command: "command03",
-								Environment: map[string]Value{
-									"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
-									"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
-								},
-							},
-							&SaveCacheStep{
-								Step:     Step{Type: "save_cache"},
-								Key:      "cache-{{ arch }}",
-								Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}},
-							},
-						},
-					},
-				},
-				Runs: map[string]*Run{
-					"run01": &Run{
 						Name: "run01",
-						Elements: map[string]*Element{
+						Tasks: []*Task{
-							"element01": &Element{
+							&Task{
-								Name:          "element01",
+								Name: "task01",
-								Task:          "task01",
+								Runtime: &Runtime{
-								Depends:       []*Depend{},
+									Type: "pod",
+									Auth: &RegistryAuth{
+										Type:     RegistryAuthTypeDefault,
+										Username: Value{Type: ValueTypeString, Value: "username"},
+										Password: Value{Type: ValueTypeFromVariable, Value: "password"},
+									},
+									Arch: "",
+									Containers: []*Container{
+										&Container{
+											Image: "image01",
+											Auth: &RegistryAuth{
+												Type:     RegistryAuthTypeDefault,
+												Username: Value{Type: ValueTypeFromVariable, Value: "username2"},
+												Password: Value{Type: ValueTypeString, Value: "password2"},
+											},
+											Environment: map[string]Value{
+												"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
+												"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
+											},
+											User: "",
+										},
+									},
+								},
+								Environment: map[string]Value{
+									"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
+								},
+								WorkingDir: "",
+								Shell:      "",
+								User:       "",
+								Steps: []interface{}{
+									&CloneStep{Step: Step{Type: "clone"}},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "command01",
+										},
+										Command: "command01",
+									},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "name different than command",
+										},
+										Command: "command02",
+									},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "command03",
+										},
+										Command: "command03",
+										Environment: map[string]Value{
+											"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
+											"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
+										},
+									},
+									&SaveCacheStep{
+										Step:     Step{Type: "save_cache"},
+										Key:      "cache-{{ arch }}",
+										Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}},
+									},
+									&CloneStep{Step: Step{Type: "clone"}},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "command01",
+										},
+										Command: "command01",
+									},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "name different than command",
+										},
+										Command: "command02",
+									},
+									&RunStep{
+										Step: Step{
+											Type: "run",
+											Name: "command03",
+										},
+										Command: "command03",
+										Environment: map[string]Value{
+											"ENV01":             Value{Type: ValueTypeString, Value: "ENV01"},
+											"ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"},
+										},
+									},
+									&SaveCacheStep{
+										Step:     Step{Type: "save_cache"},
+										Key:      "cache-{{ arch }}",
+										Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}},
+									},
+								},
 								IgnoreFailure: false,
 								Approval:      false,
 								When: &types.When{
@@ -291,6 +357,56 @@ func TestParseOutput(t *testing.T) {
 										},
 									},
 								},
+								Depends: []*Depend{
+									&Depend{TaskName: "task02", Conditions: []DependCondition{DependConditionOnSuccess, DependConditionOnFailure}},
+									&Depend{TaskName: "task03", Conditions: nil},
+									&Depend{TaskName: "task04", Conditions: []DependCondition{DependConditionOnSuccess}},
+								},
+							},
+							&Task{
+								Name: "task02",
+								Runtime: &Runtime{
+									Type: "pod",
+									Arch: "",
+									Containers: []*Container{
+										&Container{
+											Image: "image01",
+										},
+									},
+								},
+								WorkingDir: "",
+								Steps:      []interface{}{},
+								Depends:    []*Depend{},
+							},
+							&Task{
+								Name: "task03",
+								Runtime: &Runtime{
+									Type: "pod",
+									Arch: "",
+									Containers: []*Container{
+										&Container{
+											Image: "image01",
+										},
+									},
+								},
+								WorkingDir: "",
+								Steps:      []interface{}{},
+								Depends:    []*Depend{},
+							},
+							&Task{
+								Name: "task04",
+								Runtime: &Runtime{
+									Type: "pod",
+									Arch: "",
+									Containers: []*Container{
+										&Container{
+											Image: "image01",
+										},
+									},
+								},
+								WorkingDir: "",
+								Steps:      []interface{}{},
+								Depends:    []*Depend{},
 							},
 						},
 					},

internal/runconfig/runconfig.go

@@ -25,9 +25,7 @@ import (
 	"github.com/sorintlab/agola/internal/util"
 )
 
-func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime {
+func genRuntime(c *config.Config, ce *config.Runtime, variables map[string]string) *rstypes.Runtime {
-	ce := c.Runtime(runtimeName)
-
 	containers := []*rstypes.Container{}
 	for _, cc := range ce.Containers {
 		env := genEnv(cc.Environment, variables)
@@ -189,11 +187,8 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string
 
 	rcts := map[string]*rstypes.RunConfigTask{}
 
-	for _, cre := range cr.Elements {
+	for _, ct := range cr.Tasks {
-		include := types.MatchWhen(cre.When, branch, tag, ref)
+		include := types.MatchWhen(ct.When, branch, tag, ref)
-
-		// resolve referenced task
-		ct := c.Task(cre.Task)
 
 		steps := make([]interface{}, len(ct.Steps))
 		for i, cpts := range ct.Steps {
@@ -203,18 +198,17 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string
 		tEnv := genEnv(ct.Environment, variables)
 
 		t := &rstypes.RunConfigTask{
-			ID: uuid.New(cre.Name).String(),
+			ID:            uuid.New(ct.Name).String(),
-			// use the element name from the config as the task name
+			Name:          ct.Name,
-			Name:          cre.Name,
 			Runtime:       genRuntime(c, ct.Runtime, variables),
 			Environment:   tEnv,
 			WorkingDir:    ct.WorkingDir,
 			Shell:         ct.Shell,
 			User:          ct.User,
 			Steps:         steps,
-			IgnoreFailure: cre.IgnoreFailure,
+			IgnoreFailure: ct.IgnoreFailure,
 			Skip:          !include,
-			NeedsApproval: cre.Approval,
+			NeedsApproval: ct.Approval,
 		}
 
 		rcts[t.ID] = t
@@ -222,10 +216,10 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string
 
 	// populate depends, needs to be done after having created all the tasks so we can resolve their id
 	for _, rct := range rcts {
-		cre := cr.Elements[rct.Name]
+		ct := cr.Task(rct.Name)
 
-		depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cre.Depends))
+		depends := make(map[string]*rstypes.RunConfigTaskDepend, len(ct.Depends))
-		for _, d := range cre.Depends {
+		for _, d := range ct.Depends {
 			conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
 			// when no conditions are defined default to on_success
 			if len(d.Conditions) == 0 {
@@ -243,7 +237,7 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string
 				}
 			}
 
-			drct := getRunConfigTaskByName(rcts, d.ElementName)
+			drct := getRunConfigTaskByName(rcts, d.TaskName)
 			depends[drct.ID] = &rstypes.RunConfigTaskDepend{
 				TaskID:     drct.ID,
 				Conditions: conditions,

internal/runconfig/runconfig_test.go

@@ -632,80 +632,71 @@ func TestGenRunConfig(t *testing.T) {
 		{
 			name: "test runconfig generation",
 			in: &config.Config{
-				Runtimes: map[string]*config.Runtime{
+				Runs: []*config.Run{
-					"runtime01": &config.Runtime{
+					&config.Run{
-						Name: "runtime01",
-						Type: "pod",
-						Auth: &config.RegistryAuth{
-							Type:     config.RegistryAuthTypeDefault,
-							Username: config.Value{Type: config.ValueTypeString, Value: "username"},
-							Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
-						},
-						Arch: "",
-						Containers: []*config.Container{
-							&config.Container{
-								Image: "image01",
-								Auth: &config.RegistryAuth{
-									Type:     config.RegistryAuthTypeDefault,
-									Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"},
-									Password: config.Value{Type: config.ValueTypeString, Value: "password2"},
-								},
-								Environment: map[string]config.Value{
-									"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
-									"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
-								},
-								User: "",
-							},
-						},
-					},
-				},
-				Tasks: map[string]*config.Task{
-					"task01": &config.Task{
-						Name:    "task01",
-						Runtime: "runtime01",
-						Environment: map[string]config.Value{
-							"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
-							"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
-						},
-						WorkingDir: "",
-						Shell:      "",
-						User:       "",
-						Steps: []interface{}{
-							&config.RunStep{
-								Step: config.Step{
-									Type: "run",
-									Name: "command01",
-								},
-								Command: "command01",
-							},
-							&config.RunStep{
-								Step: config.Step{
-									Type: "run",
-									Name: "name different than command",
-								},
-								Command: "command02",
-							},
-							&config.RunStep{
-								Step: config.Step{
-									Type: "run",
-									Name: "command03",
-								},
-								Command: "command03",
-								Environment: map[string]config.Value{
-									"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
-									"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
-								},
-							},
-						},
-					},
-				},
-				Runs: map[string]*config.Run{
-					"run01": &config.Run{
 						Name: "run01",
-						Elements: map[string]*config.Element{
+						Tasks: []*config.Task{
-							"element01": &config.Element{
+							&config.Task{
-								Name:          "element01",
+								Name: "task01",
-								Task:          "task01",
+								Runtime: &config.Runtime{
+									Type: "pod",
+									Auth: &config.RegistryAuth{
+										Type:     config.RegistryAuthTypeDefault,
+										Username: config.Value{Type: config.ValueTypeString, Value: "username"},
+										Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
+									},
+									Arch: "",
+									Containers: []*config.Container{
+										&config.Container{
+											Image: "image01",
+											Auth: &config.RegistryAuth{
+												Type:     config.RegistryAuthTypeDefault,
+												Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"},
+												Password: config.Value{Type: config.ValueTypeString, Value: "password2"},
+											},
+											Environment: map[string]config.Value{
+												"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
+												"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
+											},
+											User: "",
+										},
+									},
+								},
+								Environment: map[string]config.Value{
+									"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
+								},
+								WorkingDir: "",
+								Shell:      "",
+								User:       "",
+								Steps: []interface{}{
+									&config.RunStep{
+										Step: config.Step{
+											Type: "run",
+											Name: "command01",
+										},
+										Command: "command01",
+									},
+									&config.RunStep{
+										Step: config.Step{
+											Type: "run",
+											Name: "name different than command",
+										},
+										Command: "command02",
+									},
+									&config.RunStep{
+										Step: config.Step{
+											Type: "run",
+											Name: "command03",
+										},
+										Command: "command03",
+										Environment: map[string]config.Value{
+											"ENV01":             config.Value{Type: config.ValueTypeString, Value: "ENV01"},
+											"ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"},
+										},
+									},
+								},
+
 								Depends:       []*config.Depend{},
 								IgnoreFailure: false,
 								Approval:      false,
@@ -727,9 +718,9 @@ func TestGenRunConfig(t *testing.T) {
 				"registry_username": "yourregistryusername",
 			},
 			out: map[string]*rstypes.RunConfigTask{
-				uuid.New("element01").String(): &rstypes.RunConfigTask{
+				uuid.New("task01").String(): &rstypes.RunConfigTask{
-					ID:   uuid.New("element01").String(),
+					ID:   uuid.New("task01").String(),
-					Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
+					Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
 					Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
 						Containers: []*rstypes.Container{
 							{
@@ -762,45 +753,35 @@ func TestGenRunConfig(t *testing.T) {
 		{
 			name: "test runtime auth used for container nil auth",
 			in: &config.Config{
-				Runtimes: map[string]*config.Runtime{
+				Runs: []*config.Run{
-					"runtime01": &config.Runtime{
+					&config.Run{
-						Name: "runtime01",
-						Type: "pod",
-						Auth: &config.RegistryAuth{
-							Type:     config.RegistryAuthTypeDefault,
-							Username: config.Value{Type: config.ValueTypeString, Value: "username"},
-							Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
-						},
-						Arch: "",
-						Containers: []*config.Container{
-							&config.Container{
-								Image: "image01",
-							},
-						},
-					},
-				},
-				Tasks: map[string]*config.Task{
-					"task01": &config.Task{
-						Name:    "task01",
-						Runtime: "runtime01",
-						Steps: []interface{}{
-							&config.RunStep{
-								Step: config.Step{
-									Type: "run",
-									Name: "command01",
-								},
-								Command: "command01",
-							},
-						},
-					},
-				},
-				Runs: map[string]*config.Run{
-					"run01": &config.Run{
 						Name: "run01",
-						Elements: map[string]*config.Element{
+						Tasks: []*config.Task{
-							"element01": &config.Element{
+							&config.Task{
-								Name: "element01",
+								Name: "task01",
-								Task: "task01",
+								Runtime: &config.Runtime{
+									Type: "pod",
+									Auth: &config.RegistryAuth{
+										Type:     config.RegistryAuthTypeDefault,
+										Username: config.Value{Type: config.ValueTypeString, Value: "username"},
+										Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
+									},
+									Arch: "",
+									Containers: []*config.Container{
+										&config.Container{
+											Image: "image01",
+										},
+									},
+								},
+								Steps: []interface{}{
+									&config.RunStep{
+										Step: config.Step{
+											Type: "run",
+											Name: "command01",
+										},
+										Command: "command01",
+									},
+								},
 							},
 						},
 					},
@@ -811,9 +792,9 @@ func TestGenRunConfig(t *testing.T) {
 				"password":   "yourregistrypassword",
 			},
 			out: map[string]*rstypes.RunConfigTask{
-				uuid.New("element01").String(): &rstypes.RunConfigTask{
+				uuid.New("task01").String(): &rstypes.RunConfigTask{
-					ID:   uuid.New("element01").String(),
+					ID:   uuid.New("task01").String(),
-					Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
+					Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
 					Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
 						Containers: []*rstypes.Container{
 							{
@@ -837,50 +818,40 @@ func TestGenRunConfig(t *testing.T) {
 		{
 			name: "test runtime auth not used for container with auth",
 			in: &config.Config{
-				Runtimes: map[string]*config.Runtime{
+				Runs: []*config.Run{
-					"runtime01": &config.Runtime{
+					&config.Run{
-						Name: "runtime01",
-						Type: "pod",
-						Auth: &config.RegistryAuth{
-							Type:     config.RegistryAuthTypeDefault,
-							Username: config.Value{Type: config.ValueTypeString, Value: "username"},
-							Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
-						},
-						Arch: "",
-						Containers: []*config.Container{
-							&config.Container{
-								Image: "image01",
-								Auth: &config.RegistryAuth{
-									Type:     config.RegistryAuthTypeDefault,
-									Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"},
-									Password: config.Value{Type: config.ValueTypeString, Value: "password2"},
-								},
-							},
-						},
-					},
-				},
-				Tasks: map[string]*config.Task{
-					"task01": &config.Task{
-						Name:    "task01",
-						Runtime: "runtime01",
-						Steps: []interface{}{
-							&config.RunStep{
-								Step: config.Step{
-									Type: "run",
-									Name: "command01",
-								},
-								Command: "command01",
-							},
-						},
-					},
-				},
-				Runs: map[string]*config.Run{
-					"run01": &config.Run{
 						Name: "run01",
-						Elements: map[string]*config.Element{
+						Tasks: []*config.Task{
-							"element01": &config.Element{
+							&config.Task{
-								Name: "element01",
+								Name: "task01",
-								Task: "task01",
+								Runtime: &config.Runtime{
+									Type: "pod",
+									Auth: &config.RegistryAuth{
+										Type:     config.RegistryAuthTypeDefault,
+										Username: config.Value{Type: config.ValueTypeString, Value: "username"},
+										Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"},
+									},
+									Arch: "",
+									Containers: []*config.Container{
+										&config.Container{
+											Image: "image01",
+											Auth: &config.RegistryAuth{
+												Type:     config.RegistryAuthTypeDefault,
+												Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"},
+												Password: config.Value{Type: config.ValueTypeString, Value: "password2"},
+											},
+										},
+									},
+								},
+								Steps: []interface{}{
+									&config.RunStep{
+										Step: config.Step{
+											Type: "run",
+											Name: "command01",
+										},
+										Command: "command01",
+									},
+								},
 							},
 						},
 					},
@@ -891,9 +862,9 @@ func TestGenRunConfig(t *testing.T) {
 				"registry_username": "yourregistryusername",
 			},
 			out: map[string]*rstypes.RunConfigTask{
-				uuid.New("element01").String(): &rstypes.RunConfigTask{
+				uuid.New("task01").String(): &rstypes.RunConfigTask{
-					ID:   uuid.New("element01").String(),
+					ID:   uuid.New("task01").String(),
-					Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
+					Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{},
 					Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
 						Containers: []*rstypes.Container{
 							{