Add initial agola config format and handling

internal/config/config.go

@@ -0,0 +1,511 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sorintlab/agola/internal/common"
+	"github.com/sorintlab/agola/internal/util"
+
+	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
+)
+
+const (
+	maxPipelineNameLength = 100
+	maxTaskNameLength     = 100
+	maxStepNameLength     = 100
+)
+
+type Config struct {
+	Runtimes  map[string]*Runtime  `yaml:"runtimes"`
+	Tasks     map[string]*Task     `yaml:"tasks"`
+	Pipelines map[string]*Pipeline `yaml:"pipelines"`
+}
+
+type Task struct {
+	Name        string            `yaml:"name"`
+	Runtime     string            `yaml:"runtime"`
+	Environment map[string]string `yaml:"environment"`
+	WorkingDir  string            `yaml:"working_dir"`
+	Shell       string            `yaml:"shell"`
+	User        string            `yaml:"user"`
+	Steps       []interface{}     `yaml:"steps"`
+}
+
+type RuntimeType string
+
+const (
+	RuntimeTypePod RuntimeType = "pod"
+)
+
+type Runtime struct {
+	Name       string       `yaml:"name"`
+	Type       RuntimeType  `yaml:"type,omitempty"`
+	Arch       common.Arch  `yaml:"arch,omitempty"`
+	Containers []*Container `yaml:"containers,omitempty"`
+}
+
+type Container struct {
+	Image       string            `yaml:"image,omitempty"`
+	Environment map[string]string `yaml:"environment,omitempty"`
+	User        string            `yaml:"user"`
+}
+
+type Pipeline struct {
+	Name     string              `yaml:"name"`
+	Elements map[string]*Element `yaml:"elements"`
+}
+
+type Element struct {
+	Name          string    `yaml:"name"`
+	Task          string    `yaml:"task"`
+	Depends       []*Depend `yaml:"depends"`
+	IgnoreFailure bool      `yaml:"ignore_failure"`
+	Approval      bool      `yaml:"approval"`
+}
+
+type DependCondition string
+
+const (
+	DependConditionOnSuccess DependCondition = "on_success"
+	DependConditionOnFailure DependCondition = "on_failure"
+)
+
+type Depend struct {
+	ElementName string            `yaml:"name"`
+	Conditions  []DependCondition `yaml: "conditions"`
+}
+
+type Step struct {
+	Type string `yaml:"type"`
+	Name string `yaml:"name"`
+}
+
+type CloneStep struct {
+	Step `yaml:",inline"`
+}
+
+type RunStep struct {
+	Step        `yaml:",inline"`
+	Command     string            `yaml:"command"`
+	Environment map[string]string `yaml:"environment,omitempty"`
+	WorkingDir  string            `yaml:"working_dir"`
+	Shell       string            `yaml:"shell"`
+	User        string            `yaml:"user"`
+}
+
+type SaveToWorkspaceContent struct {
+	SourceDir string   `yaml:"source_dir"`
+	DestDir   string   `yaml:"dest_dir"`
+	Paths     []string `yaml:"paths"`
+}
+
+type SaveToWorkspaceStep struct {
+	Step     `yaml:",inline"`
+	Contents []SaveToWorkspaceContent
+}
+
+type RestoreWorkspaceStep struct {
+	Step    `yaml:",inline"`
+	DestDir string `yaml:"dest_dir"`
+}
+
+func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type task Task
+	type tasksteps struct {
+		Steps []map[string]interface{} `yaml:"steps"`
+	}
+	tt := (*task)(t)
+	if err := unmarshal(&tt); err != nil {
+		return err
+	}
+	log.Debugf("tt: %s", util.Dump(tt))
+
+	var st tasksteps
+	if err := unmarshal(&st); err != nil {
+		return err
+	}
+
+	steps := make([]interface{}, len(tt.Steps))
+	for i, stepEntry := range st.Steps {
+		if len(stepEntry) > 1 {
+			return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
+		}
+		for stepType, stepSpec := range stepEntry {
+			log.Debugf("s: %s", util.Dump(stepSpec))
+			o, err := yaml.Marshal(stepSpec)
+			if err != nil {
+				return err
+			}
+			log.Debugf("o: %s", o)
+			switch stepType {
+			case "clone":
+				var cs CloneStep
+				cs.Type = stepType
+				steps[i] = &cs
+
+			case "run":
+				var rs RunStep
+				rs.Type = stepType
+				switch stepSpec.(type) {
+				case string:
+					rs.Command = stepSpec.(string)
+				default:
+					if err := yaml.Unmarshal(o, &rs); err != nil {
+						return err
+					}
+				}
+				steps[i] = &rs
+
+			case "save_to_workspace":
+				var sws SaveToWorkspaceStep
+				sws.Type = stepType
+				if err := yaml.Unmarshal(o, &sws); err != nil {
+					return err
+				}
+				steps[i] = &sws
+
+			case "restore_workspace":
+				var rws RestoreWorkspaceStep
+				rws.Type = stepType
+				if err := yaml.Unmarshal(o, &rws); err != nil {
+					return err
+				}
+				steps[i] = &rws
+			default:
+				return errors.Errorf("unknown step type: %s", stepType)
+			}
+			log.Debugf("s: %s", util.Dump(steps[i]))
+		}
+	}
+	log.Debugf("steps: %s", util.Dump(steps))
+
+	t.Steps = steps
+
+	log.Debugf("t: %s", util.Dump(t))
+	return nil
+}
+
+func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type element struct {
+		Name          string        `yaml:"name"`
+		Task          string        `yaml:"task"`
+		Depends       []interface{} `yaml:"depends"`
+		IgnoreFailure bool          `yaml:"ignore_failure"`
+	}
+	var te *element
+
+	if err := unmarshal(&te); err != nil {
+		return err
+	}
+	log.Debugf("te: %s", util.Dump(te))
+
+	e.Name = te.Name
+	e.Task = te.Task
+	e.IgnoreFailure = te.IgnoreFailure
+
+	depends := make([]*Depend, len(te.Depends))
+	for i, dependEntry := range te.Depends {
+		var depend *Depend
+		log.Debugf("dependEntry: %v", util.Dump(dependEntry))
+		switch dependEntry.(type) {
+		case string:
+			depend = &Depend{
+				ElementName: dependEntry.(string),
+			}
+		case map[interface{}]interface{}:
+			type deplist map[string][]DependCondition
+			var dl deplist
+			o, err := yaml.Marshal(dependEntry)
+			if err != nil {
+				return err
+			}
+			if err := yaml.Unmarshal(o, &dl); err != nil {
+				return err
+			}
+			log.Debugf("dl: %v", util.Dump(dl))
+			if len(dl) != 1 {
+				return errors.Errorf("unsupported depend format. Must be a string or a list")
+			}
+			for k, v := range dl {
+				depend = &Depend{
+					ElementName: k,
+					Conditions:  v,
+				}
+			}
+
+		default:
+			return errors.Errorf("unsupported depend format. Must be a string or a list")
+		}
+		depends[i] = depend
+	}
+	log.Debugf("depends: %s", util.Dump(depends))
+
+	e.Depends = depends
+
+	log.Debugf("e: %s", util.Dump(e))
+	return nil
+}
+
+func (c *Config) Runtime(runtimeName string) *Runtime {
+	for n, r := range c.Runtimes {
+		if n == runtimeName {
+			return r
+		}
+	}
+	panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName))
+}
+
+func (c *Config) Task(taskName string) *Task {
+	for n, t := range c.Tasks {
+		if n == taskName {
+			return t
+		}
+	}
+	panic(fmt.Sprintf("task %q doesn't exists", taskName))
+}
+
+func (c *Config) Pipeline(pipelineName string) *Pipeline {
+	for n, p := range c.Pipelines {
+		if n == pipelineName {
+			return p
+		}
+	}
+	panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName))
+}
+
+var DefaultConfig = Config{}
+
+func ParseConfig(configData []byte) (*Config, error) {
+	config := DefaultConfig
+	if err := yaml.Unmarshal(configData, &config); err != nil {
+		return nil, err
+	}
+
+	if len(config.Pipelines) == 0 {
+		return nil, errors.Errorf("no pipelines defined")
+	}
+
+	// Set names from maps keys
+	for n, runtime := range config.Runtimes {
+		if runtime == nil {
+			return nil, errors.Errorf("runtime %q is empty", n)
+		}
+		runtime.Name = n
+	}
+
+	for n, task := range config.Tasks {
+		if task == nil {
+			return nil, errors.Errorf("task %q is empty", n)
+		}
+		task.Name = n
+	}
+
+	for n, pipeline := range config.Pipelines {
+		if pipeline == nil {
+			return nil, errors.Errorf("pipeline %q is empty", n)
+		}
+		pipeline.Name = n
+	}
+
+	for _, pipeline := range config.Pipelines {
+		for n, element := range pipeline.Elements {
+			if element == nil {
+				return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n)
+			}
+			element.Name = n
+		}
+	}
+
+	return &config, checkConfig(&config)
+}
+
+func checkConfig(config *Config) error {
+	log.Debugf("config: %s", util.Dump(config))
+
+	// check broken dependencies
+	for _, pipeline := range config.Pipelines {
+		// collect all task names
+		allElements := map[string]struct{}{}
+		for _, element := range pipeline.Elements {
+			allElements[element.Name] = struct{}{}
+		}
+
+		for _, element := range pipeline.Elements {
+			for _, dep := range element.Depends {
+				if _, ok := allElements[dep.ElementName]; !ok {
+					return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
+				}
+			}
+		}
+	}
+
+	// check circular dependencies
+	for _, pipeline := range config.Pipelines {
+		cerrs := &util.Errors{}
+		for _, element := range pipeline.Elements {
+			allParents := getAllElementParents(pipeline, element)
+			for _, parent := range allParents {
+				if parent.Name == element.Name {
+					// TODO(sgotti) get the parent that depends on task to report it
+					dep := []string{}
+					for _, parent := range allParents {
+						pparents := getElementParents(pipeline, parent)
+						for _, pparent := range pparents {
+							if pparent.Name == element.Name {
+								dep = append(dep, fmt.Sprintf("%q", parent.Name))
+							}
+						}
+					}
+					cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " ")))
+				}
+			}
+		}
+		if cerrs.IsErr() {
+			return cerrs
+		}
+	}
+
+	// check that the task and its parent don't have a common dependency
+	for _, pipeline := range config.Pipelines {
+		for _, element := range pipeline.Elements {
+			parents := getElementParents(pipeline, element)
+			for _, parent := range parents {
+				allParents := getAllElementParents(pipeline, element)
+				allParentParents := getAllElementParents(pipeline, parent)
+				for _, p := range allParents {
+					for _, pp := range allParentParents {
+						if p.Name == pp.Name {
+							return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for _, r := range config.Runtimes {
+		if r.Type != RuntimeTypePod {
+			return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type)
+		}
+		if len(r.Containers) == 0 {
+			return errors.Errorf("runtime %q: at least one container must be defined", r.Name)
+		}
+	}
+
+	for _, t := range config.Tasks {
+		if len(t.Name) > maxTaskNameLength {
+			return errors.Errorf("task name %q too long", t.Name)
+		}
+		if t.Runtime == "" {
+			return errors.Errorf("task %q: undefined runtime", t.Name)
+		}
+		if _, ok := config.Runtimes[t.Runtime]; !ok {
+			return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name)
+		}
+		for i, s := range t.Steps {
+			log.Debugf("s: %s", util.Dump(s))
+			switch step := s.(type) {
+			// TODO(sgotti) we could use the run step command as step name but when the
+			// command is very long or multi line it doesn't makes sense and will
+			// probably be quite unuseful/confusing from an UI point of view
+			case *RunStep:
+				if step.Name == "" {
+					lines, err := util.CountLines(step.Command)
+					// if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred
+					if err != nil || lines > 1 {
+						return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name)
+					}
+					len := len(step.Command)
+					if len > maxStepNameLength {
+						len = maxStepNameLength
+					}
+					step.Name = step.Command[:len]
+				}
+			}
+		}
+	}
+
+	for _, pipeline := range config.Pipelines {
+		if len(pipeline.Name) > maxPipelineNameLength {
+			return errors.Errorf("pipeline name %q too long", pipeline.Name)
+		}
+		for _, element := range pipeline.Elements {
+			// check missing tasks reference
+			if element.Task == "" {
+				return errors.Errorf("no task defined for pipeline element %q", element.Name)
+			}
+			if _, ok := config.Tasks[element.Task]; !ok {
+				return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name)
+			}
+			// check duplicate dependencies in task
+			seenDependencies := map[string]struct{}{}
+			for _, dep := range element.Depends {
+				if _, ok := seenDependencies[dep.ElementName]; ok {
+					return errors.Errorf("duplicate task dependency: %s", element.Name)
+				}
+				seenDependencies[dep.ElementName] = struct{}{}
+			}
+		}
+	}
+
+	return nil
+}
+
+// getElementParents returns direct parents of element.
+func getElementParents(pipeline *Pipeline, element *Element) []*Element {
+	parents := []*Element{}
+	for _, el := range pipeline.Elements {
+		isParent := false
+		for _, d := range element.Depends {
+			if d.ElementName == el.Name {
+				isParent = true
+			}
+		}
+		if isParent {
+			parents = append(parents, el)
+		}
+	}
+	return parents
+}
+
+// getAllElementParents returns all the parents (both direct and ancestors) of an element.
+// In case of circular dependency it won't loop forever but will also return
+// the element as parent of itself
+func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
+	pMap := map[string]*Element{}
+	nextParents := getElementParents(pipeline, element)
+
+	for len(nextParents) > 0 {
+		parents := nextParents
+		nextParents = []*Element{}
+		for _, parent := range parents {
+			if _, ok := pMap[parent.Name]; ok {
+				continue
+			}
+			pMap[parent.Name] = parent
+			nextParents = append(nextParents, getElementParents(pipeline, parent)...)
+		}
+	}
+
+	parents := make([]*Element, 0, len(pMap))
+	for _, v := range pMap {
+		parents = append(parents, v)
+	}
+	return parents
+}

internal/config/config_test.go

@@ -0,0 +1,120 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/util"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		err  error
+	}{
+		{
+			name: "test no pipelines 1",
+			in:   ``,
+			err:  fmt.Errorf(`no pipelines defined`),
+		},
+		{
+			name: "test no pipelines 2",
+			in: `
+                pipelines:
+                `,
+			err: fmt.Errorf(`no pipelines defined`),
+		},
+		{
+			name: "test empty pipeline",
+			in: `
+                pipelines:
+                  pipeline01:
+                `,
+			err: fmt.Errorf(`pipeline "pipeline01" is empty`),
+		},
+		{
+			name: "test missing element dependency",
+			in: `
+                tasks:
+                  task0k1:
+                    environment:
+                      ENV01: ENV01
+
+                pipelines:
+                  pipeline01:
+                    elements:
+                      element01:
+                        task: task01
+                        depends:
+                          - element02
+                `,
+			err: fmt.Errorf(`pipeline element "element02" needed by element "element01" doesn't exist`),
+		},
+		{
+			name: "test circular dependency between 2 elements a -> b -> a",
+			in: `
+                tasks:
+                  task01:
+                    environment:
+                      ENV01: ENV01
+
+                pipelines:
+                  pipeline01:
+                    elements:
+                      element01:
+                        task: task01
+                        depends:
+                          - element02
+                      element02:
+                        task: task01
+                        depends:
+                          - element01
+                `,
+			err: &util.Errors{
+				Errs: []error{
+					errors.Errorf("circular dependency between element %q and elements %q", "element01", "element02"),
+					errors.Errorf("circular dependency between element %q and elements %q", "element02", "element01"),
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, err := ParseConfig([]byte(tt.in)); err != nil {
+				if tt.err == nil {
+					t.Fatalf("got error: %v, expected no error", err)
+				}
+				if errs, ok := err.(*util.Errors); ok {
+					if !errs.Equal(tt.err) {
+						t.Fatalf("got error: %v, want error: %v", err, tt.err)
+					}
+				} else {
+					if err.Error() != tt.err.Error() {
+						t.Fatalf("got error: %v, want error: %v", err, tt.err)
+					}
+				}
+				return
+			}
+			if tt.err != nil {
+				t.Fatalf("got nil error, want error: %v", tt.err)
+			}
+		})
+	}
+}