From f70dc167383c52817783d7772610c358b588f187 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Thu, 7 Mar 2019 14:42:32 +0100 Subject: [PATCH] Add initial agola config format and handling --- internal/config/config.go | 511 +++++++++++++++++++++++++++++++++ internal/config/config_test.go | 120 ++++++++ 2 files changed, 631 insertions(+) create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..c7c8761 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,511 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "strings" + + "github.com/sorintlab/agola/internal/common" + "github.com/sorintlab/agola/internal/util" + + "github.com/pkg/errors" + yaml "gopkg.in/yaml.v2" +) + +const ( + maxPipelineNameLength = 100 + maxTaskNameLength = 100 + maxStepNameLength = 100 +) + +type Config struct { + Runtimes map[string]*Runtime `yaml:"runtimes"` + Tasks map[string]*Task `yaml:"tasks"` + Pipelines map[string]*Pipeline `yaml:"pipelines"` +} + +type Task struct { + Name string `yaml:"name"` + Runtime string `yaml:"runtime"` + Environment map[string]string `yaml:"environment"` + WorkingDir string `yaml:"working_dir"` + Shell string `yaml:"shell"` + User string `yaml:"user"` + Steps []interface{} `yaml:"steps"` +} + +type RuntimeType string + +const ( + RuntimeTypePod RuntimeType = "pod" +) + +type Runtime struct { + Name string `yaml:"name"` + Type RuntimeType `yaml:"type,omitempty"` + Arch common.Arch `yaml:"arch,omitempty"` + Containers []*Container `yaml:"containers,omitempty"` +} + +type Container struct { + Image string `yaml:"image,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + User string `yaml:"user"` +} + +type Pipeline struct { + Name string `yaml:"name"` + Elements map[string]*Element `yaml:"elements"` +} + +type Element struct { + Name string `yaml:"name"` + Task string `yaml:"task"` + Depends []*Depend `yaml:"depends"` + IgnoreFailure bool `yaml:"ignore_failure"` + Approval bool `yaml:"approval"` +} + +type DependCondition string + +const ( + DependConditionOnSuccess DependCondition = "on_success" + DependConditionOnFailure DependCondition = "on_failure" +) + +type Depend struct { + ElementName string `yaml:"name"` + Conditions []DependCondition `yaml: "conditions"` +} + +type Step struct { + Type string `yaml:"type"` + Name string `yaml:"name"` +} + +type CloneStep struct { + Step `yaml:",inline"` +} + +type RunStep struct { + Step `yaml:",inline"` + Command string `yaml:"command"` + Environment map[string]string `yaml:"environment,omitempty"` + WorkingDir string `yaml:"working_dir"` + Shell string `yaml:"shell"` + User string `yaml:"user"` +} + +type SaveToWorkspaceContent struct { + SourceDir string `yaml:"source_dir"` + DestDir string `yaml:"dest_dir"` + Paths []string `yaml:"paths"` +} + +type SaveToWorkspaceStep struct { + Step `yaml:",inline"` + Contents []SaveToWorkspaceContent +} + +type RestoreWorkspaceStep struct { + Step `yaml:",inline"` + DestDir string `yaml:"dest_dir"` +} + +func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { + type task Task + type tasksteps struct { + Steps []map[string]interface{} `yaml:"steps"` + } + tt := (*task)(t) + if err := unmarshal(&tt); err != nil { + return err + } + log.Debugf("tt: %s", util.Dump(tt)) + + var st tasksteps + if err := unmarshal(&st); err != nil { + return err + } + + steps := make([]interface{}, len(tt.Steps)) + for i, stepEntry := range st.Steps { + if len(stepEntry) > 1 { + return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i) + } + for stepType, stepSpec := range stepEntry { + log.Debugf("s: %s", util.Dump(stepSpec)) + o, err := yaml.Marshal(stepSpec) + if err != nil { + return err + } + log.Debugf("o: %s", o) + switch stepType { + case "clone": + var cs CloneStep + cs.Type = stepType + steps[i] = &cs + + case "run": + var rs RunStep + rs.Type = stepType + switch stepSpec.(type) { + case string: + rs.Command = stepSpec.(string) + default: + if err := yaml.Unmarshal(o, &rs); err != nil { + return err + } + } + steps[i] = &rs + + case "save_to_workspace": + var sws SaveToWorkspaceStep + sws.Type = stepType + if err := yaml.Unmarshal(o, &sws); err != nil { + return err + } + steps[i] = &sws + + case "restore_workspace": + var rws RestoreWorkspaceStep + rws.Type = stepType + if err := yaml.Unmarshal(o, &rws); err != nil { + return err + } + steps[i] = &rws + default: + return errors.Errorf("unknown step type: %s", stepType) + } + log.Debugf("s: %s", util.Dump(steps[i])) + } + } + log.Debugf("steps: %s", util.Dump(steps)) + + t.Steps = steps + + log.Debugf("t: %s", util.Dump(t)) + return nil +} + +func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error { + type element struct { + Name string `yaml:"name"` + Task string `yaml:"task"` + Depends []interface{} `yaml:"depends"` + IgnoreFailure bool `yaml:"ignore_failure"` + } + var te *element + + if err := unmarshal(&te); err != nil { + return err + } + log.Debugf("te: %s", util.Dump(te)) + + e.Name = te.Name + e.Task = te.Task + e.IgnoreFailure = te.IgnoreFailure + + depends := make([]*Depend, len(te.Depends)) + for i, dependEntry := range te.Depends { + var depend *Depend + log.Debugf("dependEntry: %v", util.Dump(dependEntry)) + switch dependEntry.(type) { + case string: + depend = &Depend{ + ElementName: dependEntry.(string), + } + case map[interface{}]interface{}: + type deplist map[string][]DependCondition + var dl deplist + o, err := yaml.Marshal(dependEntry) + if err != nil { + return err + } + if err := yaml.Unmarshal(o, &dl); err != nil { + return err + } + log.Debugf("dl: %v", util.Dump(dl)) + if len(dl) != 1 { + return errors.Errorf("unsupported depend format. Must be a string or a list") + } + for k, v := range dl { + depend = &Depend{ + ElementName: k, + Conditions: v, + } + } + + default: + return errors.Errorf("unsupported depend format. Must be a string or a list") + } + depends[i] = depend + } + log.Debugf("depends: %s", util.Dump(depends)) + + e.Depends = depends + + log.Debugf("e: %s", util.Dump(e)) + return nil +} + +func (c *Config) Runtime(runtimeName string) *Runtime { + for n, r := range c.Runtimes { + if n == runtimeName { + return r + } + } + panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName)) +} + +func (c *Config) Task(taskName string) *Task { + for n, t := range c.Tasks { + if n == taskName { + return t + } + } + panic(fmt.Sprintf("task %q doesn't exists", taskName)) +} + +func (c *Config) Pipeline(pipelineName string) *Pipeline { + for n, p := range c.Pipelines { + if n == pipelineName { + return p + } + } + panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName)) +} + +var DefaultConfig = Config{} + +func ParseConfig(configData []byte) (*Config, error) { + config := DefaultConfig + if err := yaml.Unmarshal(configData, &config); err != nil { + return nil, err + } + + if len(config.Pipelines) == 0 { + return nil, errors.Errorf("no pipelines defined") + } + + // Set names from maps keys + for n, runtime := range config.Runtimes { + if runtime == nil { + return nil, errors.Errorf("runtime %q is empty", n) + } + runtime.Name = n + } + + for n, task := range config.Tasks { + if task == nil { + return nil, errors.Errorf("task %q is empty", n) + } + task.Name = n + } + + for n, pipeline := range config.Pipelines { + if pipeline == nil { + return nil, errors.Errorf("pipeline %q is empty", n) + } + pipeline.Name = n + } + + for _, pipeline := range config.Pipelines { + for n, element := range pipeline.Elements { + if element == nil { + return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n) + } + element.Name = n + } + } + + return &config, checkConfig(&config) +} + +func checkConfig(config *Config) error { + log.Debugf("config: %s", util.Dump(config)) + + // check broken dependencies + for _, pipeline := range config.Pipelines { + // collect all task names + allElements := map[string]struct{}{} + for _, element := range pipeline.Elements { + allElements[element.Name] = struct{}{} + } + + for _, element := range pipeline.Elements { + for _, dep := range element.Depends { + if _, ok := allElements[dep.ElementName]; !ok { + return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name) + } + } + } + } + + // check circular dependencies + for _, pipeline := range config.Pipelines { + cerrs := &util.Errors{} + for _, element := range pipeline.Elements { + allParents := getAllElementParents(pipeline, element) + for _, parent := range allParents { + if parent.Name == element.Name { + // TODO(sgotti) get the parent that depends on task to report it + dep := []string{} + for _, parent := range allParents { + pparents := getElementParents(pipeline, parent) + for _, pparent := range pparents { + if pparent.Name == element.Name { + dep = append(dep, fmt.Sprintf("%q", parent.Name)) + } + } + } + cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " "))) + } + } + } + if cerrs.IsErr() { + return cerrs + } + } + + // check that the task and its parent don't have a common dependency + for _, pipeline := range config.Pipelines { + for _, element := range pipeline.Elements { + parents := getElementParents(pipeline, element) + for _, parent := range parents { + allParents := getAllElementParents(pipeline, element) + allParentParents := getAllElementParents(pipeline, parent) + for _, p := range allParents { + for _, pp := range allParentParents { + if p.Name == pp.Name { + return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name) + } + } + } + } + } + } + + for _, r := range config.Runtimes { + if r.Type != RuntimeTypePod { + return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type) + } + if len(r.Containers) == 0 { + return errors.Errorf("runtime %q: at least one container must be defined", r.Name) + } + } + + for _, t := range config.Tasks { + if len(t.Name) > maxTaskNameLength { + return errors.Errorf("task name %q too long", t.Name) + } + if t.Runtime == "" { + return errors.Errorf("task %q: undefined runtime", t.Name) + } + if _, ok := config.Runtimes[t.Runtime]; !ok { + return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name) + } + for i, s := range t.Steps { + log.Debugf("s: %s", util.Dump(s)) + switch step := s.(type) { + // TODO(sgotti) we could use the run step command as step name but when the + // command is very long or multi line it doesn't makes sense and will + // probably be quite unuseful/confusing from an UI point of view + case *RunStep: + if step.Name == "" { + lines, err := util.CountLines(step.Command) + // if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred + if err != nil || lines > 1 { + return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name) + } + len := len(step.Command) + if len > maxStepNameLength { + len = maxStepNameLength + } + step.Name = step.Command[:len] + } + } + } + } + + for _, pipeline := range config.Pipelines { + if len(pipeline.Name) > maxPipelineNameLength { + return errors.Errorf("pipeline name %q too long", pipeline.Name) + } + for _, element := range pipeline.Elements { + // check missing tasks reference + if element.Task == "" { + return errors.Errorf("no task defined for pipeline element %q", element.Name) + } + if _, ok := config.Tasks[element.Task]; !ok { + return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name) + } + // check duplicate dependencies in task + seenDependencies := map[string]struct{}{} + for _, dep := range element.Depends { + if _, ok := seenDependencies[dep.ElementName]; ok { + return errors.Errorf("duplicate task dependency: %s", element.Name) + } + seenDependencies[dep.ElementName] = struct{}{} + } + } + } + + return nil +} + +// getElementParents returns direct parents of element. +func getElementParents(pipeline *Pipeline, element *Element) []*Element { + parents := []*Element{} + for _, el := range pipeline.Elements { + isParent := false + for _, d := range element.Depends { + if d.ElementName == el.Name { + isParent = true + } + } + if isParent { + parents = append(parents, el) + } + } + return parents +} + +// getAllElementParents returns all the parents (both direct and ancestors) of an element. +// In case of circular dependency it won't loop forever but will also return +// the element as parent of itself +func getAllElementParents(pipeline *Pipeline, element *Element) []*Element { + pMap := map[string]*Element{} + nextParents := getElementParents(pipeline, element) + + for len(nextParents) > 0 { + parents := nextParents + nextParents = []*Element{} + for _, parent := range parents { + if _, ok := pMap[parent.Name]; ok { + continue + } + pMap[parent.Name] = parent + nextParents = append(nextParents, getElementParents(pipeline, parent)...) + } + } + + parents := make([]*Element, 0, len(pMap)) + for _, v := range pMap { + parents = append(parents, v) + } + return parents +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..445785a --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,120 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/sorintlab/agola/internal/util" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + in string + err error + }{ + { + name: "test no pipelines 1", + in: ``, + err: fmt.Errorf(`no pipelines defined`), + }, + { + name: "test no pipelines 2", + in: ` + pipelines: + `, + err: fmt.Errorf(`no pipelines defined`), + }, + { + name: "test empty pipeline", + in: ` + pipelines: + pipeline01: + `, + err: fmt.Errorf(`pipeline "pipeline01" is empty`), + }, + { + name: "test missing element dependency", + in: ` + tasks: + task0k1: + environment: + ENV01: ENV01 + + pipelines: + pipeline01: + elements: + element01: + task: task01 + depends: + - element02 + `, + err: fmt.Errorf(`pipeline element "element02" needed by element "element01" doesn't exist`), + }, + { + name: "test circular dependency between 2 elements a -> b -> a", + in: ` + tasks: + task01: + environment: + ENV01: ENV01 + + pipelines: + pipeline01: + elements: + element01: + task: task01 + depends: + - element02 + element02: + task: task01 + depends: + - element01 + `, + err: &util.Errors{ + Errs: []error{ + errors.Errorf("circular dependency between element %q and elements %q", "element01", "element02"), + errors.Errorf("circular dependency between element %q and elements %q", "element02", "element01"), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := ParseConfig([]byte(tt.in)); err != nil { + if tt.err == nil { + t.Fatalf("got error: %v, expected no error", err) + } + if errs, ok := err.(*util.Errors); ok { + if !errs.Equal(tt.err) { + t.Fatalf("got error: %v, want error: %v", err, tt.err) + } + } else { + if err.Error() != tt.err.Error() { + t.Fatalf("got error: %v, want error: %v", err, tt.err) + } + } + return + } + if tt.err != nil { + t.Fatalf("got nil error, want error: %v", tt.err) + } + }) + } +}