// Copyright 2019 Sorint.lab // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "strings" "github.com/sorintlab/agola/internal/common" "github.com/sorintlab/agola/internal/util" "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) const ( maxPipelineNameLength = 100 maxTaskNameLength = 100 maxStepNameLength = 100 ) type Config struct { Runtimes map[string]*Runtime `yaml:"runtimes"` Tasks map[string]*Task `yaml:"tasks"` Pipelines map[string]*Pipeline `yaml:"pipelines"` } type Task struct { Name string `yaml:"name"` Runtime string `yaml:"runtime"` Environment map[string]string `yaml:"environment"` WorkingDir string `yaml:"working_dir"` Shell string `yaml:"shell"` User string `yaml:"user"` Steps []interface{} `yaml:"steps"` } type RuntimeType string const ( RuntimeTypePod RuntimeType = "pod" ) type Runtime struct { Name string `yaml:"name"` Type RuntimeType `yaml:"type,omitempty"` Arch common.Arch `yaml:"arch,omitempty"` Containers []*Container `yaml:"containers,omitempty"` } type Container struct { Image string `yaml:"image,omitempty"` Environment map[string]string `yaml:"environment,omitempty"` User string `yaml:"user"` } type Pipeline struct { Name string `yaml:"name"` Elements map[string]*Element `yaml:"elements"` } type Element struct { Name string `yaml:"name"` Task string `yaml:"task"` Depends []*Depend `yaml:"depends"` IgnoreFailure bool `yaml:"ignore_failure"` Approval bool `yaml:"approval"` } type DependCondition string const ( DependConditionOnSuccess DependCondition = "on_success" DependConditionOnFailure DependCondition = "on_failure" ) type Depend struct { ElementName string `yaml:"name"` Conditions []DependCondition `yaml: "conditions"` } type Step struct { Type string `yaml:"type"` Name string `yaml:"name"` } type CloneStep struct { Step `yaml:",inline"` } type RunStep struct { Step `yaml:",inline"` Command string `yaml:"command"` Environment map[string]string `yaml:"environment,omitempty"` WorkingDir string `yaml:"working_dir"` Shell string `yaml:"shell"` User string `yaml:"user"` } type SaveToWorkspaceContent struct { SourceDir string `yaml:"source_dir"` DestDir string `yaml:"dest_dir"` Paths []string `yaml:"paths"` } type SaveToWorkspaceStep struct { Step `yaml:",inline"` Contents []SaveToWorkspaceContent } type RestoreWorkspaceStep struct { Step `yaml:",inline"` DestDir string `yaml:"dest_dir"` } func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { type task Task type tasksteps struct { Steps []map[string]interface{} `yaml:"steps"` } tt := (*task)(t) if err := unmarshal(&tt); err != nil { return err } var st tasksteps if err := unmarshal(&st); err != nil { return err } steps := make([]interface{}, len(tt.Steps)) for i, stepEntry := range st.Steps { if len(stepEntry) > 1 { return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i) } for stepType, stepSpec := range stepEntry { o, err := yaml.Marshal(stepSpec) if err != nil { return err } switch stepType { case "clone": var cs CloneStep cs.Type = stepType steps[i] = &cs case "run": var rs RunStep rs.Type = stepType switch stepSpec.(type) { case string: rs.Command = stepSpec.(string) default: if err := yaml.Unmarshal(o, &rs); err != nil { return err } } steps[i] = &rs case "save_to_workspace": var sws SaveToWorkspaceStep sws.Type = stepType if err := yaml.Unmarshal(o, &sws); err != nil { return err } steps[i] = &sws case "restore_workspace": var rws RestoreWorkspaceStep rws.Type = stepType if err := yaml.Unmarshal(o, &rws); err != nil { return err } steps[i] = &rws default: return errors.Errorf("unknown step type: %s", stepType) } } } t.Steps = steps return nil } func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error { type element struct { Name string `yaml:"name"` Task string `yaml:"task"` Depends []interface{} `yaml:"depends"` IgnoreFailure bool `yaml:"ignore_failure"` } var te *element if err := unmarshal(&te); err != nil { return err } e.Name = te.Name e.Task = te.Task e.IgnoreFailure = te.IgnoreFailure depends := make([]*Depend, len(te.Depends)) for i, dependEntry := range te.Depends { var depend *Depend switch dependEntry.(type) { case string: depend = &Depend{ ElementName: dependEntry.(string), } case map[interface{}]interface{}: type deplist map[string][]DependCondition var dl deplist o, err := yaml.Marshal(dependEntry) if err != nil { return err } if err := yaml.Unmarshal(o, &dl); err != nil { return err } if len(dl) != 1 { return errors.Errorf("unsupported depend format. Must be a string or a list") } for k, v := range dl { depend = &Depend{ ElementName: k, Conditions: v, } } default: return errors.Errorf("unsupported depend format. Must be a string or a list") } depends[i] = depend } e.Depends = depends return nil } func (c *Config) Runtime(runtimeName string) *Runtime { for n, r := range c.Runtimes { if n == runtimeName { return r } } panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName)) } func (c *Config) Task(taskName string) *Task { for n, t := range c.Tasks { if n == taskName { return t } } panic(fmt.Sprintf("task %q doesn't exists", taskName)) } func (c *Config) Pipeline(pipelineName string) *Pipeline { for n, p := range c.Pipelines { if n == pipelineName { return p } } panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName)) } var DefaultConfig = Config{} func ParseConfig(configData []byte) (*Config, error) { config := DefaultConfig if err := yaml.Unmarshal(configData, &config); err != nil { return nil, err } if len(config.Pipelines) == 0 { return nil, errors.Errorf("no pipelines defined") } // Set names from maps keys for n, runtime := range config.Runtimes { if runtime == nil { return nil, errors.Errorf("runtime %q is empty", n) } runtime.Name = n } for n, task := range config.Tasks { if task == nil { return nil, errors.Errorf("task %q is empty", n) } task.Name = n } for n, pipeline := range config.Pipelines { if pipeline == nil { return nil, errors.Errorf("pipeline %q is empty", n) } pipeline.Name = n } for _, pipeline := range config.Pipelines { for n, element := range pipeline.Elements { if element == nil { return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n) } element.Name = n } } return &config, checkConfig(&config) } func checkConfig(config *Config) error { // check broken dependencies for _, pipeline := range config.Pipelines { // collect all task names allElements := map[string]struct{}{} for _, element := range pipeline.Elements { allElements[element.Name] = struct{}{} } for _, element := range pipeline.Elements { for _, dep := range element.Depends { if _, ok := allElements[dep.ElementName]; !ok { return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name) } } } } // check circular dependencies for _, pipeline := range config.Pipelines { cerrs := &util.Errors{} for _, element := range pipeline.Elements { allParents := getAllElementParents(pipeline, element) for _, parent := range allParents { if parent.Name == element.Name { // TODO(sgotti) get the parent that depends on task to report it dep := []string{} for _, parent := range allParents { pparents := getElementParents(pipeline, parent) for _, pparent := range pparents { if pparent.Name == element.Name { dep = append(dep, fmt.Sprintf("%q", parent.Name)) } } } cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " "))) } } } if cerrs.IsErr() { return cerrs } } // check that the task and its parent don't have a common dependency for _, pipeline := range config.Pipelines { for _, element := range pipeline.Elements { parents := getElementParents(pipeline, element) for _, parent := range parents { allParents := getAllElementParents(pipeline, element) allParentParents := getAllElementParents(pipeline, parent) for _, p := range allParents { for _, pp := range allParentParents { if p.Name == pp.Name { return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name) } } } } } } for _, r := range config.Runtimes { if r.Type != RuntimeTypePod { return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type) } if len(r.Containers) == 0 { return errors.Errorf("runtime %q: at least one container must be defined", r.Name) } } for _, t := range config.Tasks { if len(t.Name) > maxTaskNameLength { return errors.Errorf("task name %q too long", t.Name) } if t.Runtime == "" { return errors.Errorf("task %q: undefined runtime", t.Name) } if _, ok := config.Runtimes[t.Runtime]; !ok { return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name) } for i, s := range t.Steps { switch step := s.(type) { // TODO(sgotti) we could use the run step command as step name but when the // command is very long or multi line it doesn't makes sense and will // probably be quite unuseful/confusing from an UI point of view case *RunStep: if step.Name == "" { lines, err := util.CountLines(step.Command) // if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred if err != nil || lines > 1 { return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name) } len := len(step.Command) if len > maxStepNameLength { len = maxStepNameLength } step.Name = step.Command[:len] } } } } for _, pipeline := range config.Pipelines { if len(pipeline.Name) > maxPipelineNameLength { return errors.Errorf("pipeline name %q too long", pipeline.Name) } for _, element := range pipeline.Elements { // check missing tasks reference if element.Task == "" { return errors.Errorf("no task defined for pipeline element %q", element.Name) } if _, ok := config.Tasks[element.Task]; !ok { return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name) } // check duplicate dependencies in task seenDependencies := map[string]struct{}{} for _, dep := range element.Depends { if _, ok := seenDependencies[dep.ElementName]; ok { return errors.Errorf("duplicate task dependency: %s", element.Name) } seenDependencies[dep.ElementName] = struct{}{} } } } return nil } // getElementParents returns direct parents of element. func getElementParents(pipeline *Pipeline, element *Element) []*Element { parents := []*Element{} for _, el := range pipeline.Elements { isParent := false for _, d := range element.Depends { if d.ElementName == el.Name { isParent = true } } if isParent { parents = append(parents, el) } } return parents } // getAllElementParents returns all the parents (both direct and ancestors) of an element. // In case of circular dependency it won't loop forever but will also return // the element as parent of itself func getAllElementParents(pipeline *Pipeline, element *Element) []*Element { pMap := map[string]*Element{} nextParents := getElementParents(pipeline, element) for len(nextParents) > 0 { parents := nextParents nextParents = []*Element{} for _, parent := range parents { if _, ok := pMap[parent.Name]; ok { continue } pMap[parent.Name] = parent nextParents = append(nextParents, getElementParents(pipeline, parent)...) } } parents := make([]*Element, 0, len(pMap)) for _, v := range pMap { parents = append(parents, v) } return parents }