Add initial agola config format and handling

This commit is contained in:
Simone Gotti 2019-03-07 14:42:32 +01:00
parent 86e8479de9
commit f70dc16738
2 changed files with 631 additions and 0 deletions

511
internal/config/config.go Normal file
View File

@ -0,0 +1,511 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"strings"
"github.com/sorintlab/agola/internal/common"
"github.com/sorintlab/agola/internal/util"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
)
const (
maxPipelineNameLength = 100
maxTaskNameLength = 100
maxStepNameLength = 100
)
type Config struct {
Runtimes map[string]*Runtime `yaml:"runtimes"`
Tasks map[string]*Task `yaml:"tasks"`
Pipelines map[string]*Pipeline `yaml:"pipelines"`
}
type Task struct {
Name string `yaml:"name"`
Runtime string `yaml:"runtime"`
Environment map[string]string `yaml:"environment"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
Steps []interface{} `yaml:"steps"`
}
type RuntimeType string
const (
RuntimeTypePod RuntimeType = "pod"
)
type Runtime struct {
Name string `yaml:"name"`
Type RuntimeType `yaml:"type,omitempty"`
Arch common.Arch `yaml:"arch,omitempty"`
Containers []*Container `yaml:"containers,omitempty"`
}
type Container struct {
Image string `yaml:"image,omitempty"`
Environment map[string]string `yaml:"environment,omitempty"`
User string `yaml:"user"`
}
type Pipeline struct {
Name string `yaml:"name"`
Elements map[string]*Element `yaml:"elements"`
}
type Element struct {
Name string `yaml:"name"`
Task string `yaml:"task"`
Depends []*Depend `yaml:"depends"`
IgnoreFailure bool `yaml:"ignore_failure"`
Approval bool `yaml:"approval"`
}
type DependCondition string
const (
DependConditionOnSuccess DependCondition = "on_success"
DependConditionOnFailure DependCondition = "on_failure"
)
type Depend struct {
ElementName string `yaml:"name"`
Conditions []DependCondition `yaml: "conditions"`
}
type Step struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
}
type CloneStep struct {
Step `yaml:",inline"`
}
type RunStep struct {
Step `yaml:",inline"`
Command string `yaml:"command"`
Environment map[string]string `yaml:"environment,omitempty"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
}
type SaveToWorkspaceContent struct {
SourceDir string `yaml:"source_dir"`
DestDir string `yaml:"dest_dir"`
Paths []string `yaml:"paths"`
}
type SaveToWorkspaceStep struct {
Step `yaml:",inline"`
Contents []SaveToWorkspaceContent
}
type RestoreWorkspaceStep struct {
Step `yaml:",inline"`
DestDir string `yaml:"dest_dir"`
}
func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
type task Task
type tasksteps struct {
Steps []map[string]interface{} `yaml:"steps"`
}
tt := (*task)(t)
if err := unmarshal(&tt); err != nil {
return err
}
log.Debugf("tt: %s", util.Dump(tt))
var st tasksteps
if err := unmarshal(&st); err != nil {
return err
}
steps := make([]interface{}, len(tt.Steps))
for i, stepEntry := range st.Steps {
if len(stepEntry) > 1 {
return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
}
for stepType, stepSpec := range stepEntry {
log.Debugf("s: %s", util.Dump(stepSpec))
o, err := yaml.Marshal(stepSpec)
if err != nil {
return err
}
log.Debugf("o: %s", o)
switch stepType {
case "clone":
var cs CloneStep
cs.Type = stepType
steps[i] = &cs
case "run":
var rs RunStep
rs.Type = stepType
switch stepSpec.(type) {
case string:
rs.Command = stepSpec.(string)
default:
if err := yaml.Unmarshal(o, &rs); err != nil {
return err
}
}
steps[i] = &rs
case "save_to_workspace":
var sws SaveToWorkspaceStep
sws.Type = stepType
if err := yaml.Unmarshal(o, &sws); err != nil {
return err
}
steps[i] = &sws
case "restore_workspace":
var rws RestoreWorkspaceStep
rws.Type = stepType
if err := yaml.Unmarshal(o, &rws); err != nil {
return err
}
steps[i] = &rws
default:
return errors.Errorf("unknown step type: %s", stepType)
}
log.Debugf("s: %s", util.Dump(steps[i]))
}
}
log.Debugf("steps: %s", util.Dump(steps))
t.Steps = steps
log.Debugf("t: %s", util.Dump(t))
return nil
}
func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
type element struct {
Name string `yaml:"name"`
Task string `yaml:"task"`
Depends []interface{} `yaml:"depends"`
IgnoreFailure bool `yaml:"ignore_failure"`
}
var te *element
if err := unmarshal(&te); err != nil {
return err
}
log.Debugf("te: %s", util.Dump(te))
e.Name = te.Name
e.Task = te.Task
e.IgnoreFailure = te.IgnoreFailure
depends := make([]*Depend, len(te.Depends))
for i, dependEntry := range te.Depends {
var depend *Depend
log.Debugf("dependEntry: %v", util.Dump(dependEntry))
switch dependEntry.(type) {
case string:
depend = &Depend{
ElementName: dependEntry.(string),
}
case map[interface{}]interface{}:
type deplist map[string][]DependCondition
var dl deplist
o, err := yaml.Marshal(dependEntry)
if err != nil {
return err
}
if err := yaml.Unmarshal(o, &dl); err != nil {
return err
}
log.Debugf("dl: %v", util.Dump(dl))
if len(dl) != 1 {
return errors.Errorf("unsupported depend format. Must be a string or a list")
}
for k, v := range dl {
depend = &Depend{
ElementName: k,
Conditions: v,
}
}
default:
return errors.Errorf("unsupported depend format. Must be a string or a list")
}
depends[i] = depend
}
log.Debugf("depends: %s", util.Dump(depends))
e.Depends = depends
log.Debugf("e: %s", util.Dump(e))
return nil
}
func (c *Config) Runtime(runtimeName string) *Runtime {
for n, r := range c.Runtimes {
if n == runtimeName {
return r
}
}
panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName))
}
func (c *Config) Task(taskName string) *Task {
for n, t := range c.Tasks {
if n == taskName {
return t
}
}
panic(fmt.Sprintf("task %q doesn't exists", taskName))
}
func (c *Config) Pipeline(pipelineName string) *Pipeline {
for n, p := range c.Pipelines {
if n == pipelineName {
return p
}
}
panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName))
}
var DefaultConfig = Config{}
func ParseConfig(configData []byte) (*Config, error) {
config := DefaultConfig
if err := yaml.Unmarshal(configData, &config); err != nil {
return nil, err
}
if len(config.Pipelines) == 0 {
return nil, errors.Errorf("no pipelines defined")
}
// Set names from maps keys
for n, runtime := range config.Runtimes {
if runtime == nil {
return nil, errors.Errorf("runtime %q is empty", n)
}
runtime.Name = n
}
for n, task := range config.Tasks {
if task == nil {
return nil, errors.Errorf("task %q is empty", n)
}
task.Name = n
}
for n, pipeline := range config.Pipelines {
if pipeline == nil {
return nil, errors.Errorf("pipeline %q is empty", n)
}
pipeline.Name = n
}
for _, pipeline := range config.Pipelines {
for n, element := range pipeline.Elements {
if element == nil {
return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n)
}
element.Name = n
}
}
return &config, checkConfig(&config)
}
func checkConfig(config *Config) error {
log.Debugf("config: %s", util.Dump(config))
// check broken dependencies
for _, pipeline := range config.Pipelines {
// collect all task names
allElements := map[string]struct{}{}
for _, element := range pipeline.Elements {
allElements[element.Name] = struct{}{}
}
for _, element := range pipeline.Elements {
for _, dep := range element.Depends {
if _, ok := allElements[dep.ElementName]; !ok {
return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
}
}
}
}
// check circular dependencies
for _, pipeline := range config.Pipelines {
cerrs := &util.Errors{}
for _, element := range pipeline.Elements {
allParents := getAllElementParents(pipeline, element)
for _, parent := range allParents {
if parent.Name == element.Name {
// TODO(sgotti) get the parent that depends on task to report it
dep := []string{}
for _, parent := range allParents {
pparents := getElementParents(pipeline, parent)
for _, pparent := range pparents {
if pparent.Name == element.Name {
dep = append(dep, fmt.Sprintf("%q", parent.Name))
}
}
}
cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " ")))
}
}
}
if cerrs.IsErr() {
return cerrs
}
}
// check that the task and its parent don't have a common dependency
for _, pipeline := range config.Pipelines {
for _, element := range pipeline.Elements {
parents := getElementParents(pipeline, element)
for _, parent := range parents {
allParents := getAllElementParents(pipeline, element)
allParentParents := getAllElementParents(pipeline, parent)
for _, p := range allParents {
for _, pp := range allParentParents {
if p.Name == pp.Name {
return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name)
}
}
}
}
}
}
for _, r := range config.Runtimes {
if r.Type != RuntimeTypePod {
return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type)
}
if len(r.Containers) == 0 {
return errors.Errorf("runtime %q: at least one container must be defined", r.Name)
}
}
for _, t := range config.Tasks {
if len(t.Name) > maxTaskNameLength {
return errors.Errorf("task name %q too long", t.Name)
}
if t.Runtime == "" {
return errors.Errorf("task %q: undefined runtime", t.Name)
}
if _, ok := config.Runtimes[t.Runtime]; !ok {
return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name)
}
for i, s := range t.Steps {
log.Debugf("s: %s", util.Dump(s))
switch step := s.(type) {
// TODO(sgotti) we could use the run step command as step name but when the
// command is very long or multi line it doesn't makes sense and will
// probably be quite unuseful/confusing from an UI point of view
case *RunStep:
if step.Name == "" {
lines, err := util.CountLines(step.Command)
// if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred
if err != nil || lines > 1 {
return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name)
}
len := len(step.Command)
if len > maxStepNameLength {
len = maxStepNameLength
}
step.Name = step.Command[:len]
}
}
}
}
for _, pipeline := range config.Pipelines {
if len(pipeline.Name) > maxPipelineNameLength {
return errors.Errorf("pipeline name %q too long", pipeline.Name)
}
for _, element := range pipeline.Elements {
// check missing tasks reference
if element.Task == "" {
return errors.Errorf("no task defined for pipeline element %q", element.Name)
}
if _, ok := config.Tasks[element.Task]; !ok {
return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name)
}
// check duplicate dependencies in task
seenDependencies := map[string]struct{}{}
for _, dep := range element.Depends {
if _, ok := seenDependencies[dep.ElementName]; ok {
return errors.Errorf("duplicate task dependency: %s", element.Name)
}
seenDependencies[dep.ElementName] = struct{}{}
}
}
}
return nil
}
// getElementParents returns direct parents of element.
func getElementParents(pipeline *Pipeline, element *Element) []*Element {
parents := []*Element{}
for _, el := range pipeline.Elements {
isParent := false
for _, d := range element.Depends {
if d.ElementName == el.Name {
isParent = true
}
}
if isParent {
parents = append(parents, el)
}
}
return parents
}
// getAllElementParents returns all the parents (both direct and ancestors) of an element.
// In case of circular dependency it won't loop forever but will also return
// the element as parent of itself
func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
pMap := map[string]*Element{}
nextParents := getElementParents(pipeline, element)
for len(nextParents) > 0 {
parents := nextParents
nextParents = []*Element{}
for _, parent := range parents {
if _, ok := pMap[parent.Name]; ok {
continue
}
pMap[parent.Name] = parent
nextParents = append(nextParents, getElementParents(pipeline, parent)...)
}
}
parents := make([]*Element, 0, len(pMap))
for _, v := range pMap {
parents = append(parents, v)
}
return parents
}

View File

@ -0,0 +1,120 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"testing"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/util"
)
func TestParseConfig(t *testing.T) {
tests := []struct {
name string
in string
err error
}{
{
name: "test no pipelines 1",
in: ``,
err: fmt.Errorf(`no pipelines defined`),
},
{
name: "test no pipelines 2",
in: `
pipelines:
`,
err: fmt.Errorf(`no pipelines defined`),
},
{
name: "test empty pipeline",
in: `
pipelines:
pipeline01:
`,
err: fmt.Errorf(`pipeline "pipeline01" is empty`),
},
{
name: "test missing element dependency",
in: `
tasks:
task0k1:
environment:
ENV01: ENV01
pipelines:
pipeline01:
elements:
element01:
task: task01
depends:
- element02
`,
err: fmt.Errorf(`pipeline element "element02" needed by element "element01" doesn't exist`),
},
{
name: "test circular dependency between 2 elements a -> b -> a",
in: `
tasks:
task01:
environment:
ENV01: ENV01
pipelines:
pipeline01:
elements:
element01:
task: task01
depends:
- element02
element02:
task: task01
depends:
- element01
`,
err: &util.Errors{
Errs: []error{
errors.Errorf("circular dependency between element %q and elements %q", "element01", "element02"),
errors.Errorf("circular dependency between element %q and elements %q", "element02", "element01"),
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := ParseConfig([]byte(tt.in)); err != nil {
if tt.err == nil {
t.Fatalf("got error: %v, expected no error", err)
}
if errs, ok := err.(*util.Errors); ok {
if !errs.Equal(tt.err) {
t.Fatalf("got error: %v, want error: %v", err, tt.err)
}
} else {
if err.Error() != tt.err.Error() {
t.Fatalf("got error: %v, want error: %v", err, tt.err)
}
}
return
}
if tt.err != nil {
t.Fatalf("got nil error, want error: %v", tt.err)
}
})
}
}