agola/internal/config/config.go

// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/sorintlab/agola/internal/common"
	"github.com/sorintlab/agola/internal/services/types"
	"github.com/sorintlab/agola/internal/util"

	"github.com/pkg/errors"
	yaml "gopkg.in/yaml.v2"
)

const (
	maxPipelineNameLength = 100
	maxTaskNameLength     = 100
	maxStepNameLength     = 100
)

var (
	regExpDelimiters = []string{"/", "#"}
)

type Config struct {
	Runtimes  map[string]*Runtime  `yaml:"runtimes"`
	Tasks     map[string]*Task     `yaml:"tasks"`
	Pipelines map[string]*Pipeline `yaml:"pipelines"`
}

type Task struct {
	Name        string           `yaml:"name"`
	Runtime     string           `yaml:"runtime"`
	Environment map[string]Value `yaml:"environment,omitempty"`
	WorkingDir  string           `yaml:"working_dir"`
	Shell       string           `yaml:"shell"`
	User        string           `yaml:"user"`
	Steps       []interface{}    `yaml:"steps"`
}

type RuntimeType string

const (
	RuntimeTypePod RuntimeType = "pod"
)

type RegistryAuthType string

const (
	RegistryAuthTypeDefault RegistryAuthType = "default"
)

type RegistryAuth struct {
	Type RegistryAuthType `yaml:"type"`

	// default auth
	Username Value `yaml:"username"`
	Password Value `yaml:"password"`
}

type Runtime struct {
	Name       string        `yaml:"name"`
	Type       RuntimeType   `yaml:"type,omitempty"`
	Auth       *RegistryAuth `yaml:"auth"`
	Arch       common.Arch   `yaml:"arch,omitempty"`
	Containers []*Container  `yaml:"containers,omitempty"`
}

type Container struct {
	Image       string           `yaml:"image,omitempty"`
	Auth        *RegistryAuth    `yaml:"auth"`
	Environment map[string]Value `yaml:"environment,omitempty"`
	User        string           `yaml:"user"`
	Privileged  bool             `yaml:"privileged"`
	Entrypoint  string           `yaml:"entrypoint"`
}

type Pipeline struct {
	Name     string              `yaml:"name"`
	Elements map[string]*Element `yaml:"elements"`
}

type Element struct {
	Name          string      `yaml:"name"`
	Task          string      `yaml:"task"`
	Depends       []*Depend   `yaml:"depends"`
	IgnoreFailure bool        `yaml:"ignore_failure"`
	Approval      bool        `yaml:"approval"`
	When          *types.When `yaml:"when"`
}

type DependCondition string

const (
	DependConditionOnSuccess DependCondition = "on_success"
	DependConditionOnFailure DependCondition = "on_failure"
	DependConditionOnSkipped DependCondition = "on_skipped"
)

type Depend struct {
	ElementName string            `yaml:"name"`
	Conditions  []DependCondition `yaml:"conditions"`
}

type Step struct {
	Type string `yaml:"type"`
	Name string `yaml:"name"`
}

type CloneStep struct {
	Step `yaml:",inline"`
}

type RunStep struct {
	Step        `yaml:",inline"`
	Command     string           `yaml:"command"`
	Environment map[string]Value `yaml:"environment,omitempty"`
	WorkingDir  string           `yaml:"working_dir"`
	Shell       string           `yaml:"shell"`
	User        string           `yaml:"user"`
}

type ValueType int

const (
	ValueTypeString ValueType = iota
	ValueTypeFromVariable
)

type Value struct {
	Type  ValueType
	Value string
}

type SaveContent struct {
	SourceDir string   `yaml:"source_dir"`
	DestDir   string   `yaml:"dest_dir"`
	Paths     []string `yaml:"paths"`
}

type SaveToWorkspaceStep struct {
	Step     `yaml:",inline"`
	Contents []*SaveContent `yaml:"contents"`
}

type RestoreWorkspaceStep struct {
	Step    `yaml:",inline"`
	DestDir string `yaml:"dest_dir"`
}

type SaveCacheStep struct {
	Step     `yaml:",inline"`
	Key      string         `yaml:"key"`
	Contents []*SaveContent `yaml:"contents"`
}

type RestoreCacheStep struct {
	Step    `yaml:",inline"`
	Keys    []string `yaml:"keys"`
	DestDir string   `yaml:"dest_dir"`
}

func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type task struct {
		Name        string                   `yaml:"name"`
		Runtime     string                   `yaml:"runtime"`
		Environment map[string]Value         `yaml:"environment,omitempty"`
		WorkingDir  string                   `yaml:"working_dir"`
		Shell       string                   `yaml:"shell"`
		User        string                   `yaml:"user"`
		Steps       []map[string]interface{} `yaml:"steps"`
	}

	var tt *task
	if err := unmarshal(&tt); err != nil {
		return err
	}

	t.Name = tt.Name
	t.Runtime = tt.Runtime
	t.Environment = tt.Environment
	t.WorkingDir = tt.WorkingDir
	t.Shell = tt.Shell
	t.User = tt.User

	steps := make([]interface{}, len(tt.Steps))
	for i, stepEntry := range tt.Steps {
		if len(stepEntry) > 1 {
			return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
		}
		for stepType, stepSpec := range stepEntry {
			o, err := yaml.Marshal(stepSpec)
			if err != nil {
				return err
			}
			switch stepType {
			case "clone":
				var s CloneStep
				s.Type = stepType
				steps[i] = &s

			case "run":
				var s RunStep
				switch stepSpec.(type) {
				case string:
					s.Command = stepSpec.(string)
				default:
					if err := yaml.Unmarshal(o, &s); err != nil {
						return err
					}
				}
				s.Type = stepType
				steps[i] = &s

			case "save_to_workspace":
				var s SaveToWorkspaceStep
				if err := yaml.Unmarshal(o, &s); err != nil {
					return err
				}
				s.Type = stepType
				steps[i] = &s

			case "restore_workspace":
				var s RestoreWorkspaceStep
				if err := yaml.Unmarshal(o, &s); err != nil {
					return err
				}
				s.Type = stepType
				steps[i] = &s

			case "save_cache":
				var s SaveCacheStep
				if err := yaml.Unmarshal(o, &s); err != nil {
					return err
				}
				s.Type = stepType
				steps[i] = &s

			case "restore_cache":
				var s RestoreCacheStep
				if err := yaml.Unmarshal(o, &s); err != nil {
					return err
				}
				s.Type = stepType
				steps[i] = &s
			default:
				return errors.Errorf("unknown step type: %s", stepType)
			}
		}
	}

	t.Steps = steps

	return nil
}

func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type when struct {
		Branch interface{} `yaml:"branch"`
		Tag    interface{} `yaml:"tag"`
		Ref    interface{} `yaml:"ref"`
	}

	type element struct {
		Name          string        `yaml:"name"`
		Task          string        `yaml:"task"`
		Depends       []interface{} `yaml:"depends"`
		IgnoreFailure bool          `yaml:"ignore_failure"`
		Approval      bool          `yaml:"approval"`
		When          *when         `yaml:"when"`
	}

	var te *element

	if err := unmarshal(&te); err != nil {
		return err
	}

	e.Name = te.Name
	e.Task = te.Task
	e.IgnoreFailure = te.IgnoreFailure
	e.Approval = te.Approval

	depends := make([]*Depend, len(te.Depends))
	for i, dependEntry := range te.Depends {
		var depend *Depend
		switch dependEntry.(type) {
		case string:
			depend = &Depend{
				ElementName: dependEntry.(string),
			}
		case map[interface{}]interface{}:
			type deplist map[string][]DependCondition
			var dl deplist
			o, err := yaml.Marshal(dependEntry)
			if err != nil {
				return err
			}
			if err := yaml.Unmarshal(o, &dl); err != nil {
				return err
			}
			if len(dl) != 1 {
				return errors.Errorf("unsupported depend format. Must be a string or a list")
			}
			for k, v := range dl {
				depend = &Depend{
					ElementName: k,
					Conditions:  v,
				}
			}

		default:
			return errors.Errorf("unsupported depend format. Must be a string or a list")
		}
		depends[i] = depend
	}

	e.Depends = depends

	if te.When != nil {
		w := &types.When{}

		var err error

		if te.When.Branch != nil {
			w.Branch, err = parseWhenConditions(te.When.Branch)
			if err != nil {
				return err
			}
		}

		if te.When.Tag != nil {
			w.Tag, err = parseWhenConditions(te.When.Tag)
			if err != nil {
				return err
			}
		}

		if te.When.Ref != nil {
			w.Ref, err = parseWhenConditions(te.When.Ref)
			if err != nil {
				return err
			}
		}

		e.When = w
	}

	return nil
}

func (val *Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var ival interface{}
	if err := unmarshal(&ival); err != nil {
		return err
	}
	switch valValue := ival.(type) {
	case string:
		val.Type = ValueTypeString
		val.Value = valValue
	case map[interface{}]interface{}:
		for k, v := range valValue {
			if k == "from_variable" {
				switch v.(type) {
				case string:
				default:
					return errors.Errorf("unknown value format: %v", v)
				}
				val.Type = ValueTypeFromVariable
				val.Value = v.(string)
			}
		}
	default:
		return errors.Errorf("unknown value format: %v", ival)
	}
	return nil
}

func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
	w := &types.WhenConditions{}

	var err error
	include := []string{}
	exclude := []string{}

	switch c := wi.(type) {
	case string:
		include = []string{c}
	case []interface{}:
		ss, err := parseSliceString(c)
		if err != nil {
			return nil, err
		}
		include = ss
	case map[interface{}]interface{}:
		for k, v := range c {
			ks, ok := k.(string)
			if !ok {
				return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
			}
			switch ks {
			case "include":
				include, err = parseStringOrSlice(v)
				if err != nil {
					return nil, err
				}
			case "exclude":
				exclude, err = parseStringOrSlice(v)
				if err != nil {
					return nil, err
				}
			default:
				return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
			}
		}
	default:
		return nil, errors.Errorf("wrong when format")
	}

	w.Include, err = parseWhenConditionSlice(include)
	if err != nil {
		return nil, err
	}
	w.Exclude, err = parseWhenConditionSlice(exclude)
	if err != nil {
		return nil, err
	}

	return w, nil
}

func parseWhenConditionSlice(conds []string) ([]types.WhenCondition, error) {
	if len(conds) == 0 {
		return nil, nil
	}

	wcs := []types.WhenCondition{}
	for _, cond := range conds {
		wc, err := parseWhenCondition(cond)
		if err != nil {
			return nil, err
		}
		wcs = append(wcs, *wc)
	}

	return wcs, nil
}

func parseWhenCondition(s string) (*types.WhenCondition, error) {
	isRegExp := false
	if len(s) > 2 {
		for _, d := range regExpDelimiters {
			if strings.HasPrefix(s, d) && strings.HasSuffix(s, d) {
				isRegExp = true
				s = s[1 : len(s)-1]
				break
			}
		}
	}

	wc := &types.WhenCondition{Match: s}

	if isRegExp {
		if _, err := regexp.Compile(s); err != nil {
			return nil, errors.Wrapf(err, "wrong regular expression")
		}
		wc.Type = types.WhenConditionTypeRegExp
	} else {
		wc.Type = types.WhenConditionTypeSimple
	}
	return wc, nil
}

func parseStringOrSlice(si interface{}) ([]string, error) {
	ss := []string{}
	switch c := si.(type) {
	case string:
		ss = []string{c}
	case []interface{}:
		var err error
		ss, err = parseSliceString(c)
		if err != nil {
			return nil, err
		}
	}
	return ss, nil
}

func parseSliceString(si []interface{}) ([]string, error) {
	ss := []string{}
	for _, v := range si {
		switch s := v.(type) {
		case string:
			ss = append(ss, s)
		default:
			return nil, errors.Errorf("expected string")
		}
	}
	return ss, nil
}

func (c *Config) Runtime(runtimeName string) *Runtime {
	for n, r := range c.Runtimes {
		if n == runtimeName {
			return r
		}
	}
	panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName))
}

func (c *Config) Task(taskName string) *Task {
	for n, t := range c.Tasks {
		if n == taskName {
			return t
		}
	}
	panic(fmt.Sprintf("task %q doesn't exists", taskName))
}

func (c *Config) Pipeline(pipelineName string) *Pipeline {
	for n, p := range c.Pipelines {
		if n == pipelineName {
			return p
		}
	}
	panic(fmt.Sprintf("pipeline %q doesn't exists", pipelineName))
}

var DefaultConfig = Config{}

func ParseConfig(configData []byte) (*Config, error) {
	config := DefaultConfig
	if err := yaml.Unmarshal(configData, &config); err != nil {
		return nil, errors.Wrapf(err, "failed to unmarshal config")
	}

	if len(config.Pipelines) == 0 {
		return nil, errors.Errorf("no pipelines defined")
	}

	// Set names from maps keys
	for n, runtime := range config.Runtimes {
		if runtime == nil {
			return nil, errors.Errorf("runtime %q is empty", n)
		}
		runtime.Name = n
	}

	for n, task := range config.Tasks {
		if task == nil {
			return nil, errors.Errorf("task %q is empty", n)
		}
		task.Name = n
	}

	for n, pipeline := range config.Pipelines {
		if pipeline == nil {
			return nil, errors.Errorf("pipeline %q is empty", n)
		}
		pipeline.Name = n
	}

	for _, pipeline := range config.Pipelines {
		for n, element := range pipeline.Elements {
			if element == nil {
				return nil, errors.Errorf("pipeline %q: element %q is empty", pipeline.Name, n)
			}
			element.Name = n
		}
	}

	// Set auth type to default if not specified
	for _, runtime := range config.Runtimes {
		if runtime.Auth != nil {
			if runtime.Auth.Type == "" {
				runtime.Auth.Type = RegistryAuthTypeDefault
			}
		}
		for _, container := range runtime.Containers {
			if container.Auth != nil {
				if container.Auth.Type == "" {
					container.Auth.Type = RegistryAuthTypeDefault
				}
			}
		}
	}

	// set steps defaults
	for _, t := range config.Tasks {
		for _, s := range t.Steps {
			switch step := s.(type) {
			// TODO(sgotti) we could use the run step command as step name but when the
			// command is very long or multi line it doesn't makes sense and will
			// probably be quite unuseful/confusing from an UI point of view
			case *SaveCacheStep:
				for _, content := range step.Contents {
					if len(content.Paths) == 0 {
						// default to all files inside the sourceDir
						content.Paths = []string{"**"}
					}
				}
				log.Infof("s: %s", util.Dump(s))
			}
		}
	}

	return &config, checkConfig(&config)
}

func checkConfig(config *Config) error {
	// check broken dependencies
	for _, pipeline := range config.Pipelines {
		// collect all task names
		allElements := map[string]struct{}{}
		for _, element := range pipeline.Elements {
			allElements[element.Name] = struct{}{}
		}

		for _, element := range pipeline.Elements {
			for _, dep := range element.Depends {
				if _, ok := allElements[dep.ElementName]; !ok {
					return errors.Errorf("pipeline element %q needed by element %q doesn't exist", dep.ElementName, element.Name)
				}
			}
		}
	}

	// check circular dependencies
	for _, pipeline := range config.Pipelines {
		cerrs := &util.Errors{}
		for _, element := range pipeline.Elements {
			allParents := getAllElementParents(pipeline, element)
			for _, parent := range allParents {
				if parent.Name == element.Name {
					// TODO(sgotti) get the parent that depends on task to report it
					dep := []string{}
					for _, parent := range allParents {
						pparents := getElementParents(pipeline, parent)
						for _, pparent := range pparents {
							if pparent.Name == element.Name {
								dep = append(dep, fmt.Sprintf("%q", parent.Name))
							}
						}
					}
					cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " ")))
				}
			}
		}
		if cerrs.IsErr() {
			return cerrs
		}
	}

	// check that the task and its parent don't have a common dependency
	for _, pipeline := range config.Pipelines {
		for _, element := range pipeline.Elements {
			parents := getElementParents(pipeline, element)
			for _, parent := range parents {
				allParents := getAllElementParents(pipeline, element)
				allParentParents := getAllElementParents(pipeline, parent)
				for _, p := range allParents {
					for _, pp := range allParentParents {
						if p.Name == pp.Name {
							return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name)
						}
					}
				}
			}
		}
	}

	for _, r := range config.Runtimes {
		if r.Type != RuntimeTypePod {
			return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type)
		}
		if len(r.Containers) == 0 {
			return errors.Errorf("runtime %q: at least one container must be defined", r.Name)
		}
	}

	for _, t := range config.Tasks {
		if len(t.Name) > maxTaskNameLength {
			return errors.Errorf("task name %q too long", t.Name)
		}
		if t.Runtime == "" {
			return errors.Errorf("task %q: undefined runtime", t.Name)
		}
		if _, ok := config.Runtimes[t.Runtime]; !ok {
			return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name)
		}
		for i, s := range t.Steps {
			switch step := s.(type) {
			// TODO(sgotti) we could use the run step command as step name but when the
			// command is very long or multi line it doesn't makes sense and will
			// probably be quite unuseful/confusing from an UI point of view
			case *RunStep:
				if step.Name == "" {
					lines, err := util.CountLines(step.Command)
					// if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred
					if err != nil || lines > 1 {
						return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name)
					}
					len := len(step.Command)
					if len > maxStepNameLength {
						len = maxStepNameLength
					}
					step.Name = step.Command[:len]
				}
			}
		}
	}

	for _, pipeline := range config.Pipelines {
		if len(pipeline.Name) > maxPipelineNameLength {
			return errors.Errorf("pipeline name %q too long", pipeline.Name)
		}
		for _, element := range pipeline.Elements {
			// check missing tasks reference
			if element.Task == "" {
				return errors.Errorf("no task defined for pipeline element %q", element.Name)
			}
			if _, ok := config.Tasks[element.Task]; !ok {
				return errors.Errorf("task %q needed by pipeline element %q doesn't exist", element.Task, element.Name)
			}
			// check duplicate dependencies in task
			seenDependencies := map[string]struct{}{}
			for _, dep := range element.Depends {
				if _, ok := seenDependencies[dep.ElementName]; ok {
					return errors.Errorf("duplicate task dependency: %s", element.Name)
				}
				seenDependencies[dep.ElementName] = struct{}{}
			}
		}
	}

	return nil
}

// getElementParents returns direct parents of element.
func getElementParents(pipeline *Pipeline, element *Element) []*Element {
	parents := []*Element{}
	for _, el := range pipeline.Elements {
		isParent := false
		for _, d := range element.Depends {
			if d.ElementName == el.Name {
				isParent = true
			}
		}
		if isParent {
			parents = append(parents, el)
		}
	}
	return parents
}

// getAllElementParents returns all the parents (both direct and ancestors) of an element.
// In case of circular dependency it won't loop forever but will also return
// the element as parent of itself
func getAllElementParents(pipeline *Pipeline, element *Element) []*Element {
	pMap := map[string]*Element{}
	nextParents := getElementParents(pipeline, element)

	for len(nextParents) > 0 {
		parents := nextParents
		nextParents = []*Element{}
		for _, parent := range parents {
			if _, ok := pMap[parent.Name]; ok {
				continue
			}
			pMap[parent.Name] = parent
			nextParents = append(nextParents, getElementParents(pipeline, parent)...)
		}
	}

	parents := make([]*Element, 0, len(pMap))
	for _, v := range pMap {
		parents = append(parents, v)
	}
	return parents
}