agola/internal/runconfig/runconfig.go

379 lines
9.5 KiB
Go
Raw Normal View History

2019-02-21 14:54:50 +00:00
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package runconfig
import (
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/config"
rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
"github.com/sorintlab/agola/internal/services/types"
2019-02-21 14:54:50 +00:00
"github.com/sorintlab/agola/internal/util"
)
2019-03-18 14:02:32 +00:00
func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime {
2019-02-21 14:54:50 +00:00
ce := c.Runtime(runtimeName)
containers := []*rstypes.Container{}
2019-02-21 14:54:50 +00:00
for _, cc := range ce.Containers {
2019-03-18 14:02:32 +00:00
env, err := genEnv(cc.Environment, variables)
if err != nil {
return nil
}
container := &rstypes.Container{
2019-02-21 14:54:50 +00:00
Image: cc.Image,
2019-03-18 14:02:32 +00:00
Environment: env,
2019-02-21 14:54:50 +00:00
User: cc.User,
Privileged: cc.Privileged,
Entrypoint: cc.Entrypoint,
2019-03-18 14:02:32 +00:00
}
containers = append(containers, container)
2019-02-21 14:54:50 +00:00
}
return &rstypes.Runtime{
Type: rstypes.RuntimeType(ce.Type),
2019-02-21 14:54:50 +00:00
Containers: containers,
}
}
2019-03-18 14:02:32 +00:00
func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} {
2019-02-21 14:54:50 +00:00
switch cs := csi.(type) {
case *config.CloneStep:
// transform a "clone" step in a "run" step command
rs := &config.RunStep{}
rs.Type = "run"
rs.Name = "Clone repository and checkout code"
rs.Command = `
set -x
mkdir ~/.ssh
chmod 700 ~/.ssh
touch ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
# Add repository deploy key
(cat <<EOF > ~/.ssh/id_rsa
$AGOLA_SSHPRIVKEY
EOF
)
if [ -n "$AGOLA_SKIPSSHHOSTKEYCHECK" ]; then
# Disable git host key verification
(cat <<EOF > ~/.ssh/config
Host $AGOLA_GIT_HOST
HostName $AGOLA_GIT_HOST
Port $AGOLA_GIT_PORT
2019-02-21 14:54:50 +00:00
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
EOF
)
fi
git clone $AGOLA_REPOSITORY_URL .
git fetch origin $AGOLA_GIT_REF
if [ -n "$AGOLA_GIT_COMMITSHA" ]; then
git checkout $AGOLA_GIT_COMMITSHA
else
git checkout FETCH_HEAD
fi
`
return rs
case *config.RunStep:
rs := &rstypes.RunStep{}
2019-02-21 14:54:50 +00:00
2019-03-18 14:02:32 +00:00
env, err := genEnv(cs.Environment, variables)
if err != nil {
return nil
}
2019-02-21 14:54:50 +00:00
rs.Type = cs.Type
rs.Name = cs.Name
rs.Command = cs.Command
2019-03-18 14:02:32 +00:00
rs.Environment = env
2019-02-21 14:54:50 +00:00
rs.WorkingDir = cs.WorkingDir
rs.Shell = cs.Shell
rs.User = cs.User
return rs
case *config.SaveToWorkspaceStep:
sws := &rstypes.SaveToWorkspaceStep{}
2019-02-21 14:54:50 +00:00
sws.Type = cs.Type
sws.Name = cs.Name
sws.Contents = make([]rstypes.SaveToWorkspaceContent, len(cs.Contents))
2019-02-21 14:54:50 +00:00
for i, csc := range cs.Contents {
sc := rstypes.SaveToWorkspaceContent{}
2019-02-21 14:54:50 +00:00
sc.SourceDir = csc.SourceDir
sc.DestDir = csc.DestDir
sc.Paths = csc.Paths
sws.Contents[i] = sc
}
return sws
case *config.RestoreWorkspaceStep:
rws := &rstypes.RestoreWorkspaceStep{}
2019-02-21 14:54:50 +00:00
rws.Name = cs.Name
rws.Type = cs.Type
rws.DestDir = cs.DestDir
return rws
default:
panic(fmt.Errorf("unknown config step type: %s", util.Dump(cs)))
}
}
// GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
2019-03-18 14:02:32 +00:00
func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string, env, variables map[string]string, branch, tag, ref string) *rstypes.RunConfig {
2019-02-21 14:54:50 +00:00
cp := c.Pipeline(pipelineName)
rc := &rstypes.RunConfig{
2019-02-21 14:54:50 +00:00
Name: cp.Name,
Tasks: make(map[string]*rstypes.RunConfigTask),
2019-02-21 14:54:50 +00:00
Environment: env,
}
for _, cpe := range cp.Elements {
include := types.MatchWhen(cpe.When, branch, tag, ref)
2019-02-21 14:54:50 +00:00
// resolve referenced task
cpt := c.Task(cpe.Task)
steps := make([]interface{}, len(cpt.Steps))
for i, cpts := range cpt.Steps {
2019-03-18 14:02:32 +00:00
steps[i] = stepFromConfigStep(cpts, variables)
}
tEnv, err := genEnv(cpt.Environment, variables)
if err != nil {
return nil
2019-02-21 14:54:50 +00:00
}
t := &rstypes.RunConfigTask{
2019-03-18 14:02:32 +00:00
ID: uuid.New(cpe.Name).String(),
2019-02-21 14:54:50 +00:00
// use the element name from the config as the task name
Name: cpe.Name,
2019-03-18 14:02:32 +00:00
Runtime: genRuntime(c, cpt.Runtime, variables),
Environment: tEnv,
2019-02-21 14:54:50 +00:00
WorkingDir: cpt.WorkingDir,
Shell: cpt.Shell,
User: cpt.User,
Steps: steps,
IgnoreFailure: cpe.IgnoreFailure,
Skip: !include,
2019-02-21 14:54:50 +00:00
}
rc.Tasks[t.ID] = t
}
// populate depends, needs to be done after having created all the tasks so we can resolve their id
for _, rct := range rc.Tasks {
cpe := cp.Elements[rct.Name]
depends := make([]*rstypes.RunConfigTaskDepend, len(cpe.Depends))
2019-02-21 14:54:50 +00:00
for id, d := range cpe.Depends {
conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
2019-02-21 14:54:50 +00:00
// when no conditions are defined default to on_success
if len(d.Conditions) == 0 {
conditions = append(conditions, rstypes.RunConfigTaskDependConditionOnSuccess)
2019-02-21 14:54:50 +00:00
} else {
for ic, c := range d.Conditions {
var condition rstypes.RunConfigTaskDependCondition
2019-02-21 14:54:50 +00:00
switch c {
case config.DependConditionOnSuccess:
condition = rstypes.RunConfigTaskDependConditionOnSuccess
2019-02-21 14:54:50 +00:00
case config.DependConditionOnFailure:
condition = rstypes.RunConfigTaskDependConditionOnFailure
2019-02-21 14:54:50 +00:00
}
conditions[ic] = condition
}
}
drct := getRunConfigTaskByName(rc, d.ElementName)
depends[id] = &rstypes.RunConfigTaskDepend{
2019-02-21 14:54:50 +00:00
TaskID: drct.ID,
Conditions: conditions,
}
}
rct.Depends = depends
}
return rc
}
func getRunConfigTaskByName(rc *rstypes.RunConfig, name string) *rstypes.RunConfigTask {
2019-02-21 14:54:50 +00:00
for _, rct := range rc.Tasks {
if rct.Name == name {
return rct
}
}
return nil
}
func CheckRunConfig(rc *rstypes.RunConfig) error {
2019-02-21 14:54:50 +00:00
// check circular dependencies
cerrs := &util.Errors{}
for _, t := range rc.Tasks {
allParents := GetAllParents(rc, t)
for _, parent := range allParents {
if parent.ID == t.ID {
// TODO(sgotti) get the parent that depends on task to report it
dep := []string{}
for _, parent := range allParents {
pparents := GetParents(rc, parent)
for _, pparent := range pparents {
if pparent.ID == t.ID {
dep = append(dep, fmt.Sprintf("%q", parent.Name))
}
}
}
cerrs.Append(errors.Errorf("circular dependency between task %q and tasks %s", t.Name, strings.Join(dep, " ")))
}
}
}
if cerrs.IsErr() {
return cerrs
}
// check that the task and its parent don't have a common dependency
for _, t := range rc.Tasks {
parents := GetParents(rc, t)
for _, parent := range parents {
allParents := GetAllParents(rc, t)
allParentParents := GetAllParents(rc, parent)
for _, p := range allParents {
for _, pp := range allParentParents {
if p.ID == pp.ID {
return errors.Errorf("task %s and its parent %s have both a dependency on task %s", t.Name, parent.Name, p.Name)
}
}
}
}
}
return nil
}
func GenTasksLevels(rc *rstypes.RunConfig) error {
2019-02-21 14:54:50 +00:00
// reset all task level
for _, t := range rc.Tasks {
t.Level = -1
}
level := 0
for {
c := 0
for _, t := range rc.Tasks {
// skip tasks with the level already set
if t.Level != -1 {
continue
}
parents := GetParents(rc, t)
ok := true
for _, p := range parents {
// * skip if the parent doesn't have a level yet
// * skip if the parent has a level equal than the current one (this happens when
// we have just set a level to a task in this same level loop)
if p.Level == -1 || p.Level >= level {
ok = false
}
}
if ok {
t.Level = level
c++
}
}
// if no tasks were updated in this level we can stop here
if c == 0 {
break
}
level++
}
for _, t := range rc.Tasks {
if t.Level == -1 {
return errors.Errorf("circular dependency detected")
}
}
return nil
}
// GetParents returns direct parents of task.
func GetParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
parents := []*rstypes.RunConfigTask{}
2019-02-21 14:54:50 +00:00
for _, t := range rc.Tasks {
isParent := false
for _, d := range task.Depends {
if d.TaskID == t.ID {
isParent = true
}
}
if isParent {
parents = append(parents, t)
}
}
return parents
}
// GetAllParents returns all the parents (both direct and ancestors) of task.
// In case of circular dependency it won't loop forever but will also return
// task as parent of itself
func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
pMap := map[string]*rstypes.RunConfigTask{}
2019-02-21 14:54:50 +00:00
nextParents := GetParents(rc, task)
for len(nextParents) > 0 {
parents := nextParents
nextParents = []*rstypes.RunConfigTask{}
2019-02-21 14:54:50 +00:00
for _, parent := range parents {
if _, ok := pMap[parent.ID]; ok {
continue
}
pMap[parent.ID] = parent
nextParents = append(nextParents, GetParents(rc, parent)...)
}
}
parents := make([]*rstypes.RunConfigTask, 0, len(pMap))
2019-02-21 14:54:50 +00:00
for _, v := range pMap {
parents = append(parents, v)
}
return parents
}
2019-03-18 14:02:32 +00:00
func genEnv(cenv map[string]config.EnvVar, variables map[string]string) (map[string]string, error) {
env := map[string]string{}
for envName, envVar := range cenv {
switch envVar.Type {
case config.EnvVarTypeString:
env[envName] = envVar.Value
case config.EnvVarTypeFromVariable:
env[envName] = variables[envVar.Value]
}
}
return env, nil
}