agola/internal/runconfig/runconfig.go

475 lines
12 KiB
Go
Raw Normal View History

2019-02-21 14:54:50 +00:00
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package runconfig
import (
"fmt"
"strings"
2019-07-01 09:40:20 +00:00
"agola.io/agola/internal/config"
itypes "agola.io/agola/internal/services/types"
2019-07-01 09:40:20 +00:00
"agola.io/agola/internal/util"
rstypes "agola.io/agola/services/runservice/types"
"agola.io/agola/services/types"
errors "golang.org/x/xerrors"
2019-02-21 14:54:50 +00:00
)
const (
defaultShell = "/bin/sh -e"
)
func genRuntime(c *config.Config, ce *config.Runtime, variables map[string]string) *rstypes.Runtime {
containers := []*rstypes.Container{}
2019-02-21 14:54:50 +00:00
for _, cc := range ce.Containers {
env := genEnv(cc.Environment, variables)
2019-03-18 14:02:32 +00:00
container := &rstypes.Container{
2019-02-21 14:54:50 +00:00
Image: cc.Image,
2019-03-18 14:02:32 +00:00
Environment: env,
2019-02-21 14:54:50 +00:00
User: cc.User,
Privileged: cc.Privileged,
Entrypoint: cc.Entrypoint,
Volumes: make([]rstypes.Volume, len(cc.Volumes)),
2019-03-18 14:02:32 +00:00
}
for i, ccVol := range cc.Volumes {
container.Volumes[i] = rstypes.Volume{
Path: ccVol.Path,
}
if ccVol.TmpFS != nil {
var size int64
if ccVol.TmpFS.Size != nil {
size = ccVol.TmpFS.Size.Value()
}
container.Volumes[i].TmpFS = &rstypes.VolumeTmpFS{
Size: size,
}
}
}
2019-03-18 14:02:32 +00:00
containers = append(containers, container)
2019-02-21 14:54:50 +00:00
}
return &rstypes.Runtime{
Type: rstypes.RuntimeType(ce.Type),
Arch: ce.Arch,
2019-02-21 14:54:50 +00:00
Containers: containers,
}
}
2019-03-18 14:02:32 +00:00
func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} {
2019-02-21 14:54:50 +00:00
switch cs := csi.(type) {
case *config.CloneStep:
// transform a "clone" step in a "run" step command
rs := &config.RunStep{}
rs.Type = "run"
rs.Name = "Clone repository and checkout code"
2019-08-19 09:20:27 +00:00
rs.Command = fmt.Sprintf(`
2019-02-21 14:54:50 +00:00
set -x
mkdir ~/.ssh
chmod 700 ~/.ssh
touch ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
touch ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
# Add public ssh host key
if [ -n "$AGOLA_SSHHOSTKEY" ]; then
echo "$AGOLA_SSHHOSTKEY" >> ~/.ssh/known_hosts
fi
2019-02-21 14:54:50 +00:00
# Add repository deploy key
(cat <<EOF > ~/.ssh/id_rsa
$AGOLA_SSHPRIVKEY
EOF
)
STRICT_HOST_KEY_CHECKING="yes"
2019-02-21 14:54:50 +00:00
if [ -n "$AGOLA_SKIPSSHHOSTKEYCHECK" ]; then
# Disable git host key verification
STRICT_HOST_KEY_CHECKING="no"
fi
(cat <<EOF > ~/.ssh/config
2019-02-21 14:54:50 +00:00
Host $AGOLA_GIT_HOST
HostName $AGOLA_GIT_HOST
Port $AGOLA_GIT_PORT
StrictHostKeyChecking ${STRICT_HOST_KEY_CHECKING}
2019-02-21 14:54:50 +00:00
EOF
)
2019-02-21 14:54:50 +00:00
2019-08-19 09:20:27 +00:00
git clone %s $AGOLA_REPOSITORY_URL .
2019-02-21 14:54:50 +00:00
git fetch origin $AGOLA_GIT_REF
if [ -n "$AGOLA_GIT_COMMITSHA" ]; then
git checkout $AGOLA_GIT_COMMITSHA
else
git checkout FETCH_HEAD
fi
2019-08-19 09:20:27 +00:00
`, genCloneOptions(cs))
2019-02-21 14:54:50 +00:00
return rs
case *config.RunStep:
rs := &rstypes.RunStep{}
2019-02-21 14:54:50 +00:00
env := genEnv(cs.Environment, variables)
2019-03-18 14:02:32 +00:00
2019-02-21 14:54:50 +00:00
rs.Type = cs.Type
rs.Name = cs.Name
rs.Command = cs.Command
2019-03-18 14:02:32 +00:00
rs.Environment = env
2019-02-21 14:54:50 +00:00
rs.WorkingDir = cs.WorkingDir
rs.Shell = cs.Shell
rs.Tty = cs.Tty
2019-02-21 14:54:50 +00:00
return rs
case *config.SaveToWorkspaceStep:
sws := &rstypes.SaveToWorkspaceStep{}
2019-02-21 14:54:50 +00:00
sws.Type = cs.Type
sws.Name = cs.Name
sws.Contents = make([]rstypes.SaveContent, len(cs.Contents))
2019-02-21 14:54:50 +00:00
for i, csc := range cs.Contents {
sc := rstypes.SaveContent{}
2019-02-21 14:54:50 +00:00
sc.SourceDir = csc.SourceDir
sc.DestDir = csc.DestDir
sc.Paths = csc.Paths
sws.Contents[i] = sc
}
return sws
case *config.RestoreWorkspaceStep:
rws := &rstypes.RestoreWorkspaceStep{}
2019-02-21 14:54:50 +00:00
rws.Name = cs.Name
rws.Type = cs.Type
rws.DestDir = cs.DestDir
return rws
case *config.SaveCacheStep:
sws := &rstypes.SaveCacheStep{}
sws.Type = cs.Type
sws.Name = cs.Name
sws.Key = cs.Key
sws.Contents = make([]rstypes.SaveContent, len(cs.Contents))
for i, csc := range cs.Contents {
sc := rstypes.SaveContent{}
sc.SourceDir = csc.SourceDir
sc.DestDir = csc.DestDir
sc.Paths = csc.Paths
sws.Contents[i] = sc
}
return sws
case *config.RestoreCacheStep:
rws := &rstypes.RestoreCacheStep{}
rws.Name = cs.Name
rws.Type = cs.Type
rws.Keys = cs.Keys
rws.DestDir = cs.DestDir
return rws
2019-02-21 14:54:50 +00:00
default:
panic(fmt.Errorf("unknown config step type: %s", util.Dump(cs)))
}
}
// GenRunConfigTasks generates a run config tasks from a run in the config, expanding all the references to tasks
2019-02-21 14:54:50 +00:00
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string, variables map[string]string, refType itypes.RunRefType, branch, tag, ref string) map[string]*rstypes.RunConfigTask {
cr := c.Run(runName)
2019-02-21 14:54:50 +00:00
rcts := map[string]*rstypes.RunConfigTask{}
2019-02-21 14:54:50 +00:00
for _, ct := range cr.Tasks {
include := types.MatchWhen(ct.When.ToWhen(), refType, branch, tag, ref)
2019-02-21 14:54:50 +00:00
steps := make(rstypes.Steps, len(ct.Steps))
for i, cpts := range ct.Steps {
2019-03-18 14:02:32 +00:00
steps[i] = stepFromConfigStep(cpts, variables)
}
tEnv := genEnv(ct.Environment, variables)
2019-02-21 14:54:50 +00:00
t := &rstypes.RunConfigTask{
ID: uuid.New(ct.Name).String(),
Name: ct.Name,
Runtime: genRuntime(c, ct.Runtime, variables),
Environment: tEnv,
WorkingDir: ct.WorkingDir,
Shell: ct.Shell,
User: ct.User,
Steps: steps,
IgnoreFailure: ct.IgnoreFailure,
Skip: !include,
NeedsApproval: ct.Approval,
DockerRegistriesAuth: make(map[string]rstypes.DockerRegistryAuth),
}
if t.Shell == "" {
t.Shell = defaultShell
}
if c.DockerRegistriesAuth != nil {
for regname, auth := range c.DockerRegistriesAuth {
t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{
Type: rstypes.DockerRegistryAuthType(auth.Type),
Username: genValue(auth.Username, variables),
Password: genValue(auth.Password, variables),
}
}
}
// override with per run docker registry auth
if cr.DockerRegistriesAuth != nil {
for regname, auth := range cr.DockerRegistriesAuth {
t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{
Type: rstypes.DockerRegistryAuthType(auth.Type),
Username: genValue(auth.Username, variables),
Password: genValue(auth.Password, variables),
}
}
}
// override with per task docker registry auth
if ct.DockerRegistriesAuth != nil {
for regname, auth := range ct.DockerRegistriesAuth {
t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{
Type: rstypes.DockerRegistryAuthType(auth.Type),
Username: genValue(auth.Username, variables),
Password: genValue(auth.Password, variables),
}
}
2019-02-21 14:54:50 +00:00
}
rcts[t.ID] = t
2019-02-21 14:54:50 +00:00
}
// populate depends, needs to be done after having created all the tasks so we can resolve their id
for _, rct := range rcts {
ct := cr.Task(rct.Name)
2019-02-21 14:54:50 +00:00
depends := make(map[string]*rstypes.RunConfigTaskDepend, len(ct.Depends))
for _, d := range ct.Depends {
conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
2019-02-21 14:54:50 +00:00
// when no conditions are defined default to on_success
if len(d.Conditions) == 0 {
conditions = append(conditions, rstypes.RunConfigTaskDependConditionOnSuccess)
2019-02-21 14:54:50 +00:00
} else {
for ic, c := range d.Conditions {
var condition rstypes.RunConfigTaskDependCondition
2019-02-21 14:54:50 +00:00
switch c {
case config.DependConditionOnSuccess:
condition = rstypes.RunConfigTaskDependConditionOnSuccess
2019-02-21 14:54:50 +00:00
case config.DependConditionOnFailure:
condition = rstypes.RunConfigTaskDependConditionOnFailure
2019-02-21 14:54:50 +00:00
}
conditions[ic] = condition
}
}
drct := getRunConfigTaskByName(rcts, d.TaskName)
depends[drct.ID] = &rstypes.RunConfigTaskDepend{
2019-02-21 14:54:50 +00:00
TaskID: drct.ID,
Conditions: conditions,
}
}
rct.Depends = depends
}
return rcts
2019-02-21 14:54:50 +00:00
}
func getRunConfigTaskByName(rcts map[string]*rstypes.RunConfigTask, name string) *rstypes.RunConfigTask {
for _, rct := range rcts {
2019-02-21 14:54:50 +00:00
if rct.Name == name {
return rct
}
}
return nil
}
func CheckRunConfigTasks(rcts map[string]*rstypes.RunConfigTask) error {
2019-02-21 14:54:50 +00:00
// check circular dependencies
cerrs := &util.Errors{}
for _, t := range rcts {
allParents := GetAllParents(rcts, t)
2019-02-21 14:54:50 +00:00
for _, parent := range allParents {
if parent.ID == t.ID {
// TODO(sgotti) get the parent that depends on task to report it
dep := []string{}
for _, parent := range allParents {
pparents := GetParents(rcts, parent)
2019-02-21 14:54:50 +00:00
for _, pparent := range pparents {
if pparent.ID == t.ID {
dep = append(dep, fmt.Sprintf("%q", parent.Name))
}
}
}
cerrs.Append(errors.Errorf("circular dependency between task %q and tasks %s", t.Name, strings.Join(dep, " ")))
}
}
}
if cerrs.IsErr() {
return cerrs
}
// check that the task and its parent don't have a common dependency
for _, t := range rcts {
parents := GetParents(rcts, t)
2019-02-21 14:54:50 +00:00
for _, parent := range parents {
allParentParents := GetAllParents(rcts, parent)
for _, p := range parents {
2019-02-21 14:54:50 +00:00
for _, pp := range allParentParents {
if p.ID == pp.ID {
return errors.Errorf("task %q and its parent %q have both a dependency on task %q", t.Name, parent.Name, p.Name)
2019-02-21 14:54:50 +00:00
}
}
}
}
}
return nil
}
func GenTasksLevels(rcts map[string]*rstypes.RunConfigTask) error {
2019-02-21 14:54:50 +00:00
// reset all task level
for _, t := range rcts {
2019-02-21 14:54:50 +00:00
t.Level = -1
}
level := 0
for {
c := 0
for _, t := range rcts {
2019-02-21 14:54:50 +00:00
// skip tasks with the level already set
if t.Level != -1 {
continue
}
parents := GetParents(rcts, t)
2019-02-21 14:54:50 +00:00
ok := true
for _, p := range parents {
// * skip if the parent doesn't have a level yet
// * skip if the parent has a level equal than the current one (this happens when
// we have just set a level to a task in this same level loop)
if p.Level == -1 || p.Level >= level {
ok = false
}
}
if ok {
t.Level = level
c++
}
}
// if no tasks were updated in this level we can stop here
if c == 0 {
break
}
level++
}
for _, t := range rcts {
2019-02-21 14:54:50 +00:00
if t.Level == -1 {
return errors.Errorf("circular dependency detected")
}
}
return nil
}
// GetParents returns direct parents of task.
func GetParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
parents := []*rstypes.RunConfigTask{}
for _, t := range rcts {
if _, ok := task.Depends[t.ID]; ok {
2019-02-21 14:54:50 +00:00
parents = append(parents, t)
}
}
return parents
}
// GetAllParents returns all the parents (both direct and ancestors) of task.
// In case of circular dependency it won't loop forever but will also return
// task as parent of itself
func GetAllParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
pMap := map[string]*rstypes.RunConfigTask{}
nextParents := GetParents(rcts, task)
2019-02-21 14:54:50 +00:00
for len(nextParents) > 0 {
parents := nextParents
nextParents = []*rstypes.RunConfigTask{}
2019-02-21 14:54:50 +00:00
for _, parent := range parents {
if _, ok := pMap[parent.ID]; ok {
continue
}
pMap[parent.ID] = parent
nextParents = append(nextParents, GetParents(rcts, parent)...)
2019-02-21 14:54:50 +00:00
}
}
parents := make([]*rstypes.RunConfigTask, 0, len(pMap))
2019-02-21 14:54:50 +00:00
for _, v := range pMap {
parents = append(parents, v)
}
return parents
}
2019-03-18 14:02:32 +00:00
func GetParentDependConditions(t, pt *rstypes.RunConfigTask) []rstypes.RunConfigTaskDependCondition {
if dt, ok := t.Depends[pt.ID]; ok {
return dt.Conditions
}
return nil
}
func genEnv(cenv map[string]config.Value, variables map[string]string) map[string]string {
2019-03-18 14:02:32 +00:00
env := map[string]string{}
for envName, envVar := range cenv {
env[envName] = genValue(envVar, variables)
}
return env
}
func genValue(val config.Value, variables map[string]string) string {
switch val.Type {
case config.ValueTypeString:
return val.Value
case config.ValueTypeFromVariable:
return variables[val.Value]
default:
panic(fmt.Errorf("wrong value type: %q", val.Value))
2019-03-18 14:02:32 +00:00
}
}
2019-08-19 09:20:27 +00:00
func genCloneOptions(c *config.CloneStep) string {
cloneoptions := []string{}
if c.Depth != nil {
cloneoptions = append(cloneoptions, fmt.Sprintf("--depth %d", *c.Depth))
}
if c.RecurseSubmodules {
cloneoptions = append(cloneoptions, "--recurse-submodules")
}
return strings.Join(cloneoptions, " ")
}