// Copyright 2019 Sorint.lab // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied // See the License for the specific language governing permissions and // limitations under the License. package runconfig import ( "fmt" "strings" "agola.io/agola/internal/config" "agola.io/agola/internal/util" rstypes "agola.io/agola/services/runservice/types" "agola.io/agola/services/types" errors "golang.org/x/xerrors" ) const ( defaultShell = "/bin/sh -e" ) func genRuntime(c *config.Config, ce *config.Runtime, variables map[string]string) *rstypes.Runtime { containers := []*rstypes.Container{} for _, cc := range ce.Containers { env := genEnv(cc.Environment, variables) container := &rstypes.Container{ Image: cc.Image, Environment: env, User: cc.User, Privileged: cc.Privileged, Entrypoint: cc.Entrypoint, Volumes: make([]rstypes.Volume, len(cc.Volumes)), } for i, ccVol := range cc.Volumes { container.Volumes[i] = rstypes.Volume{ Path: ccVol.Path, } if ccVol.TmpFS != nil { var size int64 if ccVol.TmpFS.Size != nil { size = ccVol.TmpFS.Size.Value() } container.Volumes[i].TmpFS = &rstypes.VolumeTmpFS{ Size: size, } } } containers = append(containers, container) } return &rstypes.Runtime{ Type: rstypes.RuntimeType(ce.Type), Arch: ce.Arch, Containers: containers, } } func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} { switch cs := csi.(type) { case *config.CloneStep: // transform a "clone" step in a "run" step command rs := &config.RunStep{} rs.Type = "run" rs.Name = "Clone repository and checkout code" rs.Command = fmt.Sprintf(` set -x mkdir ~/.ssh chmod 700 ~/.ssh touch ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa touch ~/.ssh/known_hosts chmod 600 ~/.ssh/known_hosts # Add public ssh host key if [ -n "$AGOLA_SSHHOSTKEY" ]; then echo "$AGOLA_SSHHOSTKEY" >> ~/.ssh/known_hosts fi # Add repository deploy key (cat < ~/.ssh/id_rsa $AGOLA_SSHPRIVKEY EOF ) STRICT_HOST_KEY_CHECKING="yes" if [ -n "$AGOLA_SKIPSSHHOSTKEYCHECK" ]; then # Disable git host key verification STRICT_HOST_KEY_CHECKING="no" fi (cat < ~/.ssh/config Host $AGOLA_GIT_HOST HostName $AGOLA_GIT_HOST Port $AGOLA_GIT_PORT StrictHostKeyChecking ${STRICT_HOST_KEY_CHECKING} EOF ) git clone %s $AGOLA_REPOSITORY_URL . git fetch origin $AGOLA_GIT_REF if [ -n "$AGOLA_GIT_COMMITSHA" ]; then git checkout $AGOLA_GIT_COMMITSHA else git checkout FETCH_HEAD fi `, genCloneOptions(cs)) return rs case *config.RunStep: rs := &rstypes.RunStep{} env := genEnv(cs.Environment, variables) rs.Type = cs.Type rs.Name = cs.Name rs.Command = cs.Command rs.Environment = env rs.WorkingDir = cs.WorkingDir rs.Shell = cs.Shell return rs case *config.SaveToWorkspaceStep: sws := &rstypes.SaveToWorkspaceStep{} sws.Type = cs.Type sws.Name = cs.Name sws.Contents = make([]rstypes.SaveContent, len(cs.Contents)) for i, csc := range cs.Contents { sc := rstypes.SaveContent{} sc.SourceDir = csc.SourceDir sc.DestDir = csc.DestDir sc.Paths = csc.Paths sws.Contents[i] = sc } return sws case *config.RestoreWorkspaceStep: rws := &rstypes.RestoreWorkspaceStep{} rws.Name = cs.Name rws.Type = cs.Type rws.DestDir = cs.DestDir return rws case *config.SaveCacheStep: sws := &rstypes.SaveCacheStep{} sws.Type = cs.Type sws.Name = cs.Name sws.Key = cs.Key sws.Contents = make([]rstypes.SaveContent, len(cs.Contents)) for i, csc := range cs.Contents { sc := rstypes.SaveContent{} sc.SourceDir = csc.SourceDir sc.DestDir = csc.DestDir sc.Paths = csc.Paths sws.Contents[i] = sc } return sws case *config.RestoreCacheStep: rws := &rstypes.RestoreCacheStep{} rws.Name = cs.Name rws.Type = cs.Type rws.Keys = cs.Keys rws.DestDir = cs.DestDir return rws default: panic(fmt.Errorf("unknown config step type: %s", util.Dump(cs))) } } // GenRunConfigTasks generates a run config tasks from a run in the config, expanding all the references to tasks // this functions assumes that the config is already checked for possible errors (i.e referenced task must exits) func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string, variables map[string]string, branch, tag, ref string) map[string]*rstypes.RunConfigTask { cr := c.Run(runName) rcts := map[string]*rstypes.RunConfigTask{} for _, ct := range cr.Tasks { include := types.MatchWhen(ct.When.ToWhen(), branch, tag, ref) steps := make(rstypes.Steps, len(ct.Steps)) for i, cpts := range ct.Steps { steps[i] = stepFromConfigStep(cpts, variables) } tEnv := genEnv(ct.Environment, variables) t := &rstypes.RunConfigTask{ ID: uuid.New(ct.Name).String(), Name: ct.Name, Runtime: genRuntime(c, ct.Runtime, variables), Environment: tEnv, WorkingDir: ct.WorkingDir, Shell: ct.Shell, User: ct.User, Steps: steps, IgnoreFailure: ct.IgnoreFailure, Skip: !include, NeedsApproval: ct.Approval, DockerRegistriesAuth: make(map[string]rstypes.DockerRegistryAuth), } if t.Shell == "" { t.Shell = defaultShell } if c.DockerRegistriesAuth != nil { for regname, auth := range c.DockerRegistriesAuth { t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{ Type: rstypes.DockerRegistryAuthType(auth.Type), Username: genValue(auth.Username, variables), Password: genValue(auth.Password, variables), } } } // override with per run docker registry auth if cr.DockerRegistriesAuth != nil { for regname, auth := range cr.DockerRegistriesAuth { t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{ Type: rstypes.DockerRegistryAuthType(auth.Type), Username: genValue(auth.Username, variables), Password: genValue(auth.Password, variables), } } } // override with per task docker registry auth if ct.DockerRegistriesAuth != nil { for regname, auth := range ct.DockerRegistriesAuth { t.DockerRegistriesAuth[regname] = rstypes.DockerRegistryAuth{ Type: rstypes.DockerRegistryAuthType(auth.Type), Username: genValue(auth.Username, variables), Password: genValue(auth.Password, variables), } } } rcts[t.ID] = t } // populate depends, needs to be done after having created all the tasks so we can resolve their id for _, rct := range rcts { ct := cr.Task(rct.Name) depends := make(map[string]*rstypes.RunConfigTaskDepend, len(ct.Depends)) for _, d := range ct.Depends { conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions)) // when no conditions are defined default to on_success if len(d.Conditions) == 0 { conditions = append(conditions, rstypes.RunConfigTaskDependConditionOnSuccess) } else { for ic, c := range d.Conditions { var condition rstypes.RunConfigTaskDependCondition switch c { case config.DependConditionOnSuccess: condition = rstypes.RunConfigTaskDependConditionOnSuccess case config.DependConditionOnFailure: condition = rstypes.RunConfigTaskDependConditionOnFailure } conditions[ic] = condition } } drct := getRunConfigTaskByName(rcts, d.TaskName) depends[drct.ID] = &rstypes.RunConfigTaskDepend{ TaskID: drct.ID, Conditions: conditions, } } rct.Depends = depends } return rcts } func getRunConfigTaskByName(rcts map[string]*rstypes.RunConfigTask, name string) *rstypes.RunConfigTask { for _, rct := range rcts { if rct.Name == name { return rct } } return nil } func CheckRunConfigTasks(rcts map[string]*rstypes.RunConfigTask) error { // check circular dependencies cerrs := &util.Errors{} for _, t := range rcts { allParents := GetAllParents(rcts, t) for _, parent := range allParents { if parent.ID == t.ID { // TODO(sgotti) get the parent that depends on task to report it dep := []string{} for _, parent := range allParents { pparents := GetParents(rcts, parent) for _, pparent := range pparents { if pparent.ID == t.ID { dep = append(dep, fmt.Sprintf("%q", parent.Name)) } } } cerrs.Append(errors.Errorf("circular dependency between task %q and tasks %s", t.Name, strings.Join(dep, " "))) } } } if cerrs.IsErr() { return cerrs } // check that the task and its parent don't have a common dependency for _, t := range rcts { parents := GetParents(rcts, t) for _, parent := range parents { allParentParents := GetAllParents(rcts, parent) for _, p := range parents { for _, pp := range allParentParents { if p.ID == pp.ID { return errors.Errorf("task %q and its parent %q have both a dependency on task %q", t.Name, parent.Name, p.Name) } } } } } return nil } func GenTasksLevels(rcts map[string]*rstypes.RunConfigTask) error { // reset all task level for _, t := range rcts { t.Level = -1 } level := 0 for { c := 0 for _, t := range rcts { // skip tasks with the level already set if t.Level != -1 { continue } parents := GetParents(rcts, t) ok := true for _, p := range parents { // * skip if the parent doesn't have a level yet // * skip if the parent has a level equal than the current one (this happens when // we have just set a level to a task in this same level loop) if p.Level == -1 || p.Level >= level { ok = false } } if ok { t.Level = level c++ } } // if no tasks were updated in this level we can stop here if c == 0 { break } level++ } for _, t := range rcts { if t.Level == -1 { return errors.Errorf("circular dependency detected") } } return nil } // GetParents returns direct parents of task. func GetParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask { parents := []*rstypes.RunConfigTask{} for _, t := range rcts { if _, ok := task.Depends[t.ID]; ok { parents = append(parents, t) } } return parents } // GetAllParents returns all the parents (both direct and ancestors) of task. // In case of circular dependency it won't loop forever but will also return // task as parent of itself func GetAllParents(rcts map[string]*rstypes.RunConfigTask, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask { pMap := map[string]*rstypes.RunConfigTask{} nextParents := GetParents(rcts, task) for len(nextParents) > 0 { parents := nextParents nextParents = []*rstypes.RunConfigTask{} for _, parent := range parents { if _, ok := pMap[parent.ID]; ok { continue } pMap[parent.ID] = parent nextParents = append(nextParents, GetParents(rcts, parent)...) } } parents := make([]*rstypes.RunConfigTask, 0, len(pMap)) for _, v := range pMap { parents = append(parents, v) } return parents } func GetParentDependConditions(t, pt *rstypes.RunConfigTask) []rstypes.RunConfigTaskDependCondition { if dt, ok := t.Depends[pt.ID]; ok { return dt.Conditions } return nil } func genEnv(cenv map[string]config.Value, variables map[string]string) map[string]string { env := map[string]string{} for envName, envVar := range cenv { env[envName] = genValue(envVar, variables) } return env } func genValue(val config.Value, variables map[string]string) string { switch val.Type { case config.ValueTypeString: return val.Value case config.ValueTypeFromVariable: return variables[val.Value] default: panic(fmt.Errorf("wrong value type: %q", val.Value)) } } func genCloneOptions(c *config.CloneStep) string { cloneoptions := []string{} if c.Depth != nil { cloneoptions = append(cloneoptions, fmt.Sprintf("--depth %d", *c.Depth)) } if c.RecurseSubmodules { cloneoptions = append(cloneoptions, "--recurse-submodules") } return strings.Join(cloneoptions, " ") }