a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

*: initial implementation of when conditions

This commit is contained in:
Simone Gotti 2019-03-07 18:01:34 +01:00
parent a4ad66ac2d
commit 6f38c48066
8 changed files with 433 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
internal/config/config.go
View File

@ -16,9 +16,11 @@ package config
import (
"fmt"
"regexp"
"strings"
"github.com/sorintlab/agola/internal/common"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
"github.com/pkg/errors"
@ -31,6 +33,10 @@ const (
maxStepNameLength = 100
)
var (
regExpDelimiters = []string{"/", "#"}
)
type Config struct {
Runtimes map[string]*Runtime `yaml:"runtimes"`
Tasks map[string]*Task `yaml:"tasks"`
@ -77,6 +83,7 @@ type Element struct {
Depends []*Depend `yaml:"depends"`
IgnoreFailure bool `yaml:"ignore_failure"`
Approval bool `yaml:"approval"`
When *types.When `yaml:"when"`
}
type DependCondition string
@ -88,7 +95,7 @@ const (
type Depend struct {
ElementName string `yaml:"name"`
Conditions []DependCondition `yaml: "conditions"`
Conditions []DependCondition `yaml:"conditions"`
}
type Step struct {
@ -196,12 +203,20 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
}
func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
type when struct {
Branch interface{} `yaml:"branch"`
Tag interface{} `yaml:"tag"`
Ref interface{} `yaml:"ref"`
}
type element struct {
Name string `yaml:"name"`
Task string `yaml:"task"`
Depends []interface{} `yaml:"depends"`
IgnoreFailure bool `yaml:"ignore_failure"`
When *when `yaml:"when"`
}
var te *element
if err := unmarshal(&te); err != nil {
@ -248,9 +263,161 @@ func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
e.Depends = depends
if te.When != nil {
w := &types.When{}
var err error
if te.When.Branch != nil {
w.Branch, err = parseWhenConditions(te.When.Branch)
if err != nil {
return err
}
}
if te.When.Tag != nil {
w.Tag, err = parseWhenConditions(te.When.Tag)
if err != nil {
return err
}
}
if te.When.Ref != nil {
w.Ref, err = parseWhenConditions(te.When.Ref)
if err != nil {
return err
}
}
e.When = w
}
return nil
}
func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
w := &types.WhenConditions{}
var err error
include := []string{}
exclude := []string{}
switch c := wi.(type) {
case string:
include = []string{c}
case []interface{}:
ss, err := parseSliceString(c)
if err != nil {
return nil, err
}
include = ss
case map[interface{}]interface{}:
for k, v := range c {
ks, ok := k.(string)
if !ok {
return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
}
switch ks {
case "include":
include, err = parseStringOrSlice(v)
if err != nil {
return nil, err
}
case "exclude":
exclude, err = parseStringOrSlice(v)
if err != nil {
return nil, err
}
default:
return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks)
}
}
default:
return nil, errors.Errorf("wrong when format")
}
w.Include, err = parseWhenConditionSlice(include)
if err != nil {
return nil, err
}
w.Exclude, err = parseWhenConditionSlice(exclude)
if err != nil {
return nil, err
}
return w, nil
}
func parseWhenConditionSlice(conds []string) ([]types.WhenCondition, error) {
if len(conds) == 0 {
return nil, nil
}
wcs := []types.WhenCondition{}
for _, cond := range conds {
wc, err := parseWhenCondition(cond)
if err != nil {
return nil, err
}
wcs = append(wcs, *wc)
}
return wcs, nil
}
func parseWhenCondition(s string) (*types.WhenCondition, error) {
isRegExp := false
if len(s) > 2 {
for _, d := range regExpDelimiters {
if strings.HasPrefix(s, d) && strings.HasSuffix(s, d) {
isRegExp = true
s = s[1 : len(s)-1]
break
}
}
}
wc := &types.WhenCondition{Match: s}
if isRegExp {
if _, err := regexp.Compile(s); err != nil {
return nil, errors.Wrapf(err, "wrong regular expression")
}
wc.Type = types.WhenConditionTypeRegExp
} else {
wc.Type = types.WhenConditionTypeSimple
}
return wc, nil
}
func parseStringOrSlice(si interface{}) ([]string, error) {
ss := []string{}
switch c := si.(type) {
case string:
ss = []string{c}
case []interface{}:
var err error
ss, err = parseSliceString(c)
if err != nil {
return nil, err
}
}
return ss, nil
}
func parseSliceString(si []interface{}) ([]string, error) {
ss := []string{}
for _, v := range si {
switch s := v.(type) {
case string:
ss = append(ss, s)
default:
return nil, errors.Errorf("expected string")
}
}
return ss, nil
}
func (c *Config) Runtime(runtimeName string) *Runtime {
for n, r := range c.Runtimes {
if n == runtimeName {

125
internal/config/config_test.go
View File

@ -18,8 +18,11 @@ import (
"fmt"
"testing"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
)
func TestParseConfig(t *testing.T) {
@ -110,11 +113,127 @@ func TestParseConfig(t *testing.T) {
t.Fatalf("got error: %v, want error: %v", err, tt.err)
}
}
return
}
} else {
if tt.err != nil {
t.Fatalf("got nil error, want error: %v", tt.err)
}
}
})
}
}
func TestParseOutput(t *testing.T) {
tests := []struct {
name string
in string
out *Config
}{
{
name: "test element when conditions",
in: `
runtimes:
runtime01:
type: pod
containers:
- image: image01
tasks:
task01:
runtime: runtime01
environment:
ENV01: ENV01
pipelines:
pipeline01:
elements:
element01:
task: task01
when:
branch: master
tag:
- v1.x
- v2.x
ref:
include: master
exclude: [ /branch01/ , branch02 ]
`,
out: &Config{
Runtimes: map[string]*Runtime{
"runtime01": &Runtime{
Name: "runtime01",
Type: "pod",
Arch: "",
Containers: []*Container{
&Container{
Image: "image01",
Environment: nil,
User: "",
},
},
},
},
Tasks: map[string]*Task{
"task01": &Task{
Name: "task01",
Runtime: "runtime01",
Environment: map[string]string{
"ENV01": "ENV01",
},
WorkingDir: "",
Shell: "",
User: "",
Steps: []interface{}{},
},
},
Pipelines: map[string]*Pipeline{
"pipeline01": &Pipeline{
Name: "pipeline01",
Elements: map[string]*Element{
"element01": &Element{
Name: "element01",
Task: "task01",
Depends: []*Depend{},
IgnoreFailure: false,
Approval: false,
When: &types.When{
Branch: &types.WhenConditions{
Include: []types.WhenCondition{
{Type: types.WhenConditionTypeSimple, Match: "master"},
},
},
Tag: &types.WhenConditions{
Include: []types.WhenCondition{
{Type: types.WhenConditionTypeSimple, Match: "v1.x"},
{Type: types.WhenConditionTypeSimple, Match: "v2.x"},
},
},
Ref: &types.WhenConditions{
Include: []types.WhenCondition{
{Type: types.WhenConditionTypeSimple, Match: "master"},
},
Exclude: []types.WhenCondition{
{Type: types.WhenConditionTypeRegExp, Match: "branch01"},
{Type: types.WhenConditionTypeSimple, Match: "branch02"},
},
},
},
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out, err := ParseConfig([]byte(tt.in))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if diff := cmp.Diff(tt.out, out); diff != "" {
t.Error(diff)
}
})
}
}

73
internal/runconfig/runconfig.go
View File

@ -20,25 +20,26 @@ import (
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/config"
"github.com/sorintlab/agola/internal/services/runservice/types"
rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
uuid "github.com/satori/go.uuid"
)
func genRuntime(c *config.Config, runtimeName string) *types.Runtime {
func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime {
ce := c.Runtime(runtimeName)
containers := []*types.Container{}
containers := []*rstypes.Container{}
for _, cc := range ce.Containers {
containers = append(containers, &types.Container{
containers = append(containers, &rstypes.Container{
Image: cc.Image,
Environment: cc.Environment,
User: cc.User,
})
}
return &types.Runtime{
Type: types.RuntimeType(ce.Type),
return &rstypes.Runtime{
Type: rstypes.RuntimeType(ce.Type),
Containers: containers,
}
}
@ -89,7 +90,7 @@ fi
return rs
case *config.RunStep:
rs := &types.RunStep{}
rs := &rstypes.RunStep{}
rs.Type = cs.Type
rs.Name = cs.Name
@ -101,14 +102,14 @@ fi
return rs
case *config.SaveToWorkspaceStep:
sws := &types.SaveToWorkspaceStep{}
sws := &rstypes.SaveToWorkspaceStep{}
sws.Type = cs.Type
sws.Name = cs.Name
sws.Contents = make([]types.SaveToWorkspaceContent, len(cs.Contents))
sws.Contents = make([]rstypes.SaveToWorkspaceContent, len(cs.Contents))
for i, csc := range cs.Contents {
sc := types.SaveToWorkspaceContent{}
sc := rstypes.SaveToWorkspaceContent{}
sc.SourceDir = csc.SourceDir
sc.DestDir = csc.DestDir
sc.Paths = csc.Paths
@ -118,7 +119,7 @@ fi
return sws
case *config.RestoreWorkspaceStep:
rws := &types.RestoreWorkspaceStep{}
rws := &rstypes.RestoreWorkspaceStep{}
rws.Name = cs.Name
rws.Type = cs.Type
rws.DestDir = cs.DestDir
@ -132,32 +133,27 @@ fi
// GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfig(c *config.Config, pipelineName string, env map[string]string) *types.RunConfig {
func GenRunConfig(c *config.Config, pipelineName string, env map[string]string, branch, tag, ref string) *rstypes.RunConfig {
cp := c.Pipeline(pipelineName)
rc := &types.RunConfig{
rc := &rstypes.RunConfig{
Name: cp.Name,
Tasks: make(map[string]*types.RunConfigTask),
Tasks: make(map[string]*rstypes.RunConfigTask),
Environment: env,
}
for _, cpe := range cp.Elements {
include := types.MatchWhen(cpe.When, branch, tag, ref)
// resolve referenced task
cpt := c.Task(cpe.Task)
//environment := map[string]string{}
//if ct.Environment != nil {
// environment = ct.Environment
//}
//mergeEnv(environment, rd.DynamicEnvironment)
//// StaticEnvironment variables ovverride every other environment variable
//mergeEnv(environment, rd.Environment)
steps := make([]interface{}, len(cpt.Steps))
for i, cpts := range cpt.Steps {
steps[i] = stepFromConfigStep(cpts)
}
t := &types.RunConfigTask{
t := &rstypes.RunConfigTask{
ID: uuid.NewV4().String(),
// use the element name from the config as the task name
Name: cpe.Name,
@ -168,6 +164,7 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string)
User: cpt.User,
Steps: steps,
IgnoreFailure: cpe.IgnoreFailure,
Skip: !include,
}
rc.Tasks[t.ID] = t
@ -177,27 +174,27 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string)
for _, rct := range rc.Tasks {
cpe := cp.Elements[rct.Name]
depends := make([]*types.RunConfigTaskDepend, len(cpe.Depends))
depends := make([]*rstypes.RunConfigTaskDepend, len(cpe.Depends))
for id, d := range cpe.Depends {
conditions := make([]types.RunConfigTaskDependCondition, len(d.Conditions))
conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions))
// when no conditions are defined default to on_success
if len(d.Conditions) == 0 {
conditions = append(conditions, types.RunConfigTaskDependConditionOnSuccess)
conditions = append(conditions, rstypes.RunConfigTaskDependConditionOnSuccess)
} else {
for ic, c := range d.Conditions {
var condition types.RunConfigTaskDependCondition
var condition rstypes.RunConfigTaskDependCondition
switch c {
case config.DependConditionOnSuccess:
condition = types.RunConfigTaskDependConditionOnSuccess
condition = rstypes.RunConfigTaskDependConditionOnSuccess
case config.DependConditionOnFailure:
condition = types.RunConfigTaskDependConditionOnFailure
condition = rstypes.RunConfigTaskDependConditionOnFailure
}
conditions[ic] = condition
}
}
drct := getRunConfigTaskByName(rc, d.ElementName)
depends[id] = &types.RunConfigTaskDepend{
depends[id] = &rstypes.RunConfigTaskDepend{
TaskID: drct.ID,
Conditions: conditions,
}
@ -209,7 +206,7 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string)
return rc
}
func getRunConfigTaskByName(rc *types.RunConfig, name string) *types.RunConfigTask {
func getRunConfigTaskByName(rc *rstypes.RunConfig, name string) *rstypes.RunConfigTask {
for _, rct := range rc.Tasks {
if rct.Name == name {
return rct
@ -218,7 +215,7 @@ func getRunConfigTaskByName(rc *types.RunConfig, name string) *types.RunConfigTa
return nil
}
func CheckRunConfig(rc *types.RunConfig) error {
func CheckRunConfig(rc *rstypes.RunConfig) error {
// check circular dependencies
cerrs := &util.Errors{}
for _, t := range rc.Tasks {
@ -262,7 +259,7 @@ func CheckRunConfig(rc *types.RunConfig) error {
return nil
}
func GenTasksLevels(rc *types.RunConfig) error {
func GenTasksLevels(rc *rstypes.RunConfig) error {
// reset all task level
for _, t := range rc.Tasks {
t.Level = -1
@ -308,8 +305,8 @@ func GenTasksLevels(rc *types.RunConfig) error {
}
// GetParents returns direct parents of task.
func GetParents(rc *types.RunConfig, task *types.RunConfigTask) []*types.RunConfigTask {
parents := []*types.RunConfigTask{}
func GetParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
parents := []*rstypes.RunConfigTask{}
for _, t := range rc.Tasks {
isParent := false
for _, d := range task.Depends {
@ -327,13 +324,13 @@ func GetParents(rc *types.RunConfig, task *types.RunConfigTask) []*types.RunConf
// GetAllParents returns all the parents (both direct and ancestors) of task.
// In case of circular dependency it won't loop forever but will also return
// task as parent of itself
func GetAllParents(rc *types.RunConfig, task *types.RunConfigTask) []*types.RunConfigTask {
pMap := map[string]*types.RunConfigTask{}
func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstypes.RunConfigTask {
pMap := map[string]*rstypes.RunConfigTask{}
nextParents := GetParents(rc, task)
for len(nextParents) > 0 {
parents := nextParents
nextParents = []*types.RunConfigTask{}
nextParents = []*rstypes.RunConfigTask{}
for _, parent := range parents {
if _, ok := pMap[parent.ID]; ok {
continue
@ -343,7 +340,7 @@ func GetAllParents(rc *types.RunConfig, task *types.RunConfigTask) []*types.RunC
}
}
parents := make([]*types.RunConfigTask, 0, len(pMap))
parents := make([]*rstypes.RunConfigTask, 0, len(pMap))
for _, v := range pMap {
parents = append(parents, v)
}

6
internal/services/gateway/webhook.go
View File

@ -251,7 +251,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
group = genGroup(userID, webhookData)
}
if err := h.createRuns(ctx, data, group, annotations, env); err != nil {
if err := h.createRuns(ctx, data, group, annotations, env, webhookData); err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")
}
//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
@ -261,7 +261,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
return 0, "", nil
}
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env map[string]string) error {
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env map[string]string, webhookData *types.WebhookData) error {
config, err := config.ParseConfig([]byte(configData))
if err != nil {
return err
@ -270,7 +270,7 @@ func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, gro
//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName)
for _, pipeline := range config.Pipelines {
rc := runconfig.GenRunConfig(config, pipeline.Name, env)
rc := runconfig.GenRunConfig(config, pipeline.Name, env, webhookData.Branch, webhookData.Tag, webhookData.Ref)
h.log.Debugf("rc: %s", util.Dump(rc))
h.log.Infof("group: %s", group)

4
internal/services/runservice/scheduler/command/command.go
View File

@ -287,9 +287,13 @@ func (s *CommandHandler) genRunTask(ctx context.Context, rct *types.RunConfigTas
rt := &types.RunTask{
ID: rct.ID,
Status: types.RunTaskStatusNotStarted,
Skip: rct.Skip,
Steps: make([]*types.RunTaskStep, len(rct.Steps)),
WorkspaceArchives: []int{},
}
if rt.Skip {
rt.Status = types.RunTaskStatusSkipped
}
for i := range rt.Steps {
s := &types.RunTaskStep{
Phase: types.ExecutorTaskPhaseNotStarted,

8
internal/services/runservice/scheduler/scheduler.go
View File

@ -95,6 +95,9 @@ func (s *Scheduler) advanceRunTasks(ctx context.Context, r *types.Run) error {
// get tasks that can be executed
for _, rt := range r.RunTasks {
log.Debugf("rt: %s", util.Dump(rt))
if rt.Skip {
continue
}
if rt.Status != types.RunTaskStatusNotStarted {
continue
}
@ -105,6 +108,9 @@ func (s *Scheduler) advanceRunTasks(ctx context.Context, r *types.Run) error {
for _, p := range parents {
rp := r.RunTasks[p.ID]
canRun = rp.Status.IsFinished() && rp.ArchivesFetchFinished()
if rp.Status == types.RunTaskStatusSkipped {
rt.Status = types.RunTaskStatusSkipped
}
}
if canRun {
@ -672,7 +678,9 @@ func (s *Scheduler) fetchLog(ctx context.Context, rt *types.RunTask, stepnum int
return err
}
if et == nil {
if !rt.Skip {
log.Errorf("executor task with id %q doesn't exist. This shouldn't happen. Skipping fetching", rt.ID)
}
return nil
}
executor, err := store.GetExecutor(ctx, s.e, et.Status.ExecutorID)

6
internal/services/runservice/types/types.go
View File

@ -134,6 +134,7 @@ type RunTaskStatus string
const (
RunTaskStatusNotStarted RunTaskStatus = "notstarted"
RunTaskStatusSkipped RunTaskStatus = "skipped"
RunTaskStatusCancelled RunTaskStatus = "cancelled"
RunTaskStatusRunning RunTaskStatus = "running"
RunTaskStatusStopped RunTaskStatus = "stopped"
@ -142,7 +143,7 @@ const (
)
func (s RunTaskStatus) IsFinished() bool {
return s == RunTaskStatusCancelled || s == RunTaskStatusStopped || s == RunTaskStatusSuccess || s == RunTaskStatusFailed
return s == RunTaskStatusCancelled || s == RunTaskStatusSkipped || s == RunTaskStatusStopped || s == RunTaskStatusSuccess || s == RunTaskStatusFailed
}
type RunTaskFetchPhase string
@ -162,6 +163,8 @@ type RunTask struct {
// there're no executor tasks scheduled
Status RunTaskStatus `json:"status,omitempty"`
Skip bool `json:"skip,omitempty"`
WaitingApproval bool `json:"waiting_approval,omitempty"`
Approved bool `json:"approved,omitempty"`
// ApprovalAnnotations stores data that the user can set on the approval. Useful
@ -252,6 +255,7 @@ type RunConfigTask struct {
Steps []interface{} `json:"steps,omitempty"`
IgnoreFailure bool `json:"ignore_failure,omitempty"`
NeedsApproval bool `json:"needs_approval,omitempty"`
Skip bool `json:"skip,omitempty"`
}
type RunConfigTaskDependCondition string

80
internal/services/types/types.go
View File

@ -15,6 +15,7 @@
package types
import (
"regexp"
"time"
)
@ -136,3 +137,82 @@ type Project struct {
SkipSSHHostKeyCheck bool `json:"skip_ssh_host_key_check,omitempty"`
}
type When struct {
Branch *WhenConditions `json:"branch,omitempty"`
Tag *WhenConditions `json:"tag,omitempty"`
Ref *WhenConditions `json:"ref,omitempty"`
}
type WhenConditions struct {
Include []WhenCondition `json:"include,omitempty"`
Exclude []WhenCondition `json:"exclude,omitempty"`
}
type WhenConditionType string
const (
WhenConditionTypeSimple WhenConditionType = "simple"
WhenConditionTypeRegExp WhenConditionType = "regexp"
)
type WhenCondition struct {
Type WhenConditionType
Match string
}
func MatchWhen(when *When, branch, tag, ref string) bool {
include := true
if when != nil {
include = false
if when.Branch != nil {
// first check includes and override with excludes
if matchCondition(when.Branch.Include, branch) {
include = true
}
if matchCondition(when.Branch.Exclude, branch) {
include = false
}
}
if when.Tag != nil {
// first check includes and override with excludes
if matchCondition(when.Tag.Include, tag) {
include = true
}
if matchCondition(when.Tag.Exclude, tag) {
include = false
}
}
if when.Ref != nil {
// first check includes and override with excludes
if matchCondition(when.Ref.Include, ref) {
include = true
}
if matchCondition(when.Ref.Exclude, ref) {
include = false
}
}
}
return include
}
func matchCondition(conds []WhenCondition, s string) bool {
for _, cond := range conds {
switch cond.Type {
case WhenConditionTypeSimple:
if cond.Match == s {
return true
}
case WhenConditionTypeRegExp:
re, err := regexp.Compile(cond.Match)
if err != nil {
panic(err)
}
if re.MatchString(s) {
return true
}
}
}
return false
}