From 61b454d4daae3dbd0d2aff23ad5ac5bb344b9fc5 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Mon, 18 Mar 2019 15:02:32 +0100 Subject: [PATCH] config environment from variable --- internal/config/config.go | 145 +++++++++++-- internal/config/config_test.go | 59 +++++- internal/runconfig/runconfig.go | 53 +++-- internal/runconfig/runconfig_test.go | 306 ++++++++++++++++++++------- internal/services/gateway/webhook.go | 52 ++++- internal/util/uuid.go | 35 +++ 6 files changed, 536 insertions(+), 114 deletions(-) create mode 100644 internal/util/uuid.go diff --git a/internal/config/config.go b/internal/config/config.go index deb7909..ac6b600 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,7 +46,7 @@ type Config struct { type Task struct { Name string `yaml:"name"` Runtime string `yaml:"runtime"` - Environment map[string]string `yaml:"environment"` + Environment map[string]EnvVar `yaml:"environment,omitempty"` WorkingDir string `yaml:"working_dir"` Shell string `yaml:"shell"` User string `yaml:"user"` @@ -68,7 +68,7 @@ type Runtime struct { type Container struct { Image string `yaml:"image,omitempty"` - Environment map[string]string `yaml:"environment,omitempty"` + Environment map[string]EnvVar `yaml:"environment,omitempty"` User string `yaml:"user"` Privileged bool `yaml:"privileged"` Entrypoint string `yaml:"entrypoint"` @@ -112,12 +112,24 @@ type CloneStep struct { type RunStep struct { Step `yaml:",inline"` Command string `yaml:"command"` - Environment map[string]string `yaml:"environment,omitempty"` + Environment map[string]EnvVar `yaml:"environment,omitempty"` WorkingDir string `yaml:"working_dir"` Shell string `yaml:"shell"` User string `yaml:"user"` } +type EnvVarType int + +const ( + EnvVarTypeString EnvVarType = iota + EnvVarTypeFromVariable +) + +type EnvVar struct { + Type EnvVarType + Value string +} + type SaveToWorkspaceContent struct { SourceDir string `yaml:"source_dir"` DestDir string `yaml:"dest_dir"` @@ -134,23 +146,92 @@ type RestoreWorkspaceStep struct { DestDir string `yaml:"dest_dir"` } -func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { - type task Task - type tasksteps struct { - Steps []map[string]interface{} `yaml:"steps"` - } - tt := (*task)(t) - if err := unmarshal(&tt); err != nil { - return err +func (s *RunStep) UnmarshalYAML(unmarshal func(interface{}) error) error { + type runStep struct { + Step `yaml:",inline"` + Command string `yaml:"command"` + Environment map[string]interface{} `yaml:"environment,omitempty"` + WorkingDir string `yaml:"working_dir"` + Shell string `yaml:"shell"` + User string `yaml:"user"` } - var st tasksteps + var st *runStep if err := unmarshal(&st); err != nil { return err } + s.Step = st.Step + s.Command = st.Command + s.WorkingDir = st.WorkingDir + s.Shell = st.Shell + s.User = st.User + + if st.Environment != nil { + env, err := parseEnv(st.Environment) + if err != nil { + return err + } + s.Environment = env + } + + return nil +} + +func (c *Container) UnmarshalYAML(unmarshal func(interface{}) error) error { + type container struct { + Image string `yaml:"image,omitempty"` + Environment map[string]interface{} `yaml:"environment,omitempty"` + User string `yaml:"user"` + Privileged bool `yaml:"privileged"` + Entrypoint string `yaml:"entrypoint"` + } + + var ct *container + if err := unmarshal(&ct); err != nil { + return err + } + + c.Image = ct.Image + c.User = ct.User + c.Privileged = ct.Privileged + c.Entrypoint = ct.Entrypoint + + if ct.Environment != nil { + env, err := parseEnv(ct.Environment) + if err != nil { + return err + } + c.Environment = env + } + + return nil +} + +func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { + type task struct { + Name string `yaml:"name"` + Runtime string `yaml:"runtime"` + Environment map[string]interface{} `yaml:"environment,omitempty"` + WorkingDir string `yaml:"working_dir"` + Shell string `yaml:"shell"` + User string `yaml:"user"` + Steps []map[string]interface{} `yaml:"steps"` + } + + var tt *task + if err := unmarshal(&tt); err != nil { + return err + } + + t.Name = tt.Name + t.Runtime = tt.Runtime + t.WorkingDir = tt.WorkingDir + t.Shell = tt.Shell + t.User = tt.User + steps := make([]interface{}, len(tt.Steps)) - for i, stepEntry := range st.Steps { + for i, stepEntry := range tt.Steps { if len(stepEntry) > 1 { return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i) } @@ -201,6 +282,14 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { t.Steps = steps + if tt.Environment != nil { + env, err := parseEnv(tt.Environment) + if err != nil { + return err + } + t.Environment = env + } + return nil } @@ -297,6 +386,36 @@ func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +func parseEnv(ienv map[string]interface{}) (map[string]EnvVar, error) { + env := map[string]EnvVar{} + for envName, envEntry := range ienv { + switch envValue := envEntry.(type) { + case string: + env[envName] = EnvVar{ + Type: EnvVarTypeString, + Value: envValue, + } + case map[interface{}]interface{}: + for k, v := range envValue { + if k == "from_variable" { + switch v.(type) { + case string: + default: + return nil, errors.Errorf("unknown environment value: %v", v) + } + env[envName] = EnvVar{ + Type: EnvVarTypeFromVariable, + Value: v.(string), + } + } + } + default: + return nil, errors.Errorf("unknown environment value: %v", envValue) + } + } + return env, nil +} + func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) { w := &types.WhenConditions{} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3886764..89558ff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -136,12 +136,29 @@ func TestParseOutput(t *testing.T) { type: pod containers: - image: image01 + environment: + ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 tasks: task01: runtime: runtime01 environment: ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 + steps: + - run: command01 + - run: + name: name different than command + command: command02 + - run: + command: command03 + environment: + ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 pipelines: pipeline01: @@ -165,9 +182,12 @@ func TestParseOutput(t *testing.T) { Arch: "", Containers: []*Container{ &Container{ - Image: "image01", - Environment: nil, - User: "", + Image: "image01", + Environment: map[string]EnvVar{ + "ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"}, + }, + User: "", }, }, }, @@ -176,13 +196,40 @@ func TestParseOutput(t *testing.T) { "task01": &Task{ Name: "task01", Runtime: "runtime01", - Environment: map[string]string{ - "ENV01": "ENV01", + Environment: map[string]EnvVar{ + "ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"}, }, WorkingDir: "", Shell: "", User: "", - Steps: []interface{}{}, + Steps: []interface{}{ + &RunStep{ + Step: Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "name different than command", + }, + Command: "command02", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "command03", + }, + Command: "command03", + Environment: map[string]EnvVar{ + "ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"}, + }, + }, + }, }, }, Pipelines: map[string]*Pipeline{ diff --git a/internal/runconfig/runconfig.go b/internal/runconfig/runconfig.go index 1fa7ea6..7624bb3 100644 --- a/internal/runconfig/runconfig.go +++ b/internal/runconfig/runconfig.go @@ -23,22 +23,26 @@ import ( rstypes "github.com/sorintlab/agola/internal/services/runservice/types" "github.com/sorintlab/agola/internal/services/types" "github.com/sorintlab/agola/internal/util" - - uuid "github.com/satori/go.uuid" ) -func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime { +func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime { ce := c.Runtime(runtimeName) containers := []*rstypes.Container{} for _, cc := range ce.Containers { - containers = append(containers, &rstypes.Container{ + + env, err := genEnv(cc.Environment, variables) + if err != nil { + return nil + } + container := &rstypes.Container{ Image: cc.Image, - Environment: cc.Environment, + Environment: env, User: cc.User, Privileged: cc.Privileged, Entrypoint: cc.Entrypoint, - }) + } + containers = append(containers, container) } return &rstypes.Runtime{ Type: rstypes.RuntimeType(ce.Type), @@ -46,7 +50,7 @@ func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime { } } -func stepFromConfigStep(csi interface{}) interface{} { +func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} { switch cs := csi.(type) { case *config.CloneStep: // transform a "clone" step in a "run" step command @@ -94,10 +98,15 @@ fi case *config.RunStep: rs := &rstypes.RunStep{} + env, err := genEnv(cs.Environment, variables) + if err != nil { + return nil + } + rs.Type = cs.Type rs.Name = cs.Name rs.Command = cs.Command - rs.Environment = cs.Environment + rs.Environment = env rs.WorkingDir = cs.WorkingDir rs.Shell = cs.Shell rs.User = cs.User @@ -135,7 +144,7 @@ fi // GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks // this functions assumes that the config is already checked for possible errors (i.e referenced task must exits) -func GenRunConfig(c *config.Config, pipelineName string, env map[string]string, branch, tag, ref string) *rstypes.RunConfig { +func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string, env, variables map[string]string, branch, tag, ref string) *rstypes.RunConfig { cp := c.Pipeline(pipelineName) rc := &rstypes.RunConfig{ @@ -152,15 +161,20 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string, steps := make([]interface{}, len(cpt.Steps)) for i, cpts := range cpt.Steps { - steps[i] = stepFromConfigStep(cpts) + steps[i] = stepFromConfigStep(cpts, variables) + } + + tEnv, err := genEnv(cpt.Environment, variables) + if err != nil { + return nil } t := &rstypes.RunConfigTask{ - ID: uuid.NewV4().String(), + ID: uuid.New(cpe.Name).String(), // use the element name from the config as the task name Name: cpe.Name, - Runtime: genRuntime(c, cpt.Runtime), - Environment: cpt.Environment, + Runtime: genRuntime(c, cpt.Runtime, variables), + Environment: tEnv, WorkingDir: cpt.WorkingDir, Shell: cpt.Shell, User: cpt.User, @@ -348,3 +362,16 @@ func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstype } return parents } + +func genEnv(cenv map[string]config.EnvVar, variables map[string]string) (map[string]string, error) { + env := map[string]string{} + for envName, envVar := range cenv { + switch envVar.Type { + case config.EnvVarTypeString: + env[envName] = envVar.Value + case config.EnvVarTypeFromVariable: + env[envName] = variables[envVar.Value] + } + } + return env, nil +} diff --git a/internal/runconfig/runconfig_test.go b/internal/runconfig/runconfig_test.go index 28659e4..0018538 100644 --- a/internal/runconfig/runconfig_test.go +++ b/internal/runconfig/runconfig_test.go @@ -19,16 +19,21 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" "github.com/pkg/errors" - "github.com/sorintlab/agola/internal/services/runservice/types" + "github.com/sorintlab/agola/internal/config" + rstypes "github.com/sorintlab/agola/internal/services/runservice/types" + "github.com/sorintlab/agola/internal/services/types" "github.com/sorintlab/agola/internal/util" ) +var uuid = &util.TestUUIDGenerator{} + func TestGenTasksLevels(t *testing.T) { type task struct { ID string Level int - Depends []*types.RunConfigTaskDepend + Depends []*rstypes.RunConfigTaskDepend } tests := []struct { name string @@ -84,8 +89,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -99,8 +104,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "2", Level: 1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -113,8 +118,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -122,8 +127,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -137,8 +142,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -146,8 +151,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -155,8 +160,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -170,8 +175,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -179,8 +184,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -188,8 +193,8 @@ func TestGenTasksLevels(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -200,18 +205,18 @@ func TestGenTasksLevels(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}} + inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} for _, t := range tt.in { - inRunConfig.Tasks[t.ID] = &types.RunConfigTask{ + inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ ID: t.ID, Level: t.Level, Depends: t.Depends, } } - outRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}} + outRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} for _, t := range tt.out { - outRunConfig.Tasks[t.ID] = &types.RunConfigTask{ + outRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ ID: t.ID, Level: t.Level, Depends: t.Depends, @@ -238,7 +243,7 @@ func TestGetAllParents(t *testing.T) { type task struct { ID string Level int - Depends []*types.RunConfigTaskDepend + Depends []*rstypes.RunConfigTaskDepend } tests := []struct { name string @@ -280,8 +285,8 @@ func TestGetAllParents(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -301,8 +306,8 @@ func TestGetAllParents(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -319,11 +324,11 @@ func TestGetAllParents(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, - &types.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -331,8 +336,8 @@ func TestGetAllParents(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "4", }, }, @@ -340,8 +345,8 @@ func TestGetAllParents(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "5", }, }, @@ -369,8 +374,8 @@ func TestGetAllParents(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -378,8 +383,8 @@ func TestGetAllParents(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -396,8 +401,8 @@ func TestGetAllParents(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -405,8 +410,8 @@ func TestGetAllParents(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -414,8 +419,8 @@ func TestGetAllParents(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -433,8 +438,8 @@ func TestGetAllParents(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -442,8 +447,8 @@ func TestGetAllParents(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -451,8 +456,8 @@ func TestGetAllParents(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -468,9 +473,9 @@ func TestGetAllParents(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}} + inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} for _, t := range tt.in { - inRunConfig.Tasks[t.ID] = &types.RunConfigTask{ + inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ ID: t.ID, Level: t.Level, Depends: t.Depends, @@ -497,7 +502,7 @@ func TestCheckRunConfig(t *testing.T) { type task struct { ID string Level int - Depends []*types.RunConfigTaskDepend + Depends []*rstypes.RunConfigTaskDepend } tests := []struct { name string @@ -536,8 +541,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -550,8 +555,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -559,8 +564,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -579,8 +584,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -588,8 +593,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -597,8 +602,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "1", }, }, @@ -618,8 +623,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "1", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -627,8 +632,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "2", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "3", }, }, @@ -636,8 +641,8 @@ func TestCheckRunConfig(t *testing.T) { { ID: "3", Level: -1, - Depends: []*types.RunConfigTaskDepend{ - &types.RunConfigTaskDepend{ + Depends: []*rstypes.RunConfigTaskDepend{ + &rstypes.RunConfigTaskDepend{ TaskID: "2", }, }, @@ -653,9 +658,9 @@ func TestCheckRunConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}} + inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}} for _, t := range tt.in { - inRunConfig.Tasks[t.ID] = &types.RunConfigTask{ + inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{ Name: fmt.Sprintf("task%s", t.ID), ID: t.ID, Level: t.Level, @@ -682,3 +687,150 @@ func TestCheckRunConfig(t *testing.T) { }) } } + +func TestGenRunConfig(t *testing.T) { + tests := []struct { + name string + in *config.Config + env map[string]string + variables map[string]string + out *rstypes.RunConfig + }{ + { + name: "test runconfig generation", + in: &config.Config{ + Runtimes: map[string]*config.Runtime{ + "runtime01": &config.Runtime{ + Name: "runtime01", + Type: "pod", + Arch: "", + Containers: []*config.Container{ + &config.Container{ + Image: "image01", + Environment: map[string]config.EnvVar{ + "ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"}, + }, + User: "", + }, + }, + }, + }, + Tasks: map[string]*config.Task{ + "task01": &config.Task{ + Name: "task01", + Runtime: "runtime01", + Environment: map[string]config.EnvVar{ + "ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"}, + }, + WorkingDir: "", + Shell: "", + User: "", + Steps: []interface{}{ + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "name different than command", + }, + Command: "command02", + }, + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command03", + }, + Command: "command03", + Environment: map[string]config.EnvVar{ + "ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"}, + }, + }, + }, + }, + }, + Pipelines: map[string]*config.Pipeline{ + "pipeline01": &config.Pipeline{ + Name: "pipeline01", + Elements: map[string]*config.Element{ + "element01": &config.Element{ + Name: "element01", + Task: "task01", + Depends: []*config.Depend{}, + IgnoreFailure: false, + Approval: false, + When: &types.When{ + Branch: &types.WhenConditions{Include: []types.WhenCondition{{Match: "master"}}}, + Tag: &types.WhenConditions{Include: []types.WhenCondition{{Match: "v1.x"}, {Match: "v2.x"}}}, + Ref: &types.WhenConditions{ + Include: []types.WhenCondition{{Match: "master"}}, + Exclude: []types.WhenCondition{{Match: "/branch01/", Type: types.WhenConditionTypeRegExp}, {Match: "branch02"}}, + }, + }, + }, + }, + }, + }, + }, + env: map[string]string{ + "ENV01": "ENVVALUE01", + }, + variables: map[string]string{ + "variable01": "VARVALUE01", + }, + out: &rstypes.RunConfig{ + Name: "pipeline01", + Environment: map[string]string{ + "ENV01": "ENVVALUE01", + }, + Tasks: map[string]*rstypes.RunConfigTask{ + uuid.New("element01").String(): &rstypes.RunConfigTask{ + ID: uuid.New("element01").String(), + Name: "element01", Depends: []*rstypes.RunConfigTaskDepend{}, + Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"), + Containers: []*rstypes.Container{ + { + Image: "image01", + Environment: map[string]string{ + "ENV01": "ENV01", + "ENVFROMVARIABLE01": "VARVALUE01", + }, + }, + }, + }, + Environment: map[string]string{ + "ENV01": "ENV01", + "ENVFROMVARIABLE01": "VARVALUE01", + }, + Steps: []interface{}{ + &rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command01"}, Command: "command01", Environment: map[string]string{}}, + &rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "name different than command"}, Command: "command02", Environment: map[string]string{}}, + &rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command03"}, Command: "command03", Environment: map[string]string{"ENV01": "ENV01", "ENVFROMVARIABLE01": "VARVALUE01"}}, + }, + Skip: true, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := GenRunConfig(uuid, tt.in, "pipeline01", tt.env, tt.variables, "", "", "") + + //if err != nil { + // t.Fatalf("unexpected error: %v", err) + //} + if diff := cmp.Diff(tt.out, out); diff != "" { + t.Error(diff) + } + }) + } +} diff --git a/internal/services/gateway/webhook.go b/internal/services/gateway/webhook.go index 6411e9f..5c4391d 100644 --- a/internal/services/gateway/webhook.go +++ b/internal/services/gateway/webhook.go @@ -121,6 +121,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { var skipSSHHostKeyCheck bool var runType types.RunType var userID string + variables := map[string]string{} var gitSource gitsource.GitSource if !isUserBuild { @@ -128,7 +129,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { if err != nil { return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID) } - h.log.Debugf("project: %s", util.Dump(project)) + h.log.Infof("project: %s", util.Dump(project)) user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID) if err != nil { @@ -159,6 +160,46 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { } webhookData.ProjectID = projectID + // get project variables + pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, project.ID, true) + if err != nil { + return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project variables") + } + h.log.Infof("pvars: %v", util.Dump(pvars)) + + // remove overriden variables + pvars = common.FilterOverridenVariables(pvars) + h.log.Infof("pvars: %v", util.Dump(pvars)) + + // get project secrets + secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, project.ID, true) + if err != nil { + return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project secrets") + } + h.log.Infof("secrets: %v", util.Dump(secrets)) + for _, pvar := range pvars { + // find the value match + var varval types.VariableValue + for _, varval = range pvar.Values { + h.log.Infof("varval: %v", util.Dump(varval)) + match := types.MatchWhen(varval.When, webhookData.Branch, webhookData.Tag, webhookData.Ref) + if !match { + continue + } + // get the secret value referenced by the variable, it must be a secret at the same level or a lower level + secret := common.GetVarValueMatchingSecret(varval, pvar.Parent.Path, secrets) + h.log.Infof("secret: %v", util.Dump(secret)) + if secret != nil { + varValue, ok := secret.Data[varval.SecretVar] + if ok { + variables[pvar.Name] = varValue + } + } + break + } + } + h.log.Infof("variables: %v", util.Dump(variables)) + } else { gitSource = agolagit.New(h.apiExposedURL + "/repos") var err error @@ -200,6 +241,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to parse clone url") } + // this env vars ovverrides other env vars env := map[string]string{ "CI": "true", "AGOLA_SSHPRIVKEY": sshPrivKey, @@ -251,7 +293,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { group = genGroup(userID, webhookData) } - if err := h.createRuns(ctx, data, group, annotations, env, webhookData); err != nil { + if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil { return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run") } //if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil { @@ -261,16 +303,16 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { return 0, "", nil } -func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env map[string]string, webhookData *types.WebhookData) error { +func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env, variables map[string]string, webhookData *types.WebhookData) error { config, err := config.ParseConfig([]byte(configData)) if err != nil { - return err + return errors.Wrapf(err, "failed to parse config") } //h.log.Debugf("config: %v", util.Dump(config)) //h.log.Debugf("pipeline: %s", createRunOpts.PipelineName) for _, pipeline := range config.Pipelines { - rc := runconfig.GenRunConfig(config, pipeline.Name, env, webhookData.Branch, webhookData.Tag, webhookData.Ref) + rc := runconfig.GenRunConfig(util.DefaultUUIDGenerator{}, config, pipeline.Name, env, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref) h.log.Debugf("rc: %s", util.Dump(rc)) h.log.Infof("group: %s", group) diff --git a/internal/util/uuid.go b/internal/util/uuid.go new file mode 100644 index 0000000..79c48a4 --- /dev/null +++ b/internal/util/uuid.go @@ -0,0 +1,35 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + uuid "github.com/satori/go.uuid" +) + +type UUIDGenerator interface { + New(s string) uuid.UUID +} + +type DefaultUUIDGenerator struct{} + +func (u DefaultUUIDGenerator) New(s string) uuid.UUID { + return uuid.NewV4() +} + +type TestUUIDGenerator struct{} + +func (u TestUUIDGenerator) New(s string) uuid.UUID { + return uuid.NewV5(uuid.NamespaceDNS, s) +}