config environment from variable

internal/config/config.go

@@ -46,7 +46,7 @@ type Config struct {
 type Task struct {
 	Name        string            `yaml:"name"`
 	Runtime     string            `yaml:"runtime"`
-	Environment map[string]string `yaml:"environment"`
+	Environment map[string]EnvVar `yaml:"environment,omitempty"`
 	WorkingDir  string            `yaml:"working_dir"`
 	Shell       string            `yaml:"shell"`
 	User        string            `yaml:"user"`
@@ -68,7 +68,7 @@ type Runtime struct {
 
 type Container struct {
 	Image       string            `yaml:"image,omitempty"`
-	Environment map[string]string `yaml:"environment,omitempty"`
+	Environment map[string]EnvVar `yaml:"environment,omitempty"`
 	User        string            `yaml:"user"`
 	Privileged  bool              `yaml:"privileged"`
 	Entrypoint  string            `yaml:"entrypoint"`
@@ -112,12 +112,24 @@ type CloneStep struct {
 type RunStep struct {
 	Step        `yaml:",inline"`
 	Command     string            `yaml:"command"`
-	Environment map[string]string `yaml:"environment,omitempty"`
+	Environment map[string]EnvVar `yaml:"environment,omitempty"`
 	WorkingDir  string            `yaml:"working_dir"`
 	Shell       string            `yaml:"shell"`
 	User        string            `yaml:"user"`
 }
 
+type EnvVarType int
+
+const (
+	EnvVarTypeString EnvVarType = iota
+	EnvVarTypeFromVariable
+)
+
+type EnvVar struct {
+	Type  EnvVarType
+	Value string
+}
+
 type SaveToWorkspaceContent struct {
 	SourceDir string   `yaml:"source_dir"`
 	DestDir   string   `yaml:"dest_dir"`
@@ -134,23 +146,92 @@ type RestoreWorkspaceStep struct {
 	DestDir string `yaml:"dest_dir"`
 }
 
-func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (s *RunStep) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	type task Task
+	type runStep struct {
-	type tasksteps struct {
+		Step        `yaml:",inline"`
-		Steps []map[string]interface{} `yaml:"steps"`
+		Command     string                 `yaml:"command"`
-	}
+		Environment map[string]interface{} `yaml:"environment,omitempty"`
-	tt := (*task)(t)
+		WorkingDir  string                 `yaml:"working_dir"`
-	if err := unmarshal(&tt); err != nil {
+		Shell       string                 `yaml:"shell"`
-		return err
+		User        string                 `yaml:"user"`
 	}
 
-	var st tasksteps
+	var st *runStep
 	if err := unmarshal(&st); err != nil {
 		return err
 	}
 
+	s.Step = st.Step
+	s.Command = st.Command
+	s.WorkingDir = st.WorkingDir
+	s.Shell = st.Shell
+	s.User = st.User
+
+	if st.Environment != nil {
+		env, err := parseEnv(st.Environment)
+		if err != nil {
+			return err
+		}
+		s.Environment = env
+	}
+
+	return nil
+}
+
+func (c *Container) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type container struct {
+		Image       string                 `yaml:"image,omitempty"`
+		Environment map[string]interface{} `yaml:"environment,omitempty"`
+		User        string                 `yaml:"user"`
+		Privileged  bool                   `yaml:"privileged"`
+		Entrypoint  string                 `yaml:"entrypoint"`
+	}
+
+	var ct *container
+	if err := unmarshal(&ct); err != nil {
+		return err
+	}
+
+	c.Image = ct.Image
+	c.User = ct.User
+	c.Privileged = ct.Privileged
+	c.Entrypoint = ct.Entrypoint
+
+	if ct.Environment != nil {
+		env, err := parseEnv(ct.Environment)
+		if err != nil {
+			return err
+		}
+		c.Environment = env
+	}
+
+	return nil
+}
+
+func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type task struct {
+		Name        string                   `yaml:"name"`
+		Runtime     string                   `yaml:"runtime"`
+		Environment map[string]interface{}   `yaml:"environment,omitempty"`
+		WorkingDir  string                   `yaml:"working_dir"`
+		Shell       string                   `yaml:"shell"`
+		User        string                   `yaml:"user"`
+		Steps       []map[string]interface{} `yaml:"steps"`
+	}
+
+	var tt *task
+	if err := unmarshal(&tt); err != nil {
+		return err
+	}
+
+	t.Name = tt.Name
+	t.Runtime = tt.Runtime
+	t.WorkingDir = tt.WorkingDir
+	t.Shell = tt.Shell
+	t.User = tt.User
+
 	steps := make([]interface{}, len(tt.Steps))
-	for i, stepEntry := range st.Steps {
+	for i, stepEntry := range tt.Steps {
 		if len(stepEntry) > 1 {
 			return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
 		}
@@ -201,6 +282,14 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	t.Steps = steps
 
+	if tt.Environment != nil {
+		env, err := parseEnv(tt.Environment)
+		if err != nil {
+			return err
+		}
+		t.Environment = env
+	}
+
 	return nil
 }
 
@@ -297,6 +386,36 @@ func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+func parseEnv(ienv map[string]interface{}) (map[string]EnvVar, error) {
+	env := map[string]EnvVar{}
+	for envName, envEntry := range ienv {
+		switch envValue := envEntry.(type) {
+		case string:
+			env[envName] = EnvVar{
+				Type:  EnvVarTypeString,
+				Value: envValue,
+			}
+		case map[interface{}]interface{}:
+			for k, v := range envValue {
+				if k == "from_variable" {
+					switch v.(type) {
+					case string:
+					default:
+						return nil, errors.Errorf("unknown environment value: %v", v)
+					}
+					env[envName] = EnvVar{
+						Type:  EnvVarTypeFromVariable,
+						Value: v.(string),
+					}
+				}
+			}
+		default:
+			return nil, errors.Errorf("unknown environment value: %v", envValue)
+		}
+	}
+	return env, nil
+}
+
 func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
 	w := &types.WhenConditions{}
 

internal/config/config_test.go

@@ -136,12 +136,29 @@ func TestParseOutput(t *testing.T) {
                     type: pod
                     containers:
                       - image: image01
+                        environment:
+                          ENV01: ENV01
+                          ENVFROMVARIABLE01:
+                            from_variable: variable01
 
                 tasks:
                   task01:
                     runtime: runtime01
                     environment:
                       ENV01: ENV01
+                      ENVFROMVARIABLE01:
+                        from_variable: variable01
+                    steps:
+                      - run: command01
+                      - run:
+                          name: name different than command
+                          command: command02
+                      - run:
+                          command: command03
+                          environment:
+                            ENV01: ENV01
+                            ENVFROMVARIABLE01:
+                              from_variable: variable01
 
                 pipelines:
                   pipeline01:
@@ -165,9 +182,12 @@ func TestParseOutput(t *testing.T) {
 						Arch: "",
 						Containers: []*Container{
 							&Container{
-								Image:       "image01",
+								Image: "image01",
-								Environment: nil,
+								Environment: map[string]EnvVar{
-								User:        "",
+									"ENV01":             EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
+								},
+								User: "",
 							},
 						},
 					},
@@ -176,13 +196,40 @@ func TestParseOutput(t *testing.T) {
 					"task01": &Task{
 						Name:    "task01",
 						Runtime: "runtime01",
-						Environment: map[string]string{
+						Environment: map[string]EnvVar{
-							"ENV01": "ENV01",
+							"ENV01":             EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
+							"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
 						},
 						WorkingDir: "",
 						Shell:      "",
 						User:       "",
-						Steps:      []interface{}{},
+						Steps: []interface{}{
+							&RunStep{
+								Step: Step{
+									Type: "run",
+									Name: "command01",
+								},
+								Command: "command01",
+							},
+							&RunStep{
+								Step: Step{
+									Type: "run",
+									Name: "name different than command",
+								},
+								Command: "command02",
+							},
+							&RunStep{
+								Step: Step{
+									Type: "run",
+									Name: "command03",
+								},
+								Command: "command03",
+								Environment: map[string]EnvVar{
+									"ENV01":             EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
+								},
+							},
+						},
 					},
 				},
 				Pipelines: map[string]*Pipeline{

internal/runconfig/runconfig.go

@@ -23,22 +23,26 @@ import (
 	rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
 	"github.com/sorintlab/agola/internal/services/types"
 	"github.com/sorintlab/agola/internal/util"
-
-	uuid "github.com/satori/go.uuid"
 )
 
-func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime {
+func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime {
 	ce := c.Runtime(runtimeName)
 
 	containers := []*rstypes.Container{}
 	for _, cc := range ce.Containers {
-		containers = append(containers, &rstypes.Container{
+
+		env, err := genEnv(cc.Environment, variables)
+		if err != nil {
+			return nil
+		}
+		container := &rstypes.Container{
 			Image:       cc.Image,
-			Environment: cc.Environment,
+			Environment: env,
 			User:        cc.User,
 			Privileged:  cc.Privileged,
 			Entrypoint:  cc.Entrypoint,
-		})
+		}
+		containers = append(containers, container)
 	}
 	return &rstypes.Runtime{
 		Type:       rstypes.RuntimeType(ce.Type),
@@ -46,7 +50,7 @@ func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime {
 	}
 }
 
-func stepFromConfigStep(csi interface{}) interface{} {
+func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} {
 	switch cs := csi.(type) {
 	case *config.CloneStep:
 		// transform a "clone" step in a "run" step command
@@ -94,10 +98,15 @@ fi
 	case *config.RunStep:
 		rs := &rstypes.RunStep{}
 
+		env, err := genEnv(cs.Environment, variables)
+		if err != nil {
+			return nil
+		}
+
 		rs.Type = cs.Type
 		rs.Name = cs.Name
 		rs.Command = cs.Command
-		rs.Environment = cs.Environment
+		rs.Environment = env
 		rs.WorkingDir = cs.WorkingDir
 		rs.Shell = cs.Shell
 		rs.User = cs.User
@@ -135,7 +144,7 @@ fi
 
 // GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks
 // this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
-func GenRunConfig(c *config.Config, pipelineName string, env map[string]string, branch, tag, ref string) *rstypes.RunConfig {
+func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string, env, variables map[string]string, branch, tag, ref string) *rstypes.RunConfig {
 	cp := c.Pipeline(pipelineName)
 
 	rc := &rstypes.RunConfig{
@@ -152,15 +161,20 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string,
 
 		steps := make([]interface{}, len(cpt.Steps))
 		for i, cpts := range cpt.Steps {
-			steps[i] = stepFromConfigStep(cpts)
+			steps[i] = stepFromConfigStep(cpts, variables)
+		}
+
+		tEnv, err := genEnv(cpt.Environment, variables)
+		if err != nil {
+			return nil
 		}
 
 		t := &rstypes.RunConfigTask{
-			ID: uuid.NewV4().String(),
+			ID: uuid.New(cpe.Name).String(),
 			// use the element name from the config as the task name
 			Name:          cpe.Name,
-			Runtime:       genRuntime(c, cpt.Runtime),
+			Runtime:       genRuntime(c, cpt.Runtime, variables),
-			Environment:   cpt.Environment,
+			Environment:   tEnv,
 			WorkingDir:    cpt.WorkingDir,
 			Shell:         cpt.Shell,
 			User:          cpt.User,
@@ -348,3 +362,16 @@ func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstype
 	}
 	return parents
 }
+
+func genEnv(cenv map[string]config.EnvVar, variables map[string]string) (map[string]string, error) {
+	env := map[string]string{}
+	for envName, envVar := range cenv {
+		switch envVar.Type {
+		case config.EnvVarTypeString:
+			env[envName] = envVar.Value
+		case config.EnvVarTypeFromVariable:
+			env[envName] = variables[envVar.Value]
+		}
+	}
+	return env, nil
+}

internal/runconfig/runconfig_test.go

@@ -19,16 +19,21 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
-	"github.com/sorintlab/agola/internal/services/runservice/types"
+	"github.com/sorintlab/agola/internal/config"
+	rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
+	"github.com/sorintlab/agola/internal/services/types"
 	"github.com/sorintlab/agola/internal/util"
 )
 
+var uuid = &util.TestUUIDGenerator{}
+
 func TestGenTasksLevels(t *testing.T) {
 	type task struct {
 		ID      string
 		Level   int
-		Depends []*types.RunConfigTaskDepend
+		Depends []*rstypes.RunConfigTaskDepend
 	}
 	tests := []struct {
 		name string
@@ -84,8 +89,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -99,8 +104,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "2",
 					Level: 1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -113,8 +118,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -122,8 +127,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -137,8 +142,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -146,8 +151,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -155,8 +160,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -170,8 +175,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -179,8 +184,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -188,8 +193,8 @@ func TestGenTasksLevels(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -200,18 +205,18 @@ func TestGenTasksLevels(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
+			inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
 			for _, t := range tt.in {
-				inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
+				inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
 					ID:      t.ID,
 					Level:   t.Level,
 					Depends: t.Depends,
 				}
 
 			}
-			outRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
+			outRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
 			for _, t := range tt.out {
-				outRunConfig.Tasks[t.ID] = &types.RunConfigTask{
+				outRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
 					ID:      t.ID,
 					Level:   t.Level,
 					Depends: t.Depends,
@@ -238,7 +243,7 @@ func TestGetAllParents(t *testing.T) {
 	type task struct {
 		ID      string
 		Level   int
-		Depends []*types.RunConfigTaskDepend
+		Depends []*rstypes.RunConfigTaskDepend
 	}
 	tests := []struct {
 		name string
@@ -280,8 +285,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -301,8 +306,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -319,11 +324,11 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -331,8 +336,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "4",
 						},
 					},
@@ -340,8 +345,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "5",
 						},
 					},
@@ -369,8 +374,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -378,8 +383,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -396,8 +401,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -405,8 +410,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -414,8 +419,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -433,8 +438,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -442,8 +447,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -451,8 +456,8 @@ func TestGetAllParents(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -468,9 +473,9 @@ func TestGetAllParents(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
+			inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
 			for _, t := range tt.in {
-				inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
+				inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
 					ID:      t.ID,
 					Level:   t.Level,
 					Depends: t.Depends,
@@ -497,7 +502,7 @@ func TestCheckRunConfig(t *testing.T) {
 	type task struct {
 		ID      string
 		Level   int
-		Depends []*types.RunConfigTaskDepend
+		Depends []*rstypes.RunConfigTaskDepend
 	}
 	tests := []struct {
 		name string
@@ -536,8 +541,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -550,8 +555,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -559,8 +564,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -579,8 +584,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -588,8 +593,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -597,8 +602,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "1",
 						},
 					},
@@ -618,8 +623,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "1",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -627,8 +632,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "2",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "3",
 						},
 					},
@@ -636,8 +641,8 @@ func TestCheckRunConfig(t *testing.T) {
 				{
 					ID:    "3",
 					Level: -1,
-					Depends: []*types.RunConfigTaskDepend{
+					Depends: []*rstypes.RunConfigTaskDepend{
-						&types.RunConfigTaskDepend{
+						&rstypes.RunConfigTaskDepend{
 							TaskID: "2",
 						},
 					},
@@ -653,9 +658,9 @@ func TestCheckRunConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
+			inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
 			for _, t := range tt.in {
-				inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
+				inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
 					Name:    fmt.Sprintf("task%s", t.ID),
 					ID:      t.ID,
 					Level:   t.Level,
@@ -682,3 +687,150 @@ func TestCheckRunConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestGenRunConfig(t *testing.T) {
+	tests := []struct {
+		name      string
+		in        *config.Config
+		env       map[string]string
+		variables map[string]string
+		out       *rstypes.RunConfig
+	}{
+		{
+			name: "test runconfig generation",
+			in: &config.Config{
+				Runtimes: map[string]*config.Runtime{
+					"runtime01": &config.Runtime{
+						Name: "runtime01",
+						Type: "pod",
+						Arch: "",
+						Containers: []*config.Container{
+							&config.Container{
+								Image: "image01",
+								Environment: map[string]config.EnvVar{
+									"ENV01":             config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
+								},
+								User: "",
+							},
+						},
+					},
+				},
+				Tasks: map[string]*config.Task{
+					"task01": &config.Task{
+						Name:    "task01",
+						Runtime: "runtime01",
+						Environment: map[string]config.EnvVar{
+							"ENV01":             config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
+							"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
+						},
+						WorkingDir: "",
+						Shell:      "",
+						User:       "",
+						Steps: []interface{}{
+							&config.RunStep{
+								Step: config.Step{
+									Type: "run",
+									Name: "command01",
+								},
+								Command: "command01",
+							},
+							&config.RunStep{
+								Step: config.Step{
+									Type: "run",
+									Name: "name different than command",
+								},
+								Command: "command02",
+							},
+							&config.RunStep{
+								Step: config.Step{
+									Type: "run",
+									Name: "command03",
+								},
+								Command: "command03",
+								Environment: map[string]config.EnvVar{
+									"ENV01":             config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
+									"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
+								},
+							},
+						},
+					},
+				},
+				Pipelines: map[string]*config.Pipeline{
+					"pipeline01": &config.Pipeline{
+						Name: "pipeline01",
+						Elements: map[string]*config.Element{
+							"element01": &config.Element{
+								Name:          "element01",
+								Task:          "task01",
+								Depends:       []*config.Depend{},
+								IgnoreFailure: false,
+								Approval:      false,
+								When: &types.When{
+									Branch: &types.WhenConditions{Include: []types.WhenCondition{{Match: "master"}}},
+									Tag:    &types.WhenConditions{Include: []types.WhenCondition{{Match: "v1.x"}, {Match: "v2.x"}}},
+									Ref: &types.WhenConditions{
+										Include: []types.WhenCondition{{Match: "master"}},
+										Exclude: []types.WhenCondition{{Match: "/branch01/", Type: types.WhenConditionTypeRegExp}, {Match: "branch02"}},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			env: map[string]string{
+				"ENV01": "ENVVALUE01",
+			},
+			variables: map[string]string{
+				"variable01": "VARVALUE01",
+			},
+			out: &rstypes.RunConfig{
+				Name: "pipeline01",
+				Environment: map[string]string{
+					"ENV01": "ENVVALUE01",
+				},
+				Tasks: map[string]*rstypes.RunConfigTask{
+					uuid.New("element01").String(): &rstypes.RunConfigTask{
+						ID:   uuid.New("element01").String(),
+						Name: "element01", Depends: []*rstypes.RunConfigTaskDepend{},
+						Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
+							Containers: []*rstypes.Container{
+								{
+									Image: "image01",
+									Environment: map[string]string{
+										"ENV01":             "ENV01",
+										"ENVFROMVARIABLE01": "VARVALUE01",
+									},
+								},
+							},
+						},
+						Environment: map[string]string{
+							"ENV01":             "ENV01",
+							"ENVFROMVARIABLE01": "VARVALUE01",
+						},
+						Steps: []interface{}{
+							&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command01"}, Command: "command01", Environment: map[string]string{}},
+							&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "name different than command"}, Command: "command02", Environment: map[string]string{}},
+							&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command03"}, Command: "command03", Environment: map[string]string{"ENV01": "ENV01", "ENVFROMVARIABLE01": "VARVALUE01"}},
+						},
+						Skip: true,
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out := GenRunConfig(uuid, tt.in, "pipeline01", tt.env, tt.variables, "", "", "")
+
+			//if err != nil {
+			//	t.Fatalf("unexpected error: %v", err)
+			//}
+			if diff := cmp.Diff(tt.out, out); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}

internal/services/gateway/webhook.go

@@ -121,6 +121,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	var skipSSHHostKeyCheck bool
 	var runType types.RunType
 	var userID string
+	variables := map[string]string{}
 
 	var gitSource gitsource.GitSource
 	if !isUserBuild {
@@ -128,7 +129,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		if err != nil {
 			return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
 		}
-		h.log.Debugf("project: %s", util.Dump(project))
+		h.log.Infof("project: %s", util.Dump(project))
 
 		user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
 		if err != nil {
@@ -159,6 +160,46 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		}
 		webhookData.ProjectID = projectID
 
+		// get project variables
+		pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, project.ID, true)
+		if err != nil {
+			return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project variables")
+		}
+		h.log.Infof("pvars: %v", util.Dump(pvars))
+
+		// remove overriden variables
+		pvars = common.FilterOverridenVariables(pvars)
+		h.log.Infof("pvars: %v", util.Dump(pvars))
+
+		// get project secrets
+		secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, project.ID, true)
+		if err != nil {
+			return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project secrets")
+		}
+		h.log.Infof("secrets: %v", util.Dump(secrets))
+		for _, pvar := range pvars {
+			// find the value match
+			var varval types.VariableValue
+			for _, varval = range pvar.Values {
+				h.log.Infof("varval: %v", util.Dump(varval))
+				match := types.MatchWhen(varval.When, webhookData.Branch, webhookData.Tag, webhookData.Ref)
+				if !match {
+					continue
+				}
+				// get the secret value referenced by the variable, it must be a secret at the same level or a lower level
+				secret := common.GetVarValueMatchingSecret(varval, pvar.Parent.Path, secrets)
+				h.log.Infof("secret: %v", util.Dump(secret))
+				if secret != nil {
+					varValue, ok := secret.Data[varval.SecretVar]
+					if ok {
+						variables[pvar.Name] = varValue
+					}
+				}
+				break
+			}
+		}
+		h.log.Infof("variables: %v", util.Dump(variables))
+
 	} else {
 		gitSource = agolagit.New(h.apiExposedURL + "/repos")
 		var err error
@@ -200,6 +241,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to parse clone url")
 	}
 
+	// this env vars ovverrides other env vars
 	env := map[string]string{
 		"CI":                   "true",
 		"AGOLA_SSHPRIVKEY":     sshPrivKey,
@@ -251,7 +293,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		group = genGroup(userID, webhookData)
 	}
 
-	if err := h.createRuns(ctx, data, group, annotations, env, webhookData); err != nil {
+	if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil {
 		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")
 	}
 	//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
@@ -261,16 +303,16 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	return 0, "", nil
 }
 
-func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env map[string]string, webhookData *types.WebhookData) error {
+func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env, variables map[string]string, webhookData *types.WebhookData) error {
 	config, err := config.ParseConfig([]byte(configData))
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to parse config")
 	}
 	//h.log.Debugf("config: %v", util.Dump(config))
 
 	//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName)
 	for _, pipeline := range config.Pipelines {
-		rc := runconfig.GenRunConfig(config, pipeline.Name, env, webhookData.Branch, webhookData.Tag, webhookData.Ref)
+		rc := runconfig.GenRunConfig(util.DefaultUUIDGenerator{}, config, pipeline.Name, env, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
 
 		h.log.Debugf("rc: %s", util.Dump(rc))
 		h.log.Infof("group: %s", group)

internal/util/uuid.go

@@ -0,0 +1,35 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	uuid "github.com/satori/go.uuid"
+)
+
+type UUIDGenerator interface {
+	New(s string) uuid.UUID
+}
+
+type DefaultUUIDGenerator struct{}
+
+func (u DefaultUUIDGenerator) New(s string) uuid.UUID {
+	return uuid.NewV4()
+}
+
+type TestUUIDGenerator struct{}
+
+func (u TestUUIDGenerator) New(s string) uuid.UUID {
+	return uuid.NewV5(uuid.NamespaceDNS, s)
+}