config environment from variable

This commit is contained in:
Simone Gotti 2019-03-18 15:02:32 +01:00
parent 50547a6490
commit 61b454d4da
6 changed files with 536 additions and 114 deletions

View File

@ -46,7 +46,7 @@ type Config struct {
type Task struct {
Name string `yaml:"name"`
Runtime string `yaml:"runtime"`
Environment map[string]string `yaml:"environment"`
Environment map[string]EnvVar `yaml:"environment,omitempty"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
@ -68,7 +68,7 @@ type Runtime struct {
type Container struct {
Image string `yaml:"image,omitempty"`
Environment map[string]string `yaml:"environment,omitempty"`
Environment map[string]EnvVar `yaml:"environment,omitempty"`
User string `yaml:"user"`
Privileged bool `yaml:"privileged"`
Entrypoint string `yaml:"entrypoint"`
@ -112,12 +112,24 @@ type CloneStep struct {
type RunStep struct {
Step `yaml:",inline"`
Command string `yaml:"command"`
Environment map[string]string `yaml:"environment,omitempty"`
Environment map[string]EnvVar `yaml:"environment,omitempty"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
}
type EnvVarType int
const (
EnvVarTypeString EnvVarType = iota
EnvVarTypeFromVariable
)
type EnvVar struct {
Type EnvVarType
Value string
}
type SaveToWorkspaceContent struct {
SourceDir string `yaml:"source_dir"`
DestDir string `yaml:"dest_dir"`
@ -134,23 +146,92 @@ type RestoreWorkspaceStep struct {
DestDir string `yaml:"dest_dir"`
}
func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
type task Task
type tasksteps struct {
Steps []map[string]interface{} `yaml:"steps"`
}
tt := (*task)(t)
if err := unmarshal(&tt); err != nil {
return err
func (s *RunStep) UnmarshalYAML(unmarshal func(interface{}) error) error {
type runStep struct {
Step `yaml:",inline"`
Command string `yaml:"command"`
Environment map[string]interface{} `yaml:"environment,omitempty"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
}
var st tasksteps
var st *runStep
if err := unmarshal(&st); err != nil {
return err
}
s.Step = st.Step
s.Command = st.Command
s.WorkingDir = st.WorkingDir
s.Shell = st.Shell
s.User = st.User
if st.Environment != nil {
env, err := parseEnv(st.Environment)
if err != nil {
return err
}
s.Environment = env
}
return nil
}
func (c *Container) UnmarshalYAML(unmarshal func(interface{}) error) error {
type container struct {
Image string `yaml:"image,omitempty"`
Environment map[string]interface{} `yaml:"environment,omitempty"`
User string `yaml:"user"`
Privileged bool `yaml:"privileged"`
Entrypoint string `yaml:"entrypoint"`
}
var ct *container
if err := unmarshal(&ct); err != nil {
return err
}
c.Image = ct.Image
c.User = ct.User
c.Privileged = ct.Privileged
c.Entrypoint = ct.Entrypoint
if ct.Environment != nil {
env, err := parseEnv(ct.Environment)
if err != nil {
return err
}
c.Environment = env
}
return nil
}
func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
type task struct {
Name string `yaml:"name"`
Runtime string `yaml:"runtime"`
Environment map[string]interface{} `yaml:"environment,omitempty"`
WorkingDir string `yaml:"working_dir"`
Shell string `yaml:"shell"`
User string `yaml:"user"`
Steps []map[string]interface{} `yaml:"steps"`
}
var tt *task
if err := unmarshal(&tt); err != nil {
return err
}
t.Name = tt.Name
t.Runtime = tt.Runtime
t.WorkingDir = tt.WorkingDir
t.Shell = tt.Shell
t.User = tt.User
steps := make([]interface{}, len(tt.Steps))
for i, stepEntry := range st.Steps {
for i, stepEntry := range tt.Steps {
if len(stepEntry) > 1 {
return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i)
}
@ -201,6 +282,14 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
t.Steps = steps
if tt.Environment != nil {
env, err := parseEnv(tt.Environment)
if err != nil {
return err
}
t.Environment = env
}
return nil
}
@ -297,6 +386,36 @@ func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}
func parseEnv(ienv map[string]interface{}) (map[string]EnvVar, error) {
env := map[string]EnvVar{}
for envName, envEntry := range ienv {
switch envValue := envEntry.(type) {
case string:
env[envName] = EnvVar{
Type: EnvVarTypeString,
Value: envValue,
}
case map[interface{}]interface{}:
for k, v := range envValue {
if k == "from_variable" {
switch v.(type) {
case string:
default:
return nil, errors.Errorf("unknown environment value: %v", v)
}
env[envName] = EnvVar{
Type: EnvVarTypeFromVariable,
Value: v.(string),
}
}
}
default:
return nil, errors.Errorf("unknown environment value: %v", envValue)
}
}
return env, nil
}
func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) {
w := &types.WhenConditions{}

View File

@ -136,12 +136,29 @@ func TestParseOutput(t *testing.T) {
type: pod
containers:
- image: image01
environment:
ENV01: ENV01
ENVFROMVARIABLE01:
from_variable: variable01
tasks:
task01:
runtime: runtime01
environment:
ENV01: ENV01
ENVFROMVARIABLE01:
from_variable: variable01
steps:
- run: command01
- run:
name: name different than command
command: command02
- run:
command: command03
environment:
ENV01: ENV01
ENVFROMVARIABLE01:
from_variable: variable01
pipelines:
pipeline01:
@ -165,9 +182,12 @@ func TestParseOutput(t *testing.T) {
Arch: "",
Containers: []*Container{
&Container{
Image: "image01",
Environment: nil,
User: "",
Image: "image01",
Environment: map[string]EnvVar{
"ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
},
User: "",
},
},
},
@ -176,13 +196,40 @@ func TestParseOutput(t *testing.T) {
"task01": &Task{
Name: "task01",
Runtime: "runtime01",
Environment: map[string]string{
"ENV01": "ENV01",
Environment: map[string]EnvVar{
"ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
},
WorkingDir: "",
Shell: "",
User: "",
Steps: []interface{}{},
Steps: []interface{}{
&RunStep{
Step: Step{
Type: "run",
Name: "command01",
},
Command: "command01",
},
&RunStep{
Step: Step{
Type: "run",
Name: "name different than command",
},
Command: "command02",
},
&RunStep{
Step: Step{
Type: "run",
Name: "command03",
},
Command: "command03",
Environment: map[string]EnvVar{
"ENV01": EnvVar{Type: EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": EnvVar{Type: EnvVarTypeFromVariable, Value: "variable01"},
},
},
},
},
},
Pipelines: map[string]*Pipeline{

View File

@ -23,22 +23,26 @@ import (
rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
uuid "github.com/satori/go.uuid"
)
func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime {
func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime {
ce := c.Runtime(runtimeName)
containers := []*rstypes.Container{}
for _, cc := range ce.Containers {
containers = append(containers, &rstypes.Container{
env, err := genEnv(cc.Environment, variables)
if err != nil {
return nil
}
container := &rstypes.Container{
Image: cc.Image,
Environment: cc.Environment,
Environment: env,
User: cc.User,
Privileged: cc.Privileged,
Entrypoint: cc.Entrypoint,
})
}
containers = append(containers, container)
}
return &rstypes.Runtime{
Type: rstypes.RuntimeType(ce.Type),
@ -46,7 +50,7 @@ func genRuntime(c *config.Config, runtimeName string) *rstypes.Runtime {
}
}
func stepFromConfigStep(csi interface{}) interface{} {
func stepFromConfigStep(csi interface{}, variables map[string]string) interface{} {
switch cs := csi.(type) {
case *config.CloneStep:
// transform a "clone" step in a "run" step command
@ -94,10 +98,15 @@ fi
case *config.RunStep:
rs := &rstypes.RunStep{}
env, err := genEnv(cs.Environment, variables)
if err != nil {
return nil
}
rs.Type = cs.Type
rs.Name = cs.Name
rs.Command = cs.Command
rs.Environment = cs.Environment
rs.Environment = env
rs.WorkingDir = cs.WorkingDir
rs.Shell = cs.Shell
rs.User = cs.User
@ -135,7 +144,7 @@ fi
// GenRunConfig generates a run config from a pipeline in the config, expanding all the references to tasks
// this functions assumes that the config is already checked for possible errors (i.e referenced task must exits)
func GenRunConfig(c *config.Config, pipelineName string, env map[string]string, branch, tag, ref string) *rstypes.RunConfig {
func GenRunConfig(uuid util.UUIDGenerator, c *config.Config, pipelineName string, env, variables map[string]string, branch, tag, ref string) *rstypes.RunConfig {
cp := c.Pipeline(pipelineName)
rc := &rstypes.RunConfig{
@ -152,15 +161,20 @@ func GenRunConfig(c *config.Config, pipelineName string, env map[string]string,
steps := make([]interface{}, len(cpt.Steps))
for i, cpts := range cpt.Steps {
steps[i] = stepFromConfigStep(cpts)
steps[i] = stepFromConfigStep(cpts, variables)
}
tEnv, err := genEnv(cpt.Environment, variables)
if err != nil {
return nil
}
t := &rstypes.RunConfigTask{
ID: uuid.NewV4().String(),
ID: uuid.New(cpe.Name).String(),
// use the element name from the config as the task name
Name: cpe.Name,
Runtime: genRuntime(c, cpt.Runtime),
Environment: cpt.Environment,
Runtime: genRuntime(c, cpt.Runtime, variables),
Environment: tEnv,
WorkingDir: cpt.WorkingDir,
Shell: cpt.Shell,
User: cpt.User,
@ -348,3 +362,16 @@ func GetAllParents(rc *rstypes.RunConfig, task *rstypes.RunConfigTask) []*rstype
}
return parents
}
func genEnv(cenv map[string]config.EnvVar, variables map[string]string) (map[string]string, error) {
env := map[string]string{}
for envName, envVar := range cenv {
switch envVar.Type {
case config.EnvVarTypeString:
env[envName] = envVar.Value
case config.EnvVarTypeFromVariable:
env[envName] = variables[envVar.Value]
}
}
return env, nil
}

View File

@ -19,16 +19,21 @@ import (
"reflect"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/services/runservice/types"
"github.com/sorintlab/agola/internal/config"
rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
)
var uuid = &util.TestUUIDGenerator{}
func TestGenTasksLevels(t *testing.T) {
type task struct {
ID string
Level int
Depends []*types.RunConfigTaskDepend
Depends []*rstypes.RunConfigTaskDepend
}
tests := []struct {
name string
@ -84,8 +89,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -99,8 +104,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "2",
Level: 1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -113,8 +118,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -122,8 +127,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -137,8 +142,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -146,8 +151,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -155,8 +160,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -170,8 +175,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -179,8 +184,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -188,8 +193,8 @@ func TestGenTasksLevels(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -200,18 +205,18 @@ func TestGenTasksLevels(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
ID: t.ID,
Level: t.Level,
Depends: t.Depends,
}
}
outRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
outRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
for _, t := range tt.out {
outRunConfig.Tasks[t.ID] = &types.RunConfigTask{
outRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
ID: t.ID,
Level: t.Level,
Depends: t.Depends,
@ -238,7 +243,7 @@ func TestGetAllParents(t *testing.T) {
type task struct {
ID string
Level int
Depends []*types.RunConfigTaskDepend
Depends []*rstypes.RunConfigTaskDepend
}
tests := []struct {
name string
@ -280,8 +285,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -301,8 +306,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -319,11 +324,11 @@ func TestGetAllParents(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
&types.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -331,8 +336,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "4",
},
},
@ -340,8 +345,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "5",
},
},
@ -369,8 +374,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -378,8 +383,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -396,8 +401,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -405,8 +410,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -414,8 +419,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -433,8 +438,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -442,8 +447,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -451,8 +456,8 @@ func TestGetAllParents(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -468,9 +473,9 @@ func TestGetAllParents(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
ID: t.ID,
Level: t.Level,
Depends: t.Depends,
@ -497,7 +502,7 @@ func TestCheckRunConfig(t *testing.T) {
type task struct {
ID string
Level int
Depends []*types.RunConfigTaskDepend
Depends []*rstypes.RunConfigTaskDepend
}
tests := []struct {
name string
@ -536,8 +541,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -550,8 +555,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -559,8 +564,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -579,8 +584,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -588,8 +593,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -597,8 +602,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "1",
},
},
@ -618,8 +623,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "1",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -627,8 +632,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "2",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "3",
},
},
@ -636,8 +641,8 @@ func TestCheckRunConfig(t *testing.T) {
{
ID: "3",
Level: -1,
Depends: []*types.RunConfigTaskDepend{
&types.RunConfigTaskDepend{
Depends: []*rstypes.RunConfigTaskDepend{
&rstypes.RunConfigTaskDepend{
TaskID: "2",
},
},
@ -653,9 +658,9 @@ func TestCheckRunConfig(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
inRunConfig := &types.RunConfig{Tasks: map[string]*types.RunConfigTask{}}
inRunConfig := &rstypes.RunConfig{Tasks: map[string]*rstypes.RunConfigTask{}}
for _, t := range tt.in {
inRunConfig.Tasks[t.ID] = &types.RunConfigTask{
inRunConfig.Tasks[t.ID] = &rstypes.RunConfigTask{
Name: fmt.Sprintf("task%s", t.ID),
ID: t.ID,
Level: t.Level,
@ -682,3 +687,150 @@ func TestCheckRunConfig(t *testing.T) {
})
}
}
func TestGenRunConfig(t *testing.T) {
tests := []struct {
name string
in *config.Config
env map[string]string
variables map[string]string
out *rstypes.RunConfig
}{
{
name: "test runconfig generation",
in: &config.Config{
Runtimes: map[string]*config.Runtime{
"runtime01": &config.Runtime{
Name: "runtime01",
Type: "pod",
Arch: "",
Containers: []*config.Container{
&config.Container{
Image: "image01",
Environment: map[string]config.EnvVar{
"ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
},
User: "",
},
},
},
},
Tasks: map[string]*config.Task{
"task01": &config.Task{
Name: "task01",
Runtime: "runtime01",
Environment: map[string]config.EnvVar{
"ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
},
WorkingDir: "",
Shell: "",
User: "",
Steps: []interface{}{
&config.RunStep{
Step: config.Step{
Type: "run",
Name: "command01",
},
Command: "command01",
},
&config.RunStep{
Step: config.Step{
Type: "run",
Name: "name different than command",
},
Command: "command02",
},
&config.RunStep{
Step: config.Step{
Type: "run",
Name: "command03",
},
Command: "command03",
Environment: map[string]config.EnvVar{
"ENV01": config.EnvVar{Type: config.EnvVarTypeString, Value: "ENV01"},
"ENVFROMVARIABLE01": config.EnvVar{Type: config.EnvVarTypeFromVariable, Value: "variable01"},
},
},
},
},
},
Pipelines: map[string]*config.Pipeline{
"pipeline01": &config.Pipeline{
Name: "pipeline01",
Elements: map[string]*config.Element{
"element01": &config.Element{
Name: "element01",
Task: "task01",
Depends: []*config.Depend{},
IgnoreFailure: false,
Approval: false,
When: &types.When{
Branch: &types.WhenConditions{Include: []types.WhenCondition{{Match: "master"}}},
Tag: &types.WhenConditions{Include: []types.WhenCondition{{Match: "v1.x"}, {Match: "v2.x"}}},
Ref: &types.WhenConditions{
Include: []types.WhenCondition{{Match: "master"}},
Exclude: []types.WhenCondition{{Match: "/branch01/", Type: types.WhenConditionTypeRegExp}, {Match: "branch02"}},
},
},
},
},
},
},
},
env: map[string]string{
"ENV01": "ENVVALUE01",
},
variables: map[string]string{
"variable01": "VARVALUE01",
},
out: &rstypes.RunConfig{
Name: "pipeline01",
Environment: map[string]string{
"ENV01": "ENVVALUE01",
},
Tasks: map[string]*rstypes.RunConfigTask{
uuid.New("element01").String(): &rstypes.RunConfigTask{
ID: uuid.New("element01").String(),
Name: "element01", Depends: []*rstypes.RunConfigTaskDepend{},
Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"),
Containers: []*rstypes.Container{
{
Image: "image01",
Environment: map[string]string{
"ENV01": "ENV01",
"ENVFROMVARIABLE01": "VARVALUE01",
},
},
},
},
Environment: map[string]string{
"ENV01": "ENV01",
"ENVFROMVARIABLE01": "VARVALUE01",
},
Steps: []interface{}{
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command01"}, Command: "command01", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "name different than command"}, Command: "command02", Environment: map[string]string{}},
&rstypes.RunStep{Step: rstypes.Step{Type: "run", Name: "command03"}, Command: "command03", Environment: map[string]string{"ENV01": "ENV01", "ENVFROMVARIABLE01": "VARVALUE01"}},
},
Skip: true,
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out := GenRunConfig(uuid, tt.in, "pipeline01", tt.env, tt.variables, "", "", "")
//if err != nil {
// t.Fatalf("unexpected error: %v", err)
//}
if diff := cmp.Diff(tt.out, out); diff != "" {
t.Error(diff)
}
})
}
}

View File

@ -121,6 +121,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
var skipSSHHostKeyCheck bool
var runType types.RunType
var userID string
variables := map[string]string{}
var gitSource gitsource.GitSource
if !isUserBuild {
@ -128,7 +129,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
}
h.log.Debugf("project: %s", util.Dump(project))
h.log.Infof("project: %s", util.Dump(project))
user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
if err != nil {
@ -159,6 +160,46 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
}
webhookData.ProjectID = projectID
// get project variables
pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, project.ID, true)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project variables")
}
h.log.Infof("pvars: %v", util.Dump(pvars))
// remove overriden variables
pvars = common.FilterOverridenVariables(pvars)
h.log.Infof("pvars: %v", util.Dump(pvars))
// get project secrets
secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, project.ID, true)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get project secrets")
}
h.log.Infof("secrets: %v", util.Dump(secrets))
for _, pvar := range pvars {
// find the value match
var varval types.VariableValue
for _, varval = range pvar.Values {
h.log.Infof("varval: %v", util.Dump(varval))
match := types.MatchWhen(varval.When, webhookData.Branch, webhookData.Tag, webhookData.Ref)
if !match {
continue
}
// get the secret value referenced by the variable, it must be a secret at the same level or a lower level
secret := common.GetVarValueMatchingSecret(varval, pvar.Parent.Path, secrets)
h.log.Infof("secret: %v", util.Dump(secret))
if secret != nil {
varValue, ok := secret.Data[varval.SecretVar]
if ok {
variables[pvar.Name] = varValue
}
}
break
}
}
h.log.Infof("variables: %v", util.Dump(variables))
} else {
gitSource = agolagit.New(h.apiExposedURL + "/repos")
var err error
@ -200,6 +241,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to parse clone url")
}
// this env vars ovverrides other env vars
env := map[string]string{
"CI": "true",
"AGOLA_SSHPRIVKEY": sshPrivKey,
@ -251,7 +293,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
group = genGroup(userID, webhookData)
}
if err := h.createRuns(ctx, data, group, annotations, env, webhookData); err != nil {
if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")
}
//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
@ -261,16 +303,16 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
return 0, "", nil
}
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env map[string]string, webhookData *types.WebhookData) error {
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, env, variables map[string]string, webhookData *types.WebhookData) error {
config, err := config.ParseConfig([]byte(configData))
if err != nil {
return err
return errors.Wrapf(err, "failed to parse config")
}
//h.log.Debugf("config: %v", util.Dump(config))
//h.log.Debugf("pipeline: %s", createRunOpts.PipelineName)
for _, pipeline := range config.Pipelines {
rc := runconfig.GenRunConfig(config, pipeline.Name, env, webhookData.Branch, webhookData.Tag, webhookData.Ref)
rc := runconfig.GenRunConfig(util.DefaultUUIDGenerator{}, config, pipeline.Name, env, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)
h.log.Debugf("rc: %s", util.Dump(rc))
h.log.Infof("group: %s", group)

35
internal/util/uuid.go Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
uuid "github.com/satori/go.uuid"
)
type UUIDGenerator interface {
New(s string) uuid.UUID
}
type DefaultUUIDGenerator struct{}
func (u DefaultUUIDGenerator) New(s string) uuid.UUID {
return uuid.NewV4()
}
type TestUUIDGenerator struct{}
func (u TestUUIDGenerator) New(s string) uuid.UUID {
return uuid.NewV5(uuid.NamespaceDNS, s)
}