From 2676770336548f9ec9579962585c851aabf56701 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Mon, 5 Aug 2019 17:33:30 +0200 Subject: [PATCH] userdirectrun: add options to define variables Add a --var and --var-file options (repeatable multiple times) to define the variables to be used in the run. --- cmd/agola/cmd/directrunstart.go | 58 ++++++ .../services/executor/driver/docker_test.go | 43 +---- internal/services/executor/driver/k8s_test.go | 4 +- internal/services/gateway/action/run.go | 6 +- internal/services/gateway/action/user.go | 2 + internal/services/gateway/api/user.go | 1 + internal/testutil/env.go | 58 ++++++ services/gateway/api/types/user.go | 3 +- services/gateway/client/client.go | 13 ++ tests/setup_test.go | 170 ++++++++++++++++++ 10 files changed, 314 insertions(+), 44 deletions(-) create mode 100644 internal/testutil/env.go diff --git a/cmd/agola/cmd/directrunstart.go b/cmd/agola/cmd/directrunstart.go index ec9383e..672c0dc 100644 --- a/cmd/agola/cmd/directrunstart.go +++ b/cmd/agola/cmd/directrunstart.go @@ -17,16 +17,21 @@ package cmd import ( "context" "fmt" + "io/ioutil" "path" "regexp" + "strings" + "unicode" gitsave "agola.io/agola/internal/git-save" "agola.io/agola/internal/util" gwapitypes "agola.io/agola/services/gateway/api/types" gwclient "agola.io/agola/services/gateway/client" + "github.com/ghodss/yaml" uuid "github.com/satori/go.uuid" "github.com/spf13/cobra" + errors "golang.org/x/xerrors" ) var cmdDirectRunStart = &cobra.Command{ @@ -47,6 +52,9 @@ type directRunStartOptions struct { tag string ref string prRefRegexes []string + + vars []string + varFiles []string } var directRunStartOpts directRunStartOptions @@ -60,10 +68,30 @@ func init() { flags.StringVar(&directRunStartOpts.tag, "tag", "", "tag to push to") flags.StringVar(&directRunStartOpts.ref, "ref", "", `ref to push to (i.e "refs/heads/master" for a branch, "refs/tags/v1.0" for a tag)`) flags.StringArrayVar(&directRunStartOpts.prRefRegexes, "pull-request-ref-regexes", []string{`refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`}, `regular expression to determine if a ref is a pull request`) + flags.StringArrayVar(&directRunStartOpts.vars, "var", []string{}, `list of variables (name=value). This option can be repeated multiple times`) + flags.StringArrayVar(&directRunStartOpts.varFiles, "var-file", []string{}, `yaml file containing the variables as a yaml/json map. This option can be repeated multiple times`) cmdDirectRun.AddCommand(cmdDirectRunStart) } +func parseVariable(variable string) (string, string, error) { + // trim white spaces at the start + variable = strings.TrimLeftFunc(variable, unicode.IsSpace) + arr := strings.SplitN(variable, "=", 2) + if len(arr) != 2 { + return "", "", fmt.Errorf("invalid variable definition: %s", variable) + } + varname := arr[0] + varvalue := arr[1] + if varname == "" { + return "", "", fmt.Errorf("invalid variable definition: %s", variable) + } + if varvalue == "" { + return "", "", fmt.Errorf("invalid variable definition: %s", variable) + } + return varname, varvalue, nil +} + func directRunStart(cmd *cobra.Command, args []string) error { gwclient := gwclient.NewClient(gatewayURL, token) @@ -102,6 +130,35 @@ func directRunStart(cmd *cobra.Command, args []string) error { return err } + variables := map[string]string{} + + // TODO(sgotti) currently vars overrides varFiles. Is this what we want or we + // want to handle var and varFiles in the order they appear in the command + // line? + for _, varFile := range directRunStartOpts.varFiles { + // "github.com/ghodss/yaml" doesn't provide a streaming decoder + var data []byte + var err error + data, err = ioutil.ReadFile(varFile) + if err != nil { + return err + } + + if err := yaml.Unmarshal(data, &variables); err != nil { + return errors.Errorf("failed to unmarshal values: %v", err) + } + + // TODO(sgotti) validate variable name + } + + for _, variable := range directRunStartOpts.vars { + varname, varvalue, err := parseVariable(variable) + if err != nil { + return err + } + variables[varname] = varvalue + } + // setup unique local git repo uuid git := &util.Git{} repoUUID, _ := git.ConfigGet(context.Background(), "agola.repouuid") @@ -154,6 +211,7 @@ func directRunStart(cmd *cobra.Command, args []string) error { CommitSHA: commitSHA, Message: message, PullRequestRefRegexes: directRunStartOpts.prRefRegexes, + Variables: variables, } if _, err := gwclient.UserCreateRun(context.TODO(), req); err != nil { return err diff --git a/internal/services/executor/driver/docker_test.go b/internal/services/executor/driver/docker_test.go index f7e1071..81dfa79 100644 --- a/internal/services/executor/driver/docker_test.go +++ b/internal/services/executor/driver/docker_test.go @@ -15,21 +15,17 @@ package driver import ( - "bufio" "bytes" "context" - "fmt" - "io" "io/ioutil" "os" - "strings" "testing" "time" - "unicode" slog "agola.io/agola/internal/log" "github.com/docker/docker/api/types" uuid "github.com/satori/go.uuid" + "agola.io/agola/internal/testutil" "github.com/google/go-cmp/cmp" "go.uber.org/zap" @@ -39,41 +35,6 @@ import ( var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) var logger = slog.New(level) -func parseEnv(envvar string) (string, string, error) { - // trim white spaces at the start - envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace) - arr := strings.SplitN(envvar, "=", 2) - varname := arr[0] - if varname == "" { - return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar) - } - if len(arr) > 1 { - if arr[1] == "" { - return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar) - } - return varname, arr[1], nil - } - return varname, "", nil -} - -func parseEnvs(r io.Reader) (map[string]string, error) { - envs := map[string]string{} - - scanner := bufio.NewScanner(r) - for scanner.Scan() { - envname, envvalue, err := parseEnv(scanner.Text()) - if err != nil { - return nil, err - } - envs[envname] = envvalue - } - if err := scanner.Err(); err != nil { - return nil, err - } - - return envs, nil -} - func TestDockerPod(t *testing.T) { if os.Getenv("SKIP_DOCKER_TESTS") == "1" { t.Skip("skipping since env var SKIP_DOCKER_TESTS is 1") @@ -192,7 +153,7 @@ func TestDockerPod(t *testing.T) { t.Fatalf("unexpected exit code: %d", code) } - curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes())) + curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes())) if err != nil { t.Fatalf("unexpected err: %v", err) } diff --git a/internal/services/executor/driver/k8s_test.go b/internal/services/executor/driver/k8s_test.go index b99d37b..e1206df 100644 --- a/internal/services/executor/driver/k8s_test.go +++ b/internal/services/executor/driver/k8s_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "agola.io/agola/internal/testutil" + uuid "github.com/satori/go.uuid" ) @@ -146,7 +148,7 @@ func TestK8sPod(t *testing.T) { t.Fatalf("unexpected exit code: %d", code) } - curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes())) + curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes())) if err != nil { t.Fatalf("unexpected err: %v", err) } diff --git a/internal/services/gateway/action/run.go b/internal/services/gateway/action/run.go index 91fbd67..b9ccf84 100644 --- a/internal/services/gateway/action/run.go +++ b/internal/services/gateway/action/run.go @@ -321,7 +321,9 @@ type CreateRunRequest struct { // commit compare link CompareLink string + // fields only used with user direct runs UserRunRepoUUID string + Variables map[string]string } func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) error { @@ -391,13 +393,15 @@ func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) e env["AGOLA_SKIPSSHHOSTKEYCHECK"] = "1" } - variables := map[string]string{} + var variables map[string]string if req.RunType == types.RunTypeProject { var err error variables, err = h.genRunVariables(ctx, req) if err != nil { return err } + } else { + variables = req.Variables } annotations := map[string]string{ diff --git a/internal/services/gateway/action/user.go b/internal/services/gateway/action/user.go index ef6ab8b..9fd88c7 100644 --- a/internal/services/gateway/action/user.go +++ b/internal/services/gateway/action/user.go @@ -824,6 +824,7 @@ type UserCreateRunRequest struct { Message string PullRequestRefRegexes []string + Variables map[string]string } func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunRequest) error { @@ -941,6 +942,7 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq PullRequestLink: "", UserRunRepoUUID: req.RepoUUID, + Variables: req.Variables, } return h.CreateRuns(ctx, creq) diff --git a/internal/services/gateway/api/user.go b/internal/services/gateway/api/user.go index 31e078a..db2a188 100644 --- a/internal/services/gateway/api/user.go +++ b/internal/services/gateway/api/user.go @@ -578,6 +578,7 @@ func (h *UserCreateRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) CommitSHA: req.CommitSHA, Message: req.Message, PullRequestRefRegexes: req.PullRequestRefRegexes, + Variables: req.Variables, } err := h.ah.UserCreateRun(ctx, creq) if httpError(w, err) { diff --git a/internal/testutil/env.go b/internal/testutil/env.go new file mode 100644 index 0000000..60abf9f --- /dev/null +++ b/internal/testutil/env.go @@ -0,0 +1,58 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "bufio" + "fmt" + "io" + "strings" + "unicode" +) + +func ParseEnv(envvar string) (string, string, error) { + // trim white spaces at the start + envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace) + arr := strings.SplitN(envvar, "=", 2) + if len(arr) == 0 { + return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar) + } + varname := arr[0] + if varname == "" { + return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar) + } + if len(arr) == 1 { + return varname, "", nil + } + return varname, arr[1], nil +} + +func ParseEnvs(r io.Reader) (map[string]string, error) { + envs := map[string]string{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + envname, envvalue, err := ParseEnv(scanner.Text()) + if err != nil { + return nil, err + } + envs[envname] = envvalue + } + if err := scanner.Err(); err != nil { + return nil, err + } + + return envs, nil +} diff --git a/services/gateway/api/types/user.go b/services/gateway/api/types/user.go index 6818840..06f7910 100644 --- a/services/gateway/api/types/user.go +++ b/services/gateway/api/types/user.go @@ -103,5 +103,6 @@ type UserCreateRunRequest struct { CommitSHA string `json:"commit_sha,omitempty"` Message string `json:"message,omitempty"` - PullRequestRefRegexes []string + PullRequestRefRegexes []string `json:"pull_request_ref_regexes,omitempty"` + Variables map[string]string `json:"variables,omitempty"` } diff --git a/services/gateway/client/client.go b/services/gateway/client/client.go index b2759ed..7ddff57 100644 --- a/services/gateway/client/client.go +++ b/services/gateway/client/client.go @@ -410,6 +410,19 @@ func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups, return getRunsResponse, resp, err } +func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) { + q := url.Values{} + q.Add("runID", runID) + q.Add("taskID", taskID) + if setup { + q.Add("setup", "") + } else { + q.Add("step", strconv.Itoa(step)) + } + + return c.getResponse(ctx, "GET", "/logs", q, nil, nil) +} + func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) { rs := new(gwapitypes.RemoteSourceResponse) resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs) diff --git a/tests/setup_test.go b/tests/setup_test.go index b1e74df..345f574 100644 --- a/tests/setup_test.go +++ b/tests/setup_test.go @@ -15,6 +15,7 @@ package tests import ( + "bytes" "context" "fmt" "io/ioutil" @@ -805,3 +806,172 @@ func TestDirectRun(t *testing.T) { }) } } + +func TestDirectRunVariables(t *testing.T) { + config := ` + { + runs: [ + { + name: 'run01', + tasks: [ + { + name: 'task01', + runtime: { + containers: [ + { + image: 'alpine/git', + }, + ], + }, + environment: { + ENV01: { from_variable: 'variable01' }, + ENV02: { from_variable: 'variable02' }, + }, + steps: [ + { type: 'clone' }, + { type: 'run', command: 'env' }, + ], + }, + ], + }, + ], + } + ` + + varfile01 := ` + variable01: "variable value 01" + variable02: variable value 02 +` + + tests := []struct { + name string + args []string + env map[string]string + }{ + { + name: "test direct run without variables", + args: []string{}, + env: map[string]string{ + "ENV01": "", + "ENV02": "", + }, + }, + { + name: "test direct run with two variables", + args: []string{"--var", "variable01=VARIABLEVALUE01", "--var", "variable02=VARIABLEVALUE02"}, + env: map[string]string{ + "ENV01": "VARIABLEVALUE01", + "ENV02": "VARIABLEVALUE02", + }, + }, + { + name: "test direct run with a var file", + args: []string{"--var-file", "../varfile01.yml"}, + env: map[string]string{ + "ENV01": "variable value 01", + "ENV02": "variable value 02", + }, + }, + { + name: "test direct run with a var file and a var that overrides", + args: []string{"--var-file", "../varfile01.yml", "--var", "variable02=VARIABLEVALUE02"}, + env: map[string]string{ + "ENV01": "variable value 01", + "ENV02": "VARIABLEVALUE02", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := ioutil.TempDir("", "agola") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + if err := ioutil.WriteFile(filepath.Join(dir, "varfile01.yml"), []byte(varfile01), 0644); err != nil { + t.Fatalf("unexpected err: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tetcd, tgitea, c := setup(ctx, t, dir) + defer shutdownGitea(tgitea) + defer shutdownEtcd(tetcd) + + gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken") + user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created agola user: %s", user.UserName) + + token := createAgolaUserToken(ctx, t, c) + + // From now use the user token + gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token) + + directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...) + + // TODO(sgotti) add an util to wait for a run phase + time.Sleep(10 * time.Second) + + runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + t.Logf("runs: %s", util.Dump(runs)) + + if len(runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs)) + } + + run, _, err := gwClient.GetRun(ctx, runs[0].ID) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if run.Phase != rstypes.RunPhaseFinished { + t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) + } + if run.Result != rstypes.RunResultSuccess { + t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result) + } + + var task *gwapitypes.RunResponseTask + for _, t := range run.Tasks { + if t.Name == "task01" { + task = t + break + } + } + + resp, err := gwClient.GetLogs(ctx, run.ID, task.ID, false, 1) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer resp.Body.Close() + + logs, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + curEnv, err := testutil.ParseEnvs(bytes.NewReader(logs)) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + for n, e := range tt.env { + if ce, ok := curEnv[n]; !ok { + t.Fatalf("missing env var %s", n) + } else { + if ce != e { + t.Fatalf("different env var %s value, want: %q, got %q", n, e, ce) + } + } + } + }) + } +}