userdirectrun: add options to define variables

Add a --var and --var-file options (repeatable multiple times) to define the
variables to be used in the run.

cmd/agola/cmd/directrunstart.go

@@ -17,16 +17,21 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"path"
 	"regexp"
+	"strings"
+	"unicode"
 
 	gitsave "agola.io/agola/internal/git-save"
 	"agola.io/agola/internal/util"
 	gwapitypes "agola.io/agola/services/gateway/api/types"
 	gwclient "agola.io/agola/services/gateway/client"
 
+	"github.com/ghodss/yaml"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/cobra"
+	errors "golang.org/x/xerrors"
 )
 
 var cmdDirectRunStart = &cobra.Command{
@@ -47,6 +52,9 @@ type directRunStartOptions struct {
 	tag          string
 	ref          string
 	prRefRegexes []string
+
+	vars     []string
+	varFiles []string
 }
 
 var directRunStartOpts directRunStartOptions
@@ -60,10 +68,30 @@ func init() {
 	flags.StringVar(&directRunStartOpts.tag, "tag", "", "tag to push to")
 	flags.StringVar(&directRunStartOpts.ref, "ref", "", `ref to push to (i.e  "refs/heads/master" for a branch, "refs/tags/v1.0" for a tag)`)
 	flags.StringArrayVar(&directRunStartOpts.prRefRegexes, "pull-request-ref-regexes", []string{`refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`}, `regular expression to determine if a ref is a pull request`)
+	flags.StringArrayVar(&directRunStartOpts.vars, "var", []string{}, `list of variables (name=value). This option can be repeated multiple times`)
+	flags.StringArrayVar(&directRunStartOpts.varFiles, "var-file", []string{}, `yaml file containing the variables as a yaml/json map. This option can be repeated multiple times`)
 
 	cmdDirectRun.AddCommand(cmdDirectRunStart)
 }
 
+func parseVariable(variable string) (string, string, error) {
+	// trim white spaces at the start
+	variable = strings.TrimLeftFunc(variable, unicode.IsSpace)
+	arr := strings.SplitN(variable, "=", 2)
+	if len(arr) != 2 {
+		return "", "", fmt.Errorf("invalid variable definition: %s", variable)
+	}
+	varname := arr[0]
+	varvalue := arr[1]
+	if varname == "" {
+		return "", "", fmt.Errorf("invalid variable definition: %s", variable)
+	}
+	if varvalue == "" {
+		return "", "", fmt.Errorf("invalid variable definition: %s", variable)
+	}
+	return varname, varvalue, nil
+}
+
 func directRunStart(cmd *cobra.Command, args []string) error {
 	gwclient := gwclient.NewClient(gatewayURL, token)
 
@@ -102,6 +130,35 @@ func directRunStart(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	variables := map[string]string{}
+
+	// TODO(sgotti) currently vars overrides varFiles. Is this what we want or we
+	// want to handle var and varFiles in the order they appear in the command
+	// line?
+	for _, varFile := range directRunStartOpts.varFiles {
+		// "github.com/ghodss/yaml" doesn't provide a streaming decoder
+		var data []byte
+		var err error
+		data, err = ioutil.ReadFile(varFile)
+		if err != nil {
+			return err
+		}
+
+		if err := yaml.Unmarshal(data, &variables); err != nil {
+			return errors.Errorf("failed to unmarshal values: %v", err)
+		}
+
+		// TODO(sgotti) validate variable name
+	}
+
+	for _, variable := range directRunStartOpts.vars {
+		varname, varvalue, err := parseVariable(variable)
+		if err != nil {
+			return err
+		}
+		variables[varname] = varvalue
+	}
+
 	// setup unique local git repo uuid
 	git := &util.Git{}
 	repoUUID, _ := git.ConfigGet(context.Background(), "agola.repouuid")
@@ -154,6 +211,7 @@ func directRunStart(cmd *cobra.Command, args []string) error {
 		CommitSHA:             commitSHA,
 		Message:               message,
 		PullRequestRefRegexes: directRunStartOpts.prRefRegexes,
+		Variables:             variables,
 	}
 	if _, err := gwclient.UserCreateRun(context.TODO(), req); err != nil {
 		return err

internal/services/executor/driver/docker_test.go

@@ -15,21 +15,17 @@
 package driver
 
 import (
-	"bufio"
 	"bytes"
 	"context"
-	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
-	"strings"
 	"testing"
 	"time"
-	"unicode"
 
 	slog "agola.io/agola/internal/log"
 	"github.com/docker/docker/api/types"
 	uuid "github.com/satori/go.uuid"
+	"agola.io/agola/internal/testutil"
 
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
@@ -39,41 +35,6 @@ import (
 var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
 var logger = slog.New(level)
 
-func parseEnv(envvar string) (string, string, error) {
-	// trim white spaces at the start
-	envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace)
-	arr := strings.SplitN(envvar, "=", 2)
-	varname := arr[0]
-	if varname == "" {
-		return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
-	}
-	if len(arr) > 1 {
-		if arr[1] == "" {
-			return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
-		}
-		return varname, arr[1], nil
-	}
-	return varname, "", nil
-}
-
-func parseEnvs(r io.Reader) (map[string]string, error) {
-	envs := map[string]string{}
-
-	scanner := bufio.NewScanner(r)
-	for scanner.Scan() {
-		envname, envvalue, err := parseEnv(scanner.Text())
-		if err != nil {
-			return nil, err
-		}
-		envs[envname] = envvalue
-	}
-	if err := scanner.Err(); err != nil {
-		return nil, err
-	}
-
-	return envs, nil
-}
-
 func TestDockerPod(t *testing.T) {
 	if os.Getenv("SKIP_DOCKER_TESTS") == "1" {
 		t.Skip("skipping since env var SKIP_DOCKER_TESTS is 1")
@@ -192,7 +153,7 @@ func TestDockerPod(t *testing.T) {
 			t.Fatalf("unexpected exit code: %d", code)
 		}
 
-		curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes()))
+		curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes()))
 		if err != nil {
 			t.Fatalf("unexpected err: %v", err)
 		}

internal/services/executor/driver/k8s_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"agola.io/agola/internal/testutil"
+
 	uuid "github.com/satori/go.uuid"
 )
 
@@ -146,7 +148,7 @@ func TestK8sPod(t *testing.T) {
 			t.Fatalf("unexpected exit code: %d", code)
 		}
 
-		curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes()))
+		curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes()))
 		if err != nil {
 			t.Fatalf("unexpected err: %v", err)
 		}

internal/services/gateway/action/run.go

@@ -321,7 +321,9 @@ type CreateRunRequest struct {
 	// commit compare link
 	CompareLink string
 
+	// fields only used with user direct runs
 	UserRunRepoUUID string
+	Variables       map[string]string
 }
 
 func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) error {
@@ -391,13 +393,15 @@ func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) e
 		env["AGOLA_SKIPSSHHOSTKEYCHECK"] = "1"
 	}
 
-	variables := map[string]string{}
+	var variables map[string]string
 	if req.RunType == types.RunTypeProject {
 		var err error
 		variables, err = h.genRunVariables(ctx, req)
 		if err != nil {
 			return err
 		}
+	} else {
+		variables = req.Variables
 	}
 
 	annotations := map[string]string{

internal/services/gateway/action/user.go

@@ -824,6 +824,7 @@ type UserCreateRunRequest struct {
 	Message   string
 
 	PullRequestRefRegexes []string
+	Variables             map[string]string
 }
 
 func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunRequest) error {
@@ -941,6 +942,7 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
 		PullRequestLink: "",
 
 		UserRunRepoUUID: req.RepoUUID,
+		Variables:       req.Variables,
 	}
 
 	return h.CreateRuns(ctx, creq)

internal/services/gateway/api/user.go

@@ -578,6 +578,7 @@ func (h *UserCreateRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 		CommitSHA:             req.CommitSHA,
 		Message:               req.Message,
 		PullRequestRefRegexes: req.PullRequestRefRegexes,
+		Variables:             req.Variables,
 	}
 	err := h.ah.UserCreateRun(ctx, creq)
 	if httpError(w, err) {

internal/testutil/env.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+	"unicode"
+)
+
+func ParseEnv(envvar string) (string, string, error) {
+	// trim white spaces at the start
+	envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace)
+	arr := strings.SplitN(envvar, "=", 2)
+	if len(arr) == 0 {
+		return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
+	}
+	varname := arr[0]
+	if varname == "" {
+		return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
+	}
+	if len(arr) == 1 {
+		return varname, "", nil
+	}
+	return varname, arr[1], nil
+}
+
+func ParseEnvs(r io.Reader) (map[string]string, error) {
+	envs := map[string]string{}
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		envname, envvalue, err := ParseEnv(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		envs[envname] = envvalue
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return envs, nil
+}

services/gateway/api/types/user.go

@@ -103,5 +103,6 @@ type UserCreateRunRequest struct {
 	CommitSHA string `json:"commit_sha,omitempty"`
 	Message   string `json:"message,omitempty"`
 
-	PullRequestRefRegexes []string
+	PullRequestRefRegexes []string          `json:"pull_request_ref_regexes,omitempty"`
+	Variables             map[string]string `json:"variables,omitempty"`
 }

services/gateway/client/client.go

@@ -410,6 +410,19 @@ func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups,
 	return getRunsResponse, resp, err
 }
 
+func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
+	q := url.Values{}
+	q.Add("runID", runID)
+	q.Add("taskID", taskID)
+	if setup {
+		q.Add("setup", "")
+	} else {
+		q.Add("step", strconv.Itoa(step))
+	}
+
+	return c.getResponse(ctx, "GET", "/logs", q, nil, nil)
+}
+
 func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) {
 	rs := new(gwapitypes.RemoteSourceResponse)
 	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs)

tests/setup_test.go

@@ -15,6 +15,7 @@
 package tests
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -805,3 +806,172 @@ func TestDirectRun(t *testing.T) {
 		})
 	}
 }
+
+func TestDirectRunVariables(t *testing.T) {
+	config := `
+      {
+        runs: [
+          {
+            name: 'run01',
+            tasks: [
+              {
+                name: 'task01',
+                runtime: {
+                  containers: [
+                    {
+                      image: 'alpine/git',
+                    },
+                  ],
+                },
+                environment: {
+                  ENV01: { from_variable: 'variable01' },
+                  ENV02: { from_variable: 'variable02' },
+                },
+                steps: [
+                  { type: 'clone' },
+                  { type: 'run', command: 'env' },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+	`
+
+	varfile01 := `
+      variable01: "variable value 01"
+      variable02: variable value 02
+`
+
+	tests := []struct {
+		name string
+		args []string
+		env  map[string]string
+	}{
+		{
+			name: "test direct run without variables",
+			args: []string{},
+			env: map[string]string{
+				"ENV01": "",
+				"ENV02": "",
+			},
+		},
+		{
+			name: "test direct run with two variables",
+			args: []string{"--var", "variable01=VARIABLEVALUE01", "--var", "variable02=VARIABLEVALUE02"},
+			env: map[string]string{
+				"ENV01": "VARIABLEVALUE01",
+				"ENV02": "VARIABLEVALUE02",
+			},
+		},
+		{
+			name: "test direct run with a var file",
+			args: []string{"--var-file", "../varfile01.yml"},
+			env: map[string]string{
+				"ENV01": "variable value 01",
+				"ENV02": "variable value 02",
+			},
+		},
+		{
+			name: "test direct run with a var file and a var that overrides",
+			args: []string{"--var-file", "../varfile01.yml", "--var", "variable02=VARIABLEVALUE02"},
+			env: map[string]string{
+				"ENV01": "variable value 01",
+				"ENV02": "VARIABLEVALUE02",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir, err := ioutil.TempDir("", "agola")
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			defer os.RemoveAll(dir)
+
+			if err := ioutil.WriteFile(filepath.Join(dir, "varfile01.yml"), []byte(varfile01), 0644); err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			tetcd, tgitea, c := setup(ctx, t, dir)
+			defer shutdownGitea(tgitea)
+			defer shutdownEtcd(tetcd)
+
+			gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
+			user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			t.Logf("created agola user: %s", user.UserName)
+
+			token := createAgolaUserToken(ctx, t, c)
+
+			// From now use the user token
+			gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
+
+			directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
+
+			// TODO(sgotti) add an util to wait for a run phase
+			time.Sleep(10 * time.Second)
+
+			runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			t.Logf("runs: %s", util.Dump(runs))
+
+			if len(runs) != 1 {
+				t.Fatalf("expected 1 run got: %d", len(runs))
+			}
+
+			run, _, err := gwClient.GetRun(ctx, runs[0].ID)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			if run.Phase != rstypes.RunPhaseFinished {
+				t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
+			}
+			if run.Result != rstypes.RunResultSuccess {
+				t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
+			}
+
+			var task *gwapitypes.RunResponseTask
+			for _, t := range run.Tasks {
+				if t.Name == "task01" {
+					task = t
+					break
+				}
+			}
+
+			resp, err := gwClient.GetLogs(ctx, run.ID, task.ID, false, 1)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			defer resp.Body.Close()
+
+			logs, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			curEnv, err := testutil.ParseEnvs(bytes.NewReader(logs))
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			for n, e := range tt.env {
+				if ce, ok := curEnv[n]; !ok {
+					t.Fatalf("missing env var %s", n)
+				} else {
+					if ce != e {
+						t.Fatalf("different env var %s value, want: %q, got %q", n, e, ce)
+					}
+				}
+			}
+		})
+	}
+}