a/agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

userdirectrun: add options to define variables

Browse Source 
Add a --var and --var-file options (repeatable multiple times) to define the
variables to be used in the run.
This commit is contained in:
Simone Gotti 2019-08-05 17:33:30 +02:00
parent 252bd95a58
commit 2676770336
10 changed files with 314 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
cmd/agola/cmd/directrunstart.go
View File

@ -17,16 +17,21 @@ package cmd
import (
"context"
"fmt"
"io/ioutil"
"path"
"regexp"
"strings"
"unicode"
gitsave "agola.io/agola/internal/git-save"
"agola.io/agola/internal/util"
gwapitypes "agola.io/agola/services/gateway/api/types"
gwclient "agola.io/agola/services/gateway/client"
"github.com/ghodss/yaml"
uuid "github.com/satori/go.uuid"
"github.com/spf13/cobra"
errors "golang.org/x/xerrors"
)
var cmdDirectRunStart = &cobra.Command{
@ -47,6 +52,9 @@ type directRunStartOptions struct {
tag string
ref string
prRefRegexes []string
vars []string
varFiles []string
}
var directRunStartOpts directRunStartOptions
@ -60,10 +68,30 @@ func init() {
flags.StringVar(&directRunStartOpts.tag, "tag", "", "tag to push to")
flags.StringVar(&directRunStartOpts.ref, "ref", "", `ref to push to (i.e "refs/heads/master" for a branch, "refs/tags/v1.0" for a tag)`)
flags.StringArrayVar(&directRunStartOpts.prRefRegexes, "pull-request-ref-regexes", []string{`refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`}, `regular expression to determine if a ref is a pull request`)
flags.StringArrayVar(&directRunStartOpts.vars, "var", []string{}, `list of variables (name=value). This option can be repeated multiple times`)
flags.StringArrayVar(&directRunStartOpts.varFiles, "var-file", []string{}, `yaml file containing the variables as a yaml/json map. This option can be repeated multiple times`)
cmdDirectRun.AddCommand(cmdDirectRunStart)
}
func parseVariable(variable string) (string, string, error) {
// trim white spaces at the start
variable = strings.TrimLeftFunc(variable, unicode.IsSpace)
arr := strings.SplitN(variable, "=", 2)
if len(arr) != 2 {
return "", "", fmt.Errorf("invalid variable definition: %s", variable)
}
varname := arr[0]
varvalue := arr[1]
if varname == "" {
return "", "", fmt.Errorf("invalid variable definition: %s", variable)
}
if varvalue == "" {
return "", "", fmt.Errorf("invalid variable definition: %s", variable)
}
return varname, varvalue, nil
}
func directRunStart(cmd *cobra.Command, args []string) error {
gwclient := gwclient.NewClient(gatewayURL, token)
@ -102,6 +130,35 @@ func directRunStart(cmd *cobra.Command, args []string) error {
return err
}
variables := map[string]string{}
// TODO(sgotti) currently vars overrides varFiles. Is this what we want or we
// want to handle var and varFiles in the order they appear in the command
// line?
for _, varFile := range directRunStartOpts.varFiles {
// "github.com/ghodss/yaml" doesn't provide a streaming decoder
var data []byte
var err error
data, err = ioutil.ReadFile(varFile)
if err != nil {
return err
}
if err := yaml.Unmarshal(data, &variables); err != nil {
return errors.Errorf("failed to unmarshal values: %v", err)
}
// TODO(sgotti) validate variable name
}
for _, variable := range directRunStartOpts.vars {
varname, varvalue, err := parseVariable(variable)
if err != nil {
return err
}
variables[varname] = varvalue
}
// setup unique local git repo uuid
git := &util.Git{}
repoUUID, _ := git.ConfigGet(context.Background(), "agola.repouuid")
@ -154,6 +211,7 @@ func directRunStart(cmd *cobra.Command, args []string) error {
CommitSHA: commitSHA,
Message: message,
PullRequestRefRegexes: directRunStartOpts.prRefRegexes,
Variables: variables,
}
if _, err := gwclient.UserCreateRun(context.TODO(), req); err != nil {
return err

43
internal/services/executor/driver/docker_test.go
View File

@ -15,21 +15,17 @@
package driver
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"testing"
"time"
"unicode"
slog "agola.io/agola/internal/log"
"github.com/docker/docker/api/types"
uuid "github.com/satori/go.uuid"
"agola.io/agola/internal/testutil"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
@ -39,41 +35,6 @@ import (
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
func parseEnv(envvar string) (string, string, error) {
// trim white spaces at the start
envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace)
arr := strings.SplitN(envvar, "=", 2)
varname := arr[0]
if varname == "" {
return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
}
if len(arr) > 1 {
if arr[1] == "" {
return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
}
return varname, arr[1], nil
}
return varname, "", nil
}
func parseEnvs(r io.Reader) (map[string]string, error) {
envs := map[string]string{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
envname, envvalue, err := parseEnv(scanner.Text())
if err != nil {
return nil, err
}
envs[envname] = envvalue
}
if err := scanner.Err(); err != nil {
return nil, err
}
return envs, nil
}
func TestDockerPod(t *testing.T) {
if os.Getenv("SKIP_DOCKER_TESTS") == "1" {
t.Skip("skipping since env var SKIP_DOCKER_TESTS is 1")
@ -192,7 +153,7 @@ func TestDockerPod(t *testing.T) {
t.Fatalf("unexpected exit code: %d", code)
}
curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes()))
curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}

4
internal/services/executor/driver/k8s_test.go
View File

@ -23,6 +23,8 @@ import (
"testing"
"time"
"agola.io/agola/internal/testutil"
uuid "github.com/satori/go.uuid"
)
@ -146,7 +148,7 @@ func TestK8sPod(t *testing.T) {
t.Fatalf("unexpected exit code: %d", code)
}
curEnv, err := parseEnvs(bytes.NewReader(buf.Bytes()))
curEnv, err := testutil.ParseEnvs(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}

6
internal/services/gateway/action/run.go
View File

@ -321,7 +321,9 @@ type CreateRunRequest struct {
// commit compare link
CompareLink string
// fields only used with user direct runs
UserRunRepoUUID string
Variables map[string]string
}
func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) error {
@ -391,13 +393,15 @@ func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) e
env["AGOLA_SKIPSSHHOSTKEYCHECK"] = "1"
}
variables := map[string]string{}
var variables map[string]string
if req.RunType == types.RunTypeProject {
var err error
variables, err = h.genRunVariables(ctx, req)
if err != nil {
return err
}
} else {
variables = req.Variables
}
annotations := map[string]string{

2
internal/services/gateway/action/user.go
View File

@ -824,6 +824,7 @@ type UserCreateRunRequest struct {
Message string
PullRequestRefRegexes []string
Variables map[string]string
}
func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunRequest) error {
@ -941,6 +942,7 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
PullRequestLink: "",
UserRunRepoUUID: req.RepoUUID,
Variables: req.Variables,
}
return h.CreateRuns(ctx, creq)

1
internal/services/gateway/api/user.go
View File

@ -578,6 +578,7 @@ func (h *UserCreateRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
CommitSHA: req.CommitSHA,
Message: req.Message,
PullRequestRefRegexes: req.PullRequestRefRegexes,
Variables: req.Variables,
}
err := h.ah.UserCreateRun(ctx, creq)
if httpError(w, err) {

58
internal/testutil/env.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package testutil
import (
"bufio"
"fmt"
"io"
"strings"
"unicode"
)
func ParseEnv(envvar string) (string, string, error) {
// trim white spaces at the start
envvar = strings.TrimLeftFunc(envvar, unicode.IsSpace)
arr := strings.SplitN(envvar, "=", 2)
if len(arr) == 0 {
return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
}
varname := arr[0]
if varname == "" {
return "", "", fmt.Errorf("invalid environment variable definition: %s", envvar)
}
if len(arr) == 1 {
return varname, "", nil
}
return varname, arr[1], nil
}
func ParseEnvs(r io.Reader) (map[string]string, error) {
envs := map[string]string{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
envname, envvalue, err := ParseEnv(scanner.Text())
if err != nil {
return nil, err
}
envs[envname] = envvalue
}
if err := scanner.Err(); err != nil {
return nil, err
}
return envs, nil
}

3
services/gateway/api/types/user.go
View File

@ -103,5 +103,6 @@ type UserCreateRunRequest struct {
CommitSHA string `json:"commit_sha,omitempty"`
Message string `json:"message,omitempty"`
PullRequestRefRegexes []string
PullRequestRefRegexes []string `json:"pull_request_ref_regexes,omitempty"`
Variables map[string]string `json:"variables,omitempty"`
}

13
services/gateway/client/client.go
View File

@ -410,6 +410,19 @@ func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups,
return getRunsResponse, resp, err
}
func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
q := url.Values{}
q.Add("runID", runID)
q.Add("taskID", taskID)
if setup {
q.Add("setup", "")
} else {
q.Add("step", strconv.Itoa(step))
}
return c.getResponse(ctx, "GET", "/logs", q, nil, nil)
}
func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) {
rs := new(gwapitypes.RemoteSourceResponse)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs)

170
tests/setup_test.go
View File

@ -15,6 +15,7 @@
package tests
import (
"bytes"
"context"
"fmt"
"io/ioutil"
@ -805,3 +806,172 @@ func TestDirectRun(t *testing.T) {
})
}
}
func TestDirectRunVariables(t *testing.T) {
config := `
{
runs: [
{
name: 'run01',
tasks: [
{
name: 'task01',
runtime: {
containers: [
{
image: 'alpine/git',
},
],
},
environment: {
ENV01: { from_variable: 'variable01' },
ENV02: { from_variable: 'variable02' },
},
steps: [
{ type: 'clone' },
{ type: 'run', command: 'env' },
],
},
],
},
],
}
`
varfile01 := `
variable01: "variable value 01"
variable02: variable value 02
`
tests := []struct {
name string
args []string
env map[string]string
}{
{
name: "test direct run without variables",
args: []string{},
env: map[string]string{
"ENV01": "",
"ENV02": "",
},
},
{
name: "test direct run with two variables",
args: []string{"--var", "variable01=VARIABLEVALUE01", "--var", "variable02=VARIABLEVALUE02"},
env: map[string]string{
"ENV01": "VARIABLEVALUE01",
"ENV02": "VARIABLEVALUE02",
},
},
{
name: "test direct run with a var file",
args: []string{"--var-file", "../varfile01.yml"},
env: map[string]string{
"ENV01": "variable value 01",
"ENV02": "variable value 02",
},
},
{
name: "test direct run with a var file and a var that overrides",
args: []string{"--var-file", "../varfile01.yml", "--var", "variable02=VARIABLEVALUE02"},
env: map[string]string{
"ENV01": "variable value 01",
"ENV02": "VARIABLEVALUE02",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
if err := ioutil.WriteFile(filepath.Join(dir, "varfile01.yml"), []byte(varfile01), 0644); err != nil {
t.Fatalf("unexpected err: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
tetcd, tgitea, c := setup(ctx, t, dir)
defer shutdownGitea(tgitea)
defer shutdownEtcd(tetcd)
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("created agola user: %s", user.UserName)
token := createAgolaUserToken(ctx, t, c)
// From now use the user token
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
// TODO(sgotti) add an util to wait for a run phase
time.Sleep(10 * time.Second)
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("runs: %s", util.Dump(runs))
if len(runs) != 1 {
t.Fatalf("expected 1 run got: %d", len(runs))
}
run, _, err := gwClient.GetRun(ctx, runs[0].ID)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if run.Phase != rstypes.RunPhaseFinished {
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
}
if run.Result != rstypes.RunResultSuccess {
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
}
var task *gwapitypes.RunResponseTask
for _, t := range run.Tasks {
if t.Name == "task01" {
task = t
break
}
}
resp, err := gwClient.GetLogs(ctx, run.ID, task.ID, false, 1)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer resp.Body.Close()
logs, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
curEnv, err := testutil.ParseEnvs(bytes.NewReader(logs))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
for n, e := range tt.env {
if ce, ok := curEnv[n]; !ok {
t.Fatalf("missing env var %s", n)
} else {
if ce != e {
t.Fatalf("different env var %s value, want: %q, got %q", n, e, ce)
}
}
}
})
}
}