userdirectrun: allow setting destination branch/tag/ref

Allow setting the destination branch/tag/ref so users can test the run
conditions based on the branch/tag/ref.

To simulate a pull request an user can define a ref that matches one of these
regular expressions: `refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`

.agola/config.jsonnet

@@ -92,7 +92,7 @@ local task_build_docker_tests(version, arch) = {
               |||,
             },
             { type: 'restore_workspace', dest_dir: '.' },
-            { type: 'run', name: 'integration tests', command: 'AGOLA_TOOLBOX_PATH="./bin" GITEA_PATH=${PWD}/bin/gitea DOCKER_BRIDGE_ADDRESS="172.18.0.1" ./bin/integration-tests -test.parallel 1 -test.v' },
+            { type: 'run', name: 'integration tests', command: 'AGOLA_BIN_DIR="./bin" GITEA_PATH=${PWD}/bin/gitea DOCKER_BRIDGE_ADDRESS="172.18.0.1" ./bin/integration-tests -test.parallel 1 -test.v' },
           ],
           depends: [
             'build go 1.12 amd64',

cmd/agola/cmd/directrunstart.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"regexp"
 
 	gitsave "agola.io/agola/internal/git-save"
 	"agola.io/agola/internal/util"
@@ -39,12 +40,13 @@ var cmdDirectRunStart = &cobra.Command{
 }
 
 type directRunStartOptions struct {
-	statusFilter []string
-	labelFilter  []string
-	limit        int
-	start        string
 	untracked bool
 	ignored   bool
+
+	branch       string
+	tag          string
+	ref          string
+	prRefRegexes []string
 }
 
 var directRunStartOpts directRunStartOptions
@@ -52,12 +54,12 @@ var directRunStartOpts directRunStartOptions
 func init() {
 	flags := cmdDirectRunStart.Flags()
 
-	flags.StringSliceVarP(&directRunStartOpts.statusFilter, "status", "s", nil, "filter runs matching the provided status. This option can be repeated multiple times")
-	flags.StringArrayVarP(&directRunStartOpts.labelFilter, "label", "l", nil, "filter runs matching the provided label. This option can be repeated multiple times, in this case only runs matching all the labels will be returned")
-	flags.IntVar(&directRunStartOpts.limit, "limit", 10, "max number of runs to show")
-	flags.StringVar(&directRunStartOpts.start, "start", "", "starting run id (excluded) to fetch")
 	flags.BoolVar(&directRunStartOpts.untracked, "untracked", true, "push untracked files")
 	flags.BoolVar(&directRunStartOpts.ignored, "ignored", false, "push ignored files")
+	flags.StringVar(&directRunStartOpts.branch, "branch", "master", "branch to push to")
+	flags.StringVar(&directRunStartOpts.tag, "tag", "", "tag to push to")
+	flags.StringVar(&directRunStartOpts.ref, "ref", "", `ref to push to (i.e  "refs/heads/master" for a branch, "refs/tags/v1.0" for a tag)`)
+	flags.StringArrayVar(&directRunStartOpts.prRefRegexes, "pull-request-ref-regexes", []string{`refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`}, `regular expression to determine if a ref is a pull request`)
 
 	cmdDirectRun.AddCommand(cmdDirectRunStart)
 }
@@ -65,6 +67,36 @@ func init() {
 func directRunStart(cmd *cobra.Command, args []string) error {
 	gwclient := gwclient.NewClient(gatewayURL, token)
 
+	for _, res := range directRunStartOpts.prRefRegexes {
+		if _, err := regexp.Compile(res); err != nil {
+			return fmt.Errorf("wrong regular expression %q: %v", res, err)
+		}
+	}
+
+	branch := directRunStartOpts.branch
+	tag := directRunStartOpts.tag
+	ref := directRunStartOpts.ref
+
+	set := 0
+
+	flags := cmd.Flags()
+	if flags.Changed("branch") {
+		set++
+	}
+	if tag != "" {
+		set++
+		// unset branch default value
+		branch = ""
+	}
+	if ref != "" {
+		set++
+		// unset branch default value
+		branch = ""
+	}
+	if set > 1 {
+		return fmt.Errorf(`only one of "--branch", "--tag" or "--ref" can be provided`)
+	}
+
 	user, _, err := gwclient.GetCurrentUser(context.TODO())
 	if err != nil {
 		return err
@@ -85,10 +117,10 @@ func directRunStart(cmd *cobra.Command, args []string) error {
 		AddIgnored:   directRunStartOpts.ignored,
 	})
 
-	branch := "gitsavebranch-" + uuid.NewV4().String()
+	localBranch := "gitsavebranch-" + uuid.NewV4().String()
 	message := "agola direct run"
 
-	commitSHA, err := gs.Save(message, branch)
+	commitSHA, err := gs.Save(message, localBranch)
 	if err != nil {
 		return err
 	}
@@ -98,17 +130,30 @@ func directRunStart(cmd *cobra.Command, args []string) error {
 	repoURL := fmt.Sprintf("%s/repos/%s/%s.git", gatewayURL, user.ID, repoUUID)
 
 	// push to a branch with default branch refs "refs/heads/branch"
-	if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/heads/%s", path.Join(gs.RefsPrefix(), branch), branch)); err != nil {
+	if branch != "" {
+		if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/heads/%s", path.Join(gs.RefsPrefix(), localBranch), branch)); err != nil {
 			return err
 		}
+	} else if tag != "" {
+		if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/tags/%s", path.Join(gs.RefsPrefix(), localBranch), tag)); err != nil {
+			return err
+		}
+	} else if ref != "" {
+		if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:%s", path.Join(gs.RefsPrefix(), localBranch), ref)); err != nil {
+			return err
+		}
+	}
 
 	log.Infof("starting direct run")
 	req := &gwapitypes.UserCreateRunRequest{
 		RepoUUID:              repoUUID,
 		RepoPath:              repoPath,
 		Branch:                branch,
+		Tag:                   tag,
+		Ref:                   ref,
 		CommitSHA:             commitSHA,
 		Message:               message,
+		PullRequestRefRegexes: directRunStartOpts.prRefRegexes,
 	}
 	if _, err := gwclient.UserCreateRun(context.TODO(), req); err != nil {
 		return err

internal/gitsources/agolagit/agolagit.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
 	"time"
 
@@ -38,10 +39,11 @@ var (
 type Client struct {
 	url                   string
 	client                *http.Client
+	pullRequestRefRegexes []*regexp.Regexp
 }
 
 // NewClient initializes and returns a API client.
-func New(url string) *Client {
+func New(url string, pullRequestRefRegexes []*regexp.Regexp) *Client {
 	// copied from net/http until it has a clone function: https://github.com/golang/go/issues/26013
 	transport := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
@@ -61,6 +63,7 @@ func New(url string) *Client {
 	return &Client{
 		url:                   strings.TrimSuffix(url, "/"),
 		client:                httpClient,
+		pullRequestRefRegexes: pullRequestRefRegexes,
 	}
 }
 
@@ -168,7 +171,22 @@ func (c *Client) GetRef(repopath, ref string) (*gitsource.Ref, error) {
 }
 
 func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
-	return -1, "", nil
+	if strings.HasPrefix(ref, branchRefPrefix) {
+		return gitsource.RefTypeBranch, strings.TrimPrefix(ref, branchRefPrefix), nil
+	}
+
+	if strings.HasPrefix(ref, tagRefPrefix) {
+		return gitsource.RefTypeTag, strings.TrimPrefix(ref, tagRefPrefix), nil
+	}
+
+	for _, re := range c.pullRequestRefRegexes {
+		if re.MatchString(ref) {
+			m := re.FindStringSubmatch(ref)
+			return gitsource.RefTypePullRequest, m[1], nil
+		}
+	}
+
+	return -1, "", fmt.Errorf("unsupported ref: %s", ref)
 }
 
 func (c *Client) GetCommit(repopath, commitSHA string) (*gitsource.Commit, error) {

internal/gitsources/gitea/gitea.go

@@ -442,7 +442,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
 
 	case pullRequestRefRegex.MatchString(ref):
 		m := pullRequestRefRegex.FindStringSubmatch(ref)
-		return gitsource.RefTypePullRequest, m[0], nil
+		return gitsource.RefTypePullRequest, m[1], nil
 
 	default:
 		return -1, "", fmt.Errorf("unsupported ref: %s", ref)

internal/gitsources/github/github.go

@@ -437,7 +437,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
 
 	case pullRequestRefRegex.MatchString(ref):
 		m := pullRequestRefRegex.FindStringSubmatch(ref)
-		return gitsource.RefTypePullRequest, m[0], nil
+		return gitsource.RefTypePullRequest, m[1], nil
 
 	default:
 		return -1, "", fmt.Errorf("unsupported ref: %s", ref)

internal/gitsources/gitlab/gitlab.go

@@ -336,7 +336,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
 
 	case pullRequestRefRegex.MatchString(ref):
 		m := pullRequestRefRegex.FindStringSubmatch(ref)
-		return gitsource.RefTypePullRequest, m[0], nil
+		return gitsource.RefTypePullRequest, m[1], nil
 
 	default:
 		return -1, "", fmt.Errorf("unsupported ref: %s", ref)

internal/services/gateway/action/project.go

@@ -528,14 +528,12 @@ func (h *ActionHandler) ProjectCreateRun(ctx context.Context, projectRef, branch
 		refType = types.RunRefTypeBranch
 		message = commit.Message
 		branchLink = gitSource.BranchLink(repoInfo, branch)
-
 	}
 
 	if tag != "" {
-		refType = types.RunRefTypeBranch
+		refType = types.RunRefTypeTag
 		message = fmt.Sprintf("Tag %s", tag)
 		tagLink = gitSource.TagLink(repoInfo, tag)
-
 	}
 
 	// use remotesource skipSSHHostKeyCheck config and override with project config if set to true there

internal/services/gateway/action/user.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
@@ -817,11 +818,24 @@ type UserCreateRunRequest struct {
 	RepoUUID  string
 	RepoPath  string
 	Branch    string
+	Tag       string
+	Ref       string
 	CommitSHA string
 	Message   string
+
+	PullRequestRefRegexes []string
 }
 
 func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunRequest) error {
+	prRefRegexes := []*regexp.Regexp{}
+	for _, res := range req.PullRequestRefRegexes {
+		re, err := regexp.Compile(res)
+		if err != nil {
+			return fmt.Errorf("wrong regular expression %q: %v", res, err)
+		}
+		prRefRegexes = append(prRefRegexes, re)
+	}
+
 	curUserID := h.CurrentUserID(ctx)
 
 	user, resp, err := h.configstoreClient.GetUser(ctx, curUserID)
@@ -841,12 +855,72 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
 		return util.NewErrUnauthorized(errors.Errorf("repo %q not owned", req.RepoPath))
 	}
 
-	gitSource := agolagit.New(h.apiExposedURL + "/repos")
+	branch := req.Branch
+	tag := req.Tag
+	ref := req.Ref
+
+	set := 0
+	if branch != "" {
+		set++
+	}
+	if tag != "" {
+		set++
+	}
+	if ref != "" {
+		set++
+	}
+	if set == 0 {
+		return util.NewErrBadRequest(errors.Errorf("one of branch, tag or ref is required"))
+	}
+	if set > 1 {
+		return util.NewErrBadRequest(errors.Errorf("only one of branch, tag or ref can be provided"))
+	}
+
+	gitSource := agolagit.New(h.apiExposedURL+"/repos", prRefRegexes)
 	cloneURL := fmt.Sprintf("%s/%s.git", h.apiExposedURL+"/repos", req.RepoPath)
 
+	if ref == "" {
+		if branch != "" {
+			ref = gitSource.BranchRef(branch)
+		}
+		if tag != "" {
+			ref = gitSource.TagRef(tag)
+		}
+	}
+
+	gitRefType, name, err := gitSource.RefType(ref)
+	if err != nil {
+		return util.NewErrBadRequest(errors.Errorf("failed to get refType for ref %q: %w", ref, err))
+	}
+
+	var pullRequestID string
+
+	switch gitRefType {
+	case gitsource.RefTypeBranch:
+		branch = name
+	case gitsource.RefTypeTag:
+		tag = name
+	case gitsource.RefTypePullRequest:
+		pullRequestID = name
+	default:
+		return errors.Errorf("unsupported ref %q for manual run creation", ref)
+	}
+
+	var refType types.RunRefType
+
+	if branch != "" {
+		refType = types.RunRefTypeBranch
+	}
+	if tag != "" {
+		refType = types.RunRefTypeTag
+	}
+	if pullRequestID != "" {
+		refType = types.RunRefTypePullRequest
+	}
+
 	creq := &CreateRunRequest{
 		RunType:            types.RunTypeUser,
-		RefType:            types.RunRefTypeBranch,
+		RefType:            refType,
 		RunCreationTrigger: types.RunCreationTriggerTypeManual,
 
 		Project:       nil,
@@ -855,10 +929,10 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
 		GitSource:     gitSource,
 		CommitSHA:     req.CommitSHA,
 		Message:       req.Message,
-		Branch:        req.Branch,
+		Branch:        branch,
-		Tag:           "",
+		Tag:           tag,
-		PullRequestID: "",
+		Ref:           ref,
-		Ref:           gitSource.BranchRef(req.Branch),
+		PullRequestID: pullRequestID,
 		CloneURL:      cloneURL,
 
 		CommitLink:      "",

internal/services/gateway/api/user.go

@@ -573,8 +573,11 @@ func (h *UserCreateRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 		RepoUUID:              req.RepoUUID,
 		RepoPath:              req.RepoPath,
 		Branch:                req.Branch,
+		Tag:                   req.Tag,
+		Ref:                   req.Ref,
 		CommitSHA:             req.CommitSHA,
 		Message:               req.Message,
+		PullRequestRefRegexes: req.PullRequestRefRegexes,
 	}
 	err := h.ah.UserCreateRun(ctx, creq)
 	if httpError(w, err) {

services/gateway/api/types/user.go

@@ -98,6 +98,10 @@ type UserCreateRunRequest struct {
 	RepoUUID  string `json:"repo_uuid,omitempty"`
 	RepoPath  string `json:"repo_path,omitempty"`
 	Branch    string `json:"branch,omitempty"`
+	Tag       string `json:"tag,omitempty"`
+	Ref       string `json:"ref,omitempty"`
 	CommitSHA string `json:"commit_sha,omitempty"`
 	Message   string `json:"message,omitempty"`
+
+	PullRequestRefRegexes []string
 }

tests/setup_test.go

@@ -46,10 +46,13 @@ import (
 	"go.uber.org/zap/zapcore"
 	errors "golang.org/x/xerrors"
 	"gopkg.in/src-d/go-billy.v4/memfs"
+	"gopkg.in/src-d/go-billy.v4/osfs"
 	"gopkg.in/src-d/go-git.v4"
 	gitconfig "gopkg.in/src-d/go-git.v4/config"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
@@ -160,9 +163,9 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
 	if dockerBridgeAddress == "" {
 		dockerBridgeAddress = "172.17.0.1"
 	}
-	toolboxPath := os.Getenv("AGOLA_TOOLBOX_PATH")
+	agolaBinDir := os.Getenv("AGOLA_BIN_DIR")
-	if toolboxPath == "" {
+	if agolaBinDir == "" {
-		t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined")
+		t.Fatalf("env var AGOLA_BIN_DIR is undefined")
 	}
 
 	c := &config.Config{
@@ -218,7 +221,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
 			Debug:         false,
 			DataDir:       filepath.Join(dir, "executor"),
 			RunserviceURL: "",
-			ToolboxPath:   toolboxPath,
+			ToolboxPath:   agolaBinDir,
 			Web: config.Web{
 				ListenAddress: ":4001",
 				TLS:           false,
@@ -342,6 +345,17 @@ func TestCreateLinkedAccount(t *testing.T) {
 	createLinkedAccount(ctx, t, tgitea, c)
 }
 
+func createAgolaUserToken(ctx context.Context, t *testing.T, c *config.Config) string {
+	gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
+	token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapitypes.CreateUserTokenRequest{TokenName: "token01"})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	t.Logf("created agola user token: %s", token.Token)
+
+	return token.Token
+}
+
 func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.TestGitea, c *config.Config) (string, string) {
 	giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.HTTPListenAddress, tgitea.HTTPPort)
 	giteaClient := gitea.NewClient(giteaAPIURL, "")
@@ -359,11 +373,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
 	}
 	t.Logf("created agola user: %s", user.UserName)
 
-	token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapitypes.CreateUserTokenRequest{TokenName: "token01"})
+	token := createAgolaUserToken(ctx, t, c)
-	if err != nil {
-		t.Fatalf("unexpected err: %v", err)
-	}
-	t.Logf("created agola user token: %s", token.Token)
 
 	rs, _, err := gwClient.CreateRemoteSource(ctx, &gwapitypes.CreateRemoteSourceRequest{
 		Name:                "gitea",
@@ -378,7 +388,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
 	t.Logf("created agola remote source: %s", rs.Name)
 
 	// From now use the user token
-	gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token.Token)
+	gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
 
 	la, _, err := gwClient.CreateUserLA(ctx, agolaUser01, &gwapitypes.CreateUserLARequest{
 		RemoteSourceName:          "gitea",
@@ -390,7 +400,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
 	}
 	t.Logf("created user linked account: %s", util.Dump(la))
 
-	return giteaToken.Token, token.Token
+	return giteaToken.Token, token
 }
 
 func TestCreateProject(t *testing.T) {
@@ -560,3 +570,172 @@ func TestRun(t *testing.T) {
 		t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
 	}
 }
+
+func directRun(t *testing.T, dir, config, gatewayURL, token string, args ...string) {
+	agolaBinDir := os.Getenv("AGOLA_BIN_DIR")
+	if agolaBinDir == "" {
+		t.Fatalf("env var AGOLA_BIN_DIR is undefined")
+	}
+	agolaBinDir, err := filepath.Abs(agolaBinDir)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	repoDir, err := ioutil.TempDir(dir, "repo")
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	gitfs := osfs.New(repoDir)
+	dot, _ := gitfs.Chroot(".git")
+
+	f, err := gitfs.Create(".agola/config.jsonnet")
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	if _, err = f.Write([]byte(config)); err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	_, err = git.Init(filesystem.NewStorage(dot, cache.NewObjectLRUDefault()), gitfs)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	args = append([]string{"--gateway-url", gatewayURL, "--token", token, "directrun", "start", "--untracked", "false"}, args...)
+	cmd := exec.Command(filepath.Join(agolaBinDir, "agola"), args...)
+	cmd.Dir = repoDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("unexpected err: %v, out: %s", err, out)
+	}
+	t.Logf("directrun start out: %s", out)
+}
+
+func TestDirectRun(t *testing.T) {
+	config := `
+      {
+        runs: [
+          {
+            name: 'run01',
+            tasks: [
+              {
+                name: 'task01',
+                runtime: {
+                  containers: [
+                    {
+                      image: 'alpine/git',
+                    },
+                  ],
+                },
+                steps: [
+                  { type: 'clone' },
+                  { type: 'run', command: 'env' },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `
+
+	tests := []struct {
+		name        string
+		args        []string
+		annotations map[string]string
+	}{
+		{
+			name: "test direct run",
+			annotations: map[string]string{
+				"branch":   "master",
+				"ref":      "refs/heads/master",
+				"ref_type": "branch",
+			},
+		},
+		{
+			name: "test direct run with destination branch",
+			args: []string{"--branch", "develop"},
+			annotations: map[string]string{
+				"branch":   "develop",
+				"ref":      "refs/heads/develop",
+				"ref_type": "branch",
+			},
+		},
+		{
+			name: "test direct run with destination tag",
+			args: []string{"--tag", "v0.1.0"},
+			annotations: map[string]string{
+				"tag":      "v0.1.0",
+				"ref":      "refs/tags/v0.1.0",
+				"ref_type": "tag",
+			},
+		},
+		{
+			name: "test direct run with destination ref as a pr",
+			args: []string{"--ref", "refs/pull/1/head"},
+			annotations: map[string]string{
+				"pull_request_id": "1",
+				"ref":             "refs/pull/1/head",
+				"ref_type":        "pull_request",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir, err := ioutil.TempDir("", "agola")
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			defer os.RemoveAll(dir)
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			tetcd, tgitea, c := setup(ctx, t, dir)
+			defer shutdownGitea(tgitea)
+			defer shutdownEtcd(tetcd)
+
+			gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
+			user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			t.Logf("created agola user: %s", user.UserName)
+
+			token := createAgolaUserToken(ctx, t, c)
+
+			// From now use the user token
+			gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
+
+			directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
+
+			// TODO(sgotti) add an util to wait for a run phase
+			time.Sleep(10 * time.Second)
+
+			runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			t.Logf("runs: %s", util.Dump(runs))
+
+			if len(runs) != 1 {
+				t.Fatalf("expected 1 run got: %d", len(runs))
+			}
+
+			run := runs[0]
+			if run.Phase != rstypes.RunPhaseFinished {
+				t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
+			}
+			if run.Result != rstypes.RunResultSuccess {
+				t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
+			}
+			for k, v := range tt.annotations {
+				if run.Annotations[k] != v {
+					t.Fatalf("expected run annotation %q value %q, got %q", k, v, run.Annotations[k])
+				}
+			}
+		})
+	}
+}