a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #83 from sgotti/user_direct_run_custom_branch_tag_ref 

userdirectrun: allow setting destination branch/tag/ref
This commit is contained in:
Simone Gotti 2019-08-05 16:06:44 +02:00 committed by GitHub
parent 0da52e2d8a 4ec0b33eb4
commit 4fe4631729
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 375 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.agola/config.jsonnet
View File

@ -92,7 +92,7 @@ local task_build_docker_tests(version, arch) = {
|||,
},
{ type: 'restore_workspace', dest_dir: '.' },
{ type: 'run', name: 'integration tests', command: 'AGOLA_TOOLBOX_PATH="./bin" GITEA_PATH=${PWD}/bin/gitea DOCKER_BRIDGE_ADDRESS="172.18.0.1" ./bin/integration-tests -test.parallel 1 -test.v' },
{ type: 'run', name: 'integration tests', command: 'AGOLA_BIN_DIR="./bin" GITEA_PATH=${PWD}/bin/gitea DOCKER_BRIDGE_ADDRESS="172.18.0.1" ./bin/integration-tests -test.parallel 1 -test.v' },
],
depends: [
'build go 1.12 amd64',

83
cmd/agola/cmd/directrunstart.go
View File

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"path"
"regexp"
gitsave "agola.io/agola/internal/git-save"
"agola.io/agola/internal/util"
@ -39,12 +40,13 @@ var cmdDirectRunStart = &cobra.Command{
}
type directRunStartOptions struct {
statusFilter []string
labelFilter []string
limit int
start string
untracked bool
ignored bool
untracked bool
ignored bool
branch string
tag string
ref string
prRefRegexes []string
}
var directRunStartOpts directRunStartOptions
@ -52,12 +54,12 @@ var directRunStartOpts directRunStartOptions
func init() {
flags := cmdDirectRunStart.Flags()
flags.StringSliceVarP(&directRunStartOpts.statusFilter, "status", "s", nil, "filter runs matching the provided status. This option can be repeated multiple times")
flags.StringArrayVarP(&directRunStartOpts.labelFilter, "label", "l", nil, "filter runs matching the provided label. This option can be repeated multiple times, in this case only runs matching all the labels will be returned")
flags.IntVar(&directRunStartOpts.limit, "limit", 10, "max number of runs to show")
flags.StringVar(&directRunStartOpts.start, "start", "", "starting run id (excluded) to fetch")
flags.BoolVar(&directRunStartOpts.untracked, "untracked", true, "push untracked files")
flags.BoolVar(&directRunStartOpts.ignored, "ignored", false, "push ignored files")
flags.StringVar(&directRunStartOpts.branch, "branch", "master", "branch to push to")
flags.StringVar(&directRunStartOpts.tag, "tag", "", "tag to push to")
flags.StringVar(&directRunStartOpts.ref, "ref", "", `ref to push to (i.e "refs/heads/master" for a branch, "refs/tags/v1.0" for a tag)`)
flags.StringArrayVar(&directRunStartOpts.prRefRegexes, "pull-request-ref-regexes", []string{`refs/pull/(\d+)/head`, `refs/merge-requests/(\d+)/head`}, `regular expression to determine if a ref is a pull request`)
cmdDirectRun.AddCommand(cmdDirectRunStart)
}
@ -65,6 +67,36 @@ func init() {
func directRunStart(cmd *cobra.Command, args []string) error {
gwclient := gwclient.NewClient(gatewayURL, token)
for _, res := range directRunStartOpts.prRefRegexes {
if _, err := regexp.Compile(res); err != nil {
return fmt.Errorf("wrong regular expression %q: %v", res, err)
}
}
branch := directRunStartOpts.branch
tag := directRunStartOpts.tag
ref := directRunStartOpts.ref
set := 0
flags := cmd.Flags()
if flags.Changed("branch") {
set++
}
if tag != "" {
set++
// unset branch default value
branch = ""
}
if ref != "" {
set++
// unset branch default value
branch = ""
}
if set > 1 {
return fmt.Errorf(`only one of "--branch", "--tag" or "--ref" can be provided`)
}
user, _, err := gwclient.GetCurrentUser(context.TODO())
if err != nil {
return err
@ -85,10 +117,10 @@ func directRunStart(cmd *cobra.Command, args []string) error {
AddIgnored: directRunStartOpts.ignored,
})
branch := "gitsavebranch-" + uuid.NewV4().String()
localBranch := "gitsavebranch-" + uuid.NewV4().String()
message := "agola direct run"
commitSHA, err := gs.Save(message, branch)
commitSHA, err := gs.Save(message, localBranch)
if err != nil {
return err
}
@ -98,17 +130,30 @@ func directRunStart(cmd *cobra.Command, args []string) error {
repoURL := fmt.Sprintf("%s/repos/%s/%s.git", gatewayURL, user.ID, repoUUID)
// push to a branch with default branch refs "refs/heads/branch"
if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/heads/%s", path.Join(gs.RefsPrefix(), branch), branch)); err != nil {
return err
if branch != "" {
if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/heads/%s", path.Join(gs.RefsPrefix(), localBranch), branch)); err != nil {
return err
}
} else if tag != "" {
if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:refs/tags/%s", path.Join(gs.RefsPrefix(), localBranch), tag)); err != nil {
return err
}
} else if ref != "" {
if err := gitsave.GitPush("", repoURL, fmt.Sprintf("%s:%s", path.Join(gs.RefsPrefix(), localBranch), ref)); err != nil {
return err
}
}
log.Infof("starting direct run")
req := &gwapitypes.UserCreateRunRequest{
RepoUUID: repoUUID,
RepoPath: repoPath,
Branch: branch,
CommitSHA: commitSHA,
Message: message,
RepoUUID: repoUUID,
RepoPath: repoPath,
Branch: branch,
Tag: tag,
Ref: ref,
CommitSHA: commitSHA,
Message: message,
PullRequestRefRegexes: directRunStartOpts.prRefRegexes,
}
if _, err := gwclient.UserCreateRun(context.TODO(), req); err != nil {
return err

30
internal/gitsources/agolagit/agolagit.go
View File

@ -22,6 +22,7 @@ import (
"net"
"net/http"
"net/url"
"regexp"
"strings"
"time"
@ -36,12 +37,13 @@ var (
)
type Client struct {
url string
client *http.Client
url string
client *http.Client
pullRequestRefRegexes []*regexp.Regexp
}
// NewClient initializes and returns a API client.
func New(url string) *Client {
func New(url string, pullRequestRefRegexes []*regexp.Regexp) *Client {
// copied from net/http until it has a clone function: https://github.com/golang/go/issues/26013
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
@ -59,8 +61,9 @@ func New(url string) *Client {
httpClient := &http.Client{Transport: transport}
return &Client{
url: strings.TrimSuffix(url, "/"),
client: httpClient,
url: strings.TrimSuffix(url, "/"),
client: httpClient,
pullRequestRefRegexes: pullRequestRefRegexes,
}
}
@ -168,7 +171,22 @@ func (c *Client) GetRef(repopath, ref string) (*gitsource.Ref, error) {
}
func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
return -1, "", nil
if strings.HasPrefix(ref, branchRefPrefix) {
return gitsource.RefTypeBranch, strings.TrimPrefix(ref, branchRefPrefix), nil
}
if strings.HasPrefix(ref, tagRefPrefix) {
return gitsource.RefTypeTag, strings.TrimPrefix(ref, tagRefPrefix), nil
}
for _, re := range c.pullRequestRefRegexes {
if re.MatchString(ref) {
m := re.FindStringSubmatch(ref)
return gitsource.RefTypePullRequest, m[1], nil
}
}
return -1, "", fmt.Errorf("unsupported ref: %s", ref)
}
func (c *Client) GetCommit(repopath, commitSHA string) (*gitsource.Commit, error) {

2
internal/gitsources/gitea/gitea.go
View File

@ -442,7 +442,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
case pullRequestRefRegex.MatchString(ref):
m := pullRequestRefRegex.FindStringSubmatch(ref)
return gitsource.RefTypePullRequest, m[0], nil
return gitsource.RefTypePullRequest, m[1], nil
default:
return -1, "", fmt.Errorf("unsupported ref: %s", ref)

2
internal/gitsources/github/github.go
View File

@ -437,7 +437,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
case pullRequestRefRegex.MatchString(ref):
m := pullRequestRefRegex.FindStringSubmatch(ref)
return gitsource.RefTypePullRequest, m[0], nil
return gitsource.RefTypePullRequest, m[1], nil
default:
return -1, "", fmt.Errorf("unsupported ref: %s", ref)

2
internal/gitsources/gitlab/gitlab.go
View File

@ -336,7 +336,7 @@ func (c *Client) RefType(ref string) (gitsource.RefType, string, error) {
case pullRequestRefRegex.MatchString(ref):
m := pullRequestRefRegex.FindStringSubmatch(ref)
return gitsource.RefTypePullRequest, m[0], nil
return gitsource.RefTypePullRequest, m[1], nil
default:
return -1, "", fmt.Errorf("unsupported ref: %s", ref)

4
internal/services/gateway/action/project.go
View File

@ -528,14 +528,12 @@ func (h *ActionHandler) ProjectCreateRun(ctx context.Context, projectRef, branch
refType = types.RunRefTypeBranch
message = commit.Message
branchLink = gitSource.BranchLink(repoInfo, branch)
}
if tag != "" {
refType = types.RunRefTypeBranch
refType = types.RunRefTypeTag
message = fmt.Sprintf("Tag %s", tag)
tagLink = gitSource.TagLink(repoInfo, tag)
}
// use remotesource skipSSHHostKeyCheck config and override with project config if set to true there

86
internal/services/gateway/action/user.go
View File

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
@ -817,11 +818,24 @@ type UserCreateRunRequest struct {
RepoUUID string
RepoPath string
Branch string
Tag string
Ref string
CommitSHA string
Message string
PullRequestRefRegexes []string
}
func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunRequest) error {
prRefRegexes := []*regexp.Regexp{}
for _, res := range req.PullRequestRefRegexes {
re, err := regexp.Compile(res)
if err != nil {
return fmt.Errorf("wrong regular expression %q: %v", res, err)
}
prRefRegexes = append(prRefRegexes, re)
}
curUserID := h.CurrentUserID(ctx)
user, resp, err := h.configstoreClient.GetUser(ctx, curUserID)
@ -841,12 +855,72 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
return util.NewErrUnauthorized(errors.Errorf("repo %q not owned", req.RepoPath))
}
gitSource := agolagit.New(h.apiExposedURL + "/repos")
branch := req.Branch
tag := req.Tag
ref := req.Ref
set := 0
if branch != "" {
set++
}
if tag != "" {
set++
}
if ref != "" {
set++
}
if set == 0 {
return util.NewErrBadRequest(errors.Errorf("one of branch, tag or ref is required"))
}
if set > 1 {
return util.NewErrBadRequest(errors.Errorf("only one of branch, tag or ref can be provided"))
}
gitSource := agolagit.New(h.apiExposedURL+"/repos", prRefRegexes)
cloneURL := fmt.Sprintf("%s/%s.git", h.apiExposedURL+"/repos", req.RepoPath)
if ref == "" {
if branch != "" {
ref = gitSource.BranchRef(branch)
}
if tag != "" {
ref = gitSource.TagRef(tag)
}
}
gitRefType, name, err := gitSource.RefType(ref)
if err != nil {
return util.NewErrBadRequest(errors.Errorf("failed to get refType for ref %q: %w", ref, err))
}
var pullRequestID string
switch gitRefType {
case gitsource.RefTypeBranch:
branch = name
case gitsource.RefTypeTag:
tag = name
case gitsource.RefTypePullRequest:
pullRequestID = name
default:
return errors.Errorf("unsupported ref %q for manual run creation", ref)
}
var refType types.RunRefType
if branch != "" {
refType = types.RunRefTypeBranch
}
if tag != "" {
refType = types.RunRefTypeTag
}
if pullRequestID != "" {
refType = types.RunRefTypePullRequest
}
creq := &CreateRunRequest{
RunType: types.RunTypeUser,
RefType: types.RunRefTypeBranch,
RefType: refType,
RunCreationTrigger: types.RunCreationTriggerTypeManual,
Project: nil,
@ -855,10 +929,10 @@ func (h *ActionHandler) UserCreateRun(ctx context.Context, req *UserCreateRunReq
GitSource: gitSource,
CommitSHA: req.CommitSHA,
Message: req.Message,
Branch: req.Branch,
Tag: "",
PullRequestID: "",
Ref: gitSource.BranchRef(req.Branch),
Branch: branch,
Tag: tag,
Ref: ref,
PullRequestID: pullRequestID,
CloneURL: cloneURL,
CommitLink: "",

13
internal/services/gateway/api/user.go
View File

@ -570,11 +570,14 @@ func (h *UserCreateRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
}
creq := &action.UserCreateRunRequest{
RepoUUID: req.RepoUUID,
RepoPath: req.RepoPath,
Branch: req.Branch,
CommitSHA: req.CommitSHA,
Message: req.Message,
RepoUUID: req.RepoUUID,
RepoPath: req.RepoPath,
Branch: req.Branch,
Tag: req.Tag,
Ref: req.Ref,
CommitSHA: req.CommitSHA,
Message: req.Message,
PullRequestRefRegexes: req.PullRequestRefRegexes,
}
err := h.ah.UserCreateRun(ctx, creq)
if httpError(w, err) {

4
services/gateway/api/types/user.go
View File

@ -98,6 +98,10 @@ type UserCreateRunRequest struct {
RepoUUID string `json:"repo_uuid,omitempty"`
RepoPath string `json:"repo_path,omitempty"`
Branch string `json:"branch,omitempty"`
Tag string `json:"tag,omitempty"`
Ref string `json:"ref,omitempty"`
CommitSHA string `json:"commit_sha,omitempty"`
Message string `json:"message,omitempty"`
PullRequestRefRegexes []string
}

201
tests/setup_test.go
View File

@ -46,10 +46,13 @@ import (
"go.uber.org/zap/zapcore"
errors "golang.org/x/xerrors"
"gopkg.in/src-d/go-billy.v4/memfs"
"gopkg.in/src-d/go-billy.v4/osfs"
"gopkg.in/src-d/go-git.v4"
gitconfig "gopkg.in/src-d/go-git.v4/config"
"gopkg.in/src-d/go-git.v4/plumbing/cache"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
"gopkg.in/src-d/go-git.v4/storage/filesystem"
"gopkg.in/src-d/go-git.v4/storage/memory"
)
@ -160,9 +163,9 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
if dockerBridgeAddress == "" {
dockerBridgeAddress = "172.17.0.1"
}
toolboxPath := os.Getenv("AGOLA_TOOLBOX_PATH")
if toolboxPath == "" {
t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined")
agolaBinDir := os.Getenv("AGOLA_BIN_DIR")
if agolaBinDir == "" {
t.Fatalf("env var AGOLA_BIN_DIR is undefined")
}
c := &config.Config{
@ -218,7 +221,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
Debug: false,
DataDir: filepath.Join(dir, "executor"),
RunserviceURL: "",
ToolboxPath: toolboxPath,
ToolboxPath: agolaBinDir,
Web: config.Web{
ListenAddress: ":4001",
TLS: false,
@ -342,6 +345,17 @@ func TestCreateLinkedAccount(t *testing.T) {
createLinkedAccount(ctx, t, tgitea, c)
}
func createAgolaUserToken(ctx context.Context, t *testing.T, c *config.Config) string {
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapitypes.CreateUserTokenRequest{TokenName: "token01"})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("created agola user token: %s", token.Token)
return token.Token
}
func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.TestGitea, c *config.Config) (string, string) {
giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.HTTPListenAddress, tgitea.HTTPPort)
giteaClient := gitea.NewClient(giteaAPIURL, "")
@ -359,11 +373,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
}
t.Logf("created agola user: %s", user.UserName)
token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapitypes.CreateUserTokenRequest{TokenName: "token01"})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("created agola user token: %s", token.Token)
token := createAgolaUserToken(ctx, t, c)
rs, _, err := gwClient.CreateRemoteSource(ctx, &gwapitypes.CreateRemoteSourceRequest{
Name: "gitea",
@ -378,7 +388,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
t.Logf("created agola remote source: %s", rs.Name)
// From now use the user token
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token.Token)
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
la, _, err := gwClient.CreateUserLA(ctx, agolaUser01, &gwapitypes.CreateUserLARequest{
RemoteSourceName: "gitea",
@ -390,7 +400,7 @@ func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.Tes
}
t.Logf("created user linked account: %s", util.Dump(la))
return giteaToken.Token, token.Token
return giteaToken.Token, token
}
func TestCreateProject(t *testing.T) {
@ -560,3 +570,172 @@ func TestRun(t *testing.T) {
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
}
}
func directRun(t *testing.T, dir, config, gatewayURL, token string, args ...string) {
agolaBinDir := os.Getenv("AGOLA_BIN_DIR")
if agolaBinDir == "" {
t.Fatalf("env var AGOLA_BIN_DIR is undefined")
}
agolaBinDir, err := filepath.Abs(agolaBinDir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
repoDir, err := ioutil.TempDir(dir, "repo")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
gitfs := osfs.New(repoDir)
dot, _ := gitfs.Chroot(".git")
f, err := gitfs.Create(".agola/config.jsonnet")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if _, err = f.Write([]byte(config)); err != nil {
t.Fatalf("unexpected err: %v", err)
}
_, err = git.Init(filesystem.NewStorage(dot, cache.NewObjectLRUDefault()), gitfs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
args = append([]string{"--gateway-url", gatewayURL, "--token", token, "directrun", "start", "--untracked", "false"}, args...)
cmd := exec.Command(filepath.Join(agolaBinDir, "agola"), args...)
cmd.Dir = repoDir
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("unexpected err: %v, out: %s", err, out)
}
t.Logf("directrun start out: %s", out)
}
func TestDirectRun(t *testing.T) {
config := `
{
runs: [
{
name: 'run01',
tasks: [
{
name: 'task01',
runtime: {
containers: [
{
image: 'alpine/git',
},
],
},
steps: [
{ type: 'clone' },
{ type: 'run', command: 'env' },
],
},
],
},
],
}
`
tests := []struct {
name string
args []string
annotations map[string]string
}{
{
name: "test direct run",
annotations: map[string]string{
"branch": "master",
"ref": "refs/heads/master",
"ref_type": "branch",
},
},
{
name: "test direct run with destination branch",
args: []string{"--branch", "develop"},
annotations: map[string]string{
"branch": "develop",
"ref": "refs/heads/develop",
"ref_type": "branch",
},
},
{
name: "test direct run with destination tag",
args: []string{"--tag", "v0.1.0"},
annotations: map[string]string{
"tag": "v0.1.0",
"ref": "refs/tags/v0.1.0",
"ref_type": "tag",
},
},
{
name: "test direct run with destination ref as a pr",
args: []string{"--ref", "refs/pull/1/head"},
annotations: map[string]string{
"pull_request_id": "1",
"ref": "refs/pull/1/head",
"ref_type": "pull_request",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
tetcd, tgitea, c := setup(ctx, t, dir)
defer shutdownGitea(tgitea)
defer shutdownEtcd(tetcd)
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("created agola user: %s", user.UserName)
token := createAgolaUserToken(ctx, t, c)
// From now use the user token
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
// TODO(sgotti) add an util to wait for a run phase
time.Sleep(10 * time.Second)
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("runs: %s", util.Dump(runs))
if len(runs) != 1 {
t.Fatalf("expected 1 run got: %d", len(runs))
}
run := runs[0]
if run.Phase != rstypes.RunPhaseFinished {
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
}
if run.Result != rstypes.RunResultSuccess {
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
}
for k, v := range tt.annotations {
if run.Annotations[k] != v {
t.Fatalf("expected run annotation %q value %q, got %q", k, v, run.Annotations[k])
}
}
})
}
}