Add user local runs feature

cmd/agola/cmd/localrunstart.go

@@ -0,0 +1,76 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	gitsave "github.com/sorintlab/agola/internal/git-save"
+
+	uuid "github.com/satori/go.uuid"
+	"github.com/spf13/cobra"
+)
+
+var cmdLocalRunStart = &cobra.Command{
+	Use: "start",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := localRunStart(cmd, args); err != nil {
+			log.Fatalf("err: %v", err)
+		}
+	},
+	Short: "executes a run from a local repository",
+}
+
+type localRunStartOptions struct {
+	statusFilter []string
+	labelFilter  []string
+	limit        int
+	start        string
+	untracked    bool
+	ignored      bool
+}
+
+var localRunStartOpts localRunStartOptions
+
+func init() {
+	flags := cmdLocalRunStart.PersistentFlags()
+
+	flags.StringSliceVarP(&localRunStartOpts.statusFilter, "status", "s", nil, "filter runs matching the provided status. This option can be repeated multiple times")
+	flags.StringArrayVarP(&localRunStartOpts.labelFilter, "label", "l", nil, "filter runs matching the provided label. This option can be repeated multiple times, in this case only runs matching all the labels will be returned")
+	flags.IntVar(&localRunStartOpts.limit, "limit", 10, "max number of runs to show")
+	flags.StringVar(&localRunStartOpts.start, "start", "", "starting run id (excluded) to fetch")
+	flags.BoolVar(&localRunStartOpts.untracked, "untracked", true, "push untracked files")
+	flags.BoolVar(&localRunStartOpts.ignored, "ignored", false, "push ignored files")
+
+	cmdLocalRun.AddCommand(cmdLocalRunStart)
+}
+
+func localRunStart(cmd *cobra.Command, args []string) error {
+	gs := gitsave.NewGitSave(logger, &gitsave.GitSaveConfig{
+		AddUntracked: localRunStartOpts.untracked,
+		AddIgnored:   localRunStartOpts.ignored,
+	})
+
+	branch := "gitsavebranch-" + uuid.NewV4().String()
+
+	if err := gs.Save("agola local run", branch); err != nil {
+		log.Fatalf("err: %v", err)
+	}
+
+	log.Infof("pushing branch")
+	if err := gitsave.GitPush("", "http://172.17.0.1:8000/repos/sgotti/test02.git", "refs/gitsave/"+branch); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/agola/cmd/locarun.go

@@ -0,0 +1,28 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var cmdLocalRun = &cobra.Command{
+	Use:   "localrun",
+	Short: "localrun",
+}
+
+func init() {
+	cmdAgola.AddCommand(cmdLocalRun)
+}

internal/git-save/save.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
+	uuid "github.com/satori/go.uuid"
 	"github.com/sorintlab/agola/internal/util"
 
 	"github.com/pkg/errors"
@@ -112,6 +113,12 @@ func gitUpdateRef(message, ref, commitSHA string) error {
 	return err
 }
 
+func gitUpdateFiles(indexPath string) error {
+	git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}}
+	_, err := git.Output(context.Background(), nil, "add", "-u")
+	return err
+}
+
 func gitAddUntrackedFiles(indexPath string) error {
 	git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}}
 	_, err := git.Output(context.Background(), nil, "add", ".")
@@ -153,12 +160,15 @@ func NewGitSave(logger *zap.Logger, conf *GitSaveConfig) *GitSave {
 // Save adds files to the provided index, creates a tree and a commit pointing to
 // that tree, finally it creates a branch poiting to that commit
 // Save will use the current worktree index if available to speed the index generation
-func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
+func (s *GitSave) Save(message, branchName string) error {
 	gitdir, err := GitDir()
 	if err != nil {
 		return err
 	}
 
+	tmpIndexPath := filepath.Join(gitdir, "gitsave-index-"+uuid.NewV4().String())
+	defer os.Remove(tmpIndexPath)
+
 	indexPath := filepath.Join(gitdir, gitIndexFile)
 
 	curBranch, err := currentGitBranch()
@@ -187,6 +197,11 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
 		s.log.Infof("index %s does not exist", indexPath)
 	}
 
+	s.log.Infof("updating files already in the index")
+	if err := gitUpdateFiles(tmpIndexPath); err != nil {
+		return err
+	}
+
 	if s.conf.AddUntracked {
 		s.log.Infof("adding untracked files")
 		if err := gitAddUntrackedFiles(tmpIndexPath); err != nil {
@@ -209,7 +224,7 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
 	s.log.Infof("tree: %s", treeSHA)
 
 	s.log.Infof("committing tree")
-	commitSHA, err := gitCommitTree("git-save", treeSHA)
+	commitSHA, err := gitCommitTree(message, treeSHA)
 	if err != nil {
 		return err
 	}

internal/services/gateway/webhook.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/sorintlab/agola/internal/config"
 	gitsource "github.com/sorintlab/agola/internal/gitsources"
+	"github.com/sorintlab/agola/internal/gitsources/agolagit"
 	"github.com/sorintlab/agola/internal/runconfig"
 	csapi "github.com/sorintlab/agola/internal/services/configstore/api"
 	"github.com/sorintlab/agola/internal/services/gateway/common"
@@ -107,49 +108,80 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	ctx := r.Context()
 
 	projectID := r.URL.Query().Get("projectid")
+	isUserBuild := false
+	if projectID == "" {
+		isUserBuild = true
+	}
 
 	defer r.Body.Close()
 
+	var webhookData *types.WebhookData
+	var sshPrivKey string
+	var cloneURL string
+	var skipSSHHostKeyCheck bool
+	var runType types.RunType
+	var userID string
+
 	var gitSource gitsource.GitSource
-	project, _, err := h.configstoreClient.GetProject(ctx, projectID)
+	if !isUserBuild {
-	if err != nil {
+		project, _, err := h.configstoreClient.GetProject(ctx, projectID)
-		return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
+		if err != nil {
-	}
+			return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
-	h.log.Debugf("project: %s", util.Dump(project))
+		}
+		h.log.Debugf("project: %s", util.Dump(project))
 
-	user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
+		user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
-	if err != nil {
+		if err != nil {
-		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID)
+			return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID)
-	}
+		}
-	la := user.LinkedAccounts[project.LinkedAccountID]
+		la := user.LinkedAccounts[project.LinkedAccountID]
-	h.log.Infof("la: %s", util.Dump(la))
+		h.log.Infof("la: %s", util.Dump(la))
-	if la == nil {
+		if la == nil {
-		return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName)
+			return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName)
-	}
+		}
-	rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID)
+		rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID)
-	if err != nil {
+		if err != nil {
-		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID)
+			return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID)
-	}
+		}
 
-	gitSource, err = common.GetGitSource(rs, la)
+		gitSource, err = common.GetGitSource(rs, la)
-	if err != nil {
+		if err != nil {
-		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client")
+			return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client")
-	}
+		}
 
-	sshPrivKey := project.SSHPrivateKey
+		sshPrivKey = project.SSHPrivateKey
-	cloneURL := project.CloneURL
+		cloneURL = project.CloneURL
-	skipSSHHostKeyCheck := project.SkipSSHHostKeyCheck
+		skipSSHHostKeyCheck = project.SkipSSHHostKeyCheck
-	runType := types.RunTypeProject
+		runType = types.RunTypeProject
-	webhookData, err := gitSource.ParseWebhook(r)
+		webhookData, err = gitSource.ParseWebhook(r)
-	if err != nil {
+		if err != nil {
-		return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
+			return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
+		}
+		webhookData.ProjectID = projectID
+
+	} else {
+		gitSource = agolagit.New(h.apiExposedURL + "/repos")
+		var err error
+		webhookData, err = gitSource.ParseWebhook(r)
+		if err != nil {
+			return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
+		}
+
+		user, _, err := h.configstoreClient.GetUserByName(ctx, webhookData.Repo.Owner)
+		if err != nil {
+			return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
+		}
+		h.log.Debugf("user: %s", util.Dump(user))
+		userID = user.ID
+
+		cloneURL = fmt.Sprintf("%s/%s/%s", h.apiExposedURL+"/repos", webhookData.Repo.Owner, webhookData.Repo.Name)
+		runType = types.RunTypeUser
 	}
-	webhookData.ProjectID = projectID
 
 	h.log.Infof("webhookData: %s", util.Dump(webhookData))
 
 	var data []byte
-	err = util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
+	err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
 		var err error
 		data, err = gitSource.GetFile(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, agolaDefaultConfigPath)
 		if err == nil {
@@ -169,7 +201,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	}
 
 	env := map[string]string{
-		"CI":                     "true",
+		"CI":                   "true",
 		"AGOLA_SSHPRIVKEY":     sshPrivKey,
 		"AGOLA_REPOSITORY_URL": cloneURL,
 		"AGOLA_GIT_HOST":       gitURL.Host,
@@ -182,7 +214,6 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	}
 
 	annotations := map[string]string{
-		AnnotationProjectID:     webhookData.ProjectID,
 		AnnotationRunType:       string(runType),
 		AnnotationEventType:     string(webhookData.Event),
 		AnnotationVirtualBranch: genAnnotationVirtualBranch(webhookData),
@@ -194,6 +225,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		AnnotationCompareLink:   webhookData.CompareLink,
 	}
 
+	if !isUserBuild {
+		annotations[AnnotationProjectID] = webhookData.ProjectID
+	} else {
+		annotations[AnnotationUserID] = userID
+	}
+
 	if webhookData.Event == types.WebhookEventPush {
 		annotations[AnnotationBranch] = webhookData.Branch
 		annotations[AnnotationBranchLink] = webhookData.BranchLink
@@ -207,7 +244,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		annotations[AnnotationPullRequestLink] = webhookData.PullRequestLink
 	}
 
-	group := genGroup(webhookData.ProjectID, webhookData)
+	var group string
+	if !isUserBuild {
+		group = genGroup(webhookData.ProjectID, webhookData)
+	} else {
+		group = genGroup(userID, webhookData)
+	}
 
 	if err := h.createRuns(ctx, data, group, annotations, env); err != nil {
 		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")