From 715596e6503cc4241eb36f6269dd2707715b46b1 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Thu, 21 Feb 2019 23:01:17 +0100 Subject: [PATCH] Add user local runs feature --- cmd/agola/cmd/localrunstart.go | 76 ++++++++++++++++++ cmd/agola/cmd/locarun.go | 28 +++++++ internal/git-save/save.go | 19 ++++- internal/services/gateway/webhook.go | 110 ++++++++++++++++++--------- 4 files changed, 197 insertions(+), 36 deletions(-) create mode 100644 cmd/agola/cmd/localrunstart.go create mode 100644 cmd/agola/cmd/locarun.go diff --git a/cmd/agola/cmd/localrunstart.go b/cmd/agola/cmd/localrunstart.go new file mode 100644 index 0000000..4f52fd6 --- /dev/null +++ b/cmd/agola/cmd/localrunstart.go @@ -0,0 +1,76 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + gitsave "github.com/sorintlab/agola/internal/git-save" + + uuid "github.com/satori/go.uuid" + "github.com/spf13/cobra" +) + +var cmdLocalRunStart = &cobra.Command{ + Use: "start", + Run: func(cmd *cobra.Command, args []string) { + if err := localRunStart(cmd, args); err != nil { + log.Fatalf("err: %v", err) + } + }, + Short: "executes a run from a local repository", +} + +type localRunStartOptions struct { + statusFilter []string + labelFilter []string + limit int + start string + untracked bool + ignored bool +} + +var localRunStartOpts localRunStartOptions + +func init() { + flags := cmdLocalRunStart.PersistentFlags() + + flags.StringSliceVarP(&localRunStartOpts.statusFilter, "status", "s", nil, "filter runs matching the provided status. This option can be repeated multiple times") + flags.StringArrayVarP(&localRunStartOpts.labelFilter, "label", "l", nil, "filter runs matching the provided label. This option can be repeated multiple times, in this case only runs matching all the labels will be returned") + flags.IntVar(&localRunStartOpts.limit, "limit", 10, "max number of runs to show") + flags.StringVar(&localRunStartOpts.start, "start", "", "starting run id (excluded) to fetch") + flags.BoolVar(&localRunStartOpts.untracked, "untracked", true, "push untracked files") + flags.BoolVar(&localRunStartOpts.ignored, "ignored", false, "push ignored files") + + cmdLocalRun.AddCommand(cmdLocalRunStart) +} + +func localRunStart(cmd *cobra.Command, args []string) error { + gs := gitsave.NewGitSave(logger, &gitsave.GitSaveConfig{ + AddUntracked: localRunStartOpts.untracked, + AddIgnored: localRunStartOpts.ignored, + }) + + branch := "gitsavebranch-" + uuid.NewV4().String() + + if err := gs.Save("agola local run", branch); err != nil { + log.Fatalf("err: %v", err) + } + + log.Infof("pushing branch") + if err := gitsave.GitPush("", "http://172.17.0.1:8000/repos/sgotti/test02.git", "refs/gitsave/"+branch); err != nil { + return err + } + + return nil +} diff --git a/cmd/agola/cmd/locarun.go b/cmd/agola/cmd/locarun.go new file mode 100644 index 0000000..b92d1f0 --- /dev/null +++ b/cmd/agola/cmd/locarun.go @@ -0,0 +1,28 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/spf13/cobra" +) + +var cmdLocalRun = &cobra.Command{ + Use: "localrun", + Short: "localrun", +} + +func init() { + cmdAgola.AddCommand(cmdLocalRun) +} diff --git a/internal/git-save/save.go b/internal/git-save/save.go index 847fa2d..2f8d9b8 100644 --- a/internal/git-save/save.go +++ b/internal/git-save/save.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" + uuid "github.com/satori/go.uuid" "github.com/sorintlab/agola/internal/util" "github.com/pkg/errors" @@ -112,6 +113,12 @@ func gitUpdateRef(message, ref, commitSHA string) error { return err } +func gitUpdateFiles(indexPath string) error { + git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}} + _, err := git.Output(context.Background(), nil, "add", "-u") + return err +} + func gitAddUntrackedFiles(indexPath string) error { git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}} _, err := git.Output(context.Background(), nil, "add", ".") @@ -153,12 +160,15 @@ func NewGitSave(logger *zap.Logger, conf *GitSaveConfig) *GitSave { // Save adds files to the provided index, creates a tree and a commit pointing to // that tree, finally it creates a branch poiting to that commit // Save will use the current worktree index if available to speed the index generation -func (s *GitSave) Save(tmpIndexPath, message, branchName string) error { +func (s *GitSave) Save(message, branchName string) error { gitdir, err := GitDir() if err != nil { return err } + tmpIndexPath := filepath.Join(gitdir, "gitsave-index-"+uuid.NewV4().String()) + defer os.Remove(tmpIndexPath) + indexPath := filepath.Join(gitdir, gitIndexFile) curBranch, err := currentGitBranch() @@ -187,6 +197,11 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error { s.log.Infof("index %s does not exist", indexPath) } + s.log.Infof("updating files already in the index") + if err := gitUpdateFiles(tmpIndexPath); err != nil { + return err + } + if s.conf.AddUntracked { s.log.Infof("adding untracked files") if err := gitAddUntrackedFiles(tmpIndexPath); err != nil { @@ -209,7 +224,7 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error { s.log.Infof("tree: %s", treeSHA) s.log.Infof("committing tree") - commitSHA, err := gitCommitTree("git-save", treeSHA) + commitSHA, err := gitCommitTree(message, treeSHA) if err != nil { return err } diff --git a/internal/services/gateway/webhook.go b/internal/services/gateway/webhook.go index 4e6f1b5..78f943a 100644 --- a/internal/services/gateway/webhook.go +++ b/internal/services/gateway/webhook.go @@ -23,6 +23,7 @@ import ( "github.com/sorintlab/agola/internal/config" gitsource "github.com/sorintlab/agola/internal/gitsources" + "github.com/sorintlab/agola/internal/gitsources/agolagit" "github.com/sorintlab/agola/internal/runconfig" csapi "github.com/sorintlab/agola/internal/services/configstore/api" "github.com/sorintlab/agola/internal/services/gateway/common" @@ -107,49 +108,80 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { ctx := r.Context() projectID := r.URL.Query().Get("projectid") + isUserBuild := false + if projectID == "" { + isUserBuild = true + } defer r.Body.Close() + var webhookData *types.WebhookData + var sshPrivKey string + var cloneURL string + var skipSSHHostKeyCheck bool + var runType types.RunType + var userID string + var gitSource gitsource.GitSource - project, _, err := h.configstoreClient.GetProject(ctx, projectID) - if err != nil { - return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID) - } - h.log.Debugf("project: %s", util.Dump(project)) + if !isUserBuild { + project, _, err := h.configstoreClient.GetProject(ctx, projectID) + if err != nil { + return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID) + } + h.log.Debugf("project: %s", util.Dump(project)) - user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID) - if err != nil { - return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID) - } - la := user.LinkedAccounts[project.LinkedAccountID] - h.log.Infof("la: %s", util.Dump(la)) - if la == nil { - return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName) - } - rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID) - if err != nil { - return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID) - } + user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID) + if err != nil { + return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID) + } + la := user.LinkedAccounts[project.LinkedAccountID] + h.log.Infof("la: %s", util.Dump(la)) + if la == nil { + return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName) + } + rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID) + if err != nil { + return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID) + } - gitSource, err = common.GetGitSource(rs, la) - if err != nil { - return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client") - } + gitSource, err = common.GetGitSource(rs, la) + if err != nil { + return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client") + } - sshPrivKey := project.SSHPrivateKey - cloneURL := project.CloneURL - skipSSHHostKeyCheck := project.SkipSSHHostKeyCheck - runType := types.RunTypeProject - webhookData, err := gitSource.ParseWebhook(r) - if err != nil { - return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook") + sshPrivKey = project.SSHPrivateKey + cloneURL = project.CloneURL + skipSSHHostKeyCheck = project.SkipSSHHostKeyCheck + runType = types.RunTypeProject + webhookData, err = gitSource.ParseWebhook(r) + if err != nil { + return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook") + } + webhookData.ProjectID = projectID + + } else { + gitSource = agolagit.New(h.apiExposedURL + "/repos") + var err error + webhookData, err = gitSource.ParseWebhook(r) + if err != nil { + return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook") + } + + user, _, err := h.configstoreClient.GetUserByName(ctx, webhookData.Repo.Owner) + if err != nil { + return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID) + } + h.log.Debugf("user: %s", util.Dump(user)) + userID = user.ID + + cloneURL = fmt.Sprintf("%s/%s/%s", h.apiExposedURL+"/repos", webhookData.Repo.Owner, webhookData.Repo.Name) + runType = types.RunTypeUser } - webhookData.ProjectID = projectID h.log.Infof("webhookData: %s", util.Dump(webhookData)) var data []byte - err = util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) { + err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) { var err error data, err = gitSource.GetFile(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, agolaDefaultConfigPath) if err == nil { @@ -169,7 +201,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { } env := map[string]string{ - "CI": "true", + "CI": "true", "AGOLA_SSHPRIVKEY": sshPrivKey, "AGOLA_REPOSITORY_URL": cloneURL, "AGOLA_GIT_HOST": gitURL.Host, @@ -182,7 +214,6 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { } annotations := map[string]string{ - AnnotationProjectID: webhookData.ProjectID, AnnotationRunType: string(runType), AnnotationEventType: string(webhookData.Event), AnnotationVirtualBranch: genAnnotationVirtualBranch(webhookData), @@ -194,6 +225,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { AnnotationCompareLink: webhookData.CompareLink, } + if !isUserBuild { + annotations[AnnotationProjectID] = webhookData.ProjectID + } else { + annotations[AnnotationUserID] = userID + } + if webhookData.Event == types.WebhookEventPush { annotations[AnnotationBranch] = webhookData.Branch annotations[AnnotationBranchLink] = webhookData.BranchLink @@ -207,7 +244,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { annotations[AnnotationPullRequestLink] = webhookData.PullRequestLink } - group := genGroup(webhookData.ProjectID, webhookData) + var group string + if !isUserBuild { + group = genGroup(webhookData.ProjectID, webhookData) + } else { + group = genGroup(userID, webhookData) + } if err := h.createRuns(ctx, data, group, annotations, env); err != nil { return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")