a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add user local runs feature

This commit is contained in:
Simone Gotti 2019-02-21 23:01:17 +01:00
parent 809b17d250
commit 715596e650
4 changed files with 197 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
cmd/agola/cmd/localrunstart.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
gitsave "github.com/sorintlab/agola/internal/git-save"
uuid "github.com/satori/go.uuid"
"github.com/spf13/cobra"
)
var cmdLocalRunStart = &cobra.Command{
Use: "start",
Run: func(cmd *cobra.Command, args []string) {
if err := localRunStart(cmd, args); err != nil {
log.Fatalf("err: %v", err)
}
},
Short: "executes a run from a local repository",
}
type localRunStartOptions struct {
statusFilter []string
labelFilter []string
limit int
start string
untracked bool
ignored bool
}
var localRunStartOpts localRunStartOptions
func init() {
flags := cmdLocalRunStart.PersistentFlags()
flags.StringSliceVarP(&localRunStartOpts.statusFilter, "status", "s", nil, "filter runs matching the provided status. This option can be repeated multiple times")
flags.StringArrayVarP(&localRunStartOpts.labelFilter, "label", "l", nil, "filter runs matching the provided label. This option can be repeated multiple times, in this case only runs matching all the labels will be returned")
flags.IntVar(&localRunStartOpts.limit, "limit", 10, "max number of runs to show")
flags.StringVar(&localRunStartOpts.start, "start", "", "starting run id (excluded) to fetch")
flags.BoolVar(&localRunStartOpts.untracked, "untracked", true, "push untracked files")
flags.BoolVar(&localRunStartOpts.ignored, "ignored", false, "push ignored files")
cmdLocalRun.AddCommand(cmdLocalRunStart)
}
func localRunStart(cmd *cobra.Command, args []string) error {
gs := gitsave.NewGitSave(logger, &gitsave.GitSaveConfig{
AddUntracked: localRunStartOpts.untracked,
AddIgnored: localRunStartOpts.ignored,
})
branch := "gitsavebranch-" + uuid.NewV4().String()
if err := gs.Save("agola local run", branch); err != nil {
log.Fatalf("err: %v", err)
}
log.Infof("pushing branch")
if err := gitsave.GitPush("", "http://172.17.0.1:8000/repos/sgotti/test02.git", "refs/gitsave/"+branch); err != nil {
return err
}
return nil
}

28
cmd/agola/cmd/locarun.go Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"github.com/spf13/cobra"
)
var cmdLocalRun = &cobra.Command{
Use: "localrun",
Short: "localrun",
}
func init() {
cmdAgola.AddCommand(cmdLocalRun)
}

19
internal/git-save/save.go
View File

@ -20,6 +20,7 @@ import (
"os"
"path/filepath"
uuid "github.com/satori/go.uuid"
"github.com/sorintlab/agola/internal/util"
"github.com/pkg/errors"
@ -112,6 +113,12 @@ func gitUpdateRef(message, ref, commitSHA string) error {
return err
}
func gitUpdateFiles(indexPath string) error {
git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}}
_, err := git.Output(context.Background(), nil, "add", "-u")
return err
}
func gitAddUntrackedFiles(indexPath string) error {
git := &util.Git{Env: []string{"GIT_INDEX_FILE=" + indexPath}}
_, err := git.Output(context.Background(), nil, "add", ".")
@ -153,12 +160,15 @@ func NewGitSave(logger *zap.Logger, conf *GitSaveConfig) *GitSave {
// Save adds files to the provided index, creates a tree and a commit pointing to
// that tree, finally it creates a branch poiting to that commit
// Save will use the current worktree index if available to speed the index generation
func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
func (s *GitSave) Save(message, branchName string) error {
gitdir, err := GitDir()
if err != nil {
return err
}
tmpIndexPath := filepath.Join(gitdir, "gitsave-index-"+uuid.NewV4().String())
defer os.Remove(tmpIndexPath)
indexPath := filepath.Join(gitdir, gitIndexFile)
curBranch, err := currentGitBranch()
@ -187,6 +197,11 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
s.log.Infof("index %s does not exist", indexPath)
}
s.log.Infof("updating files already in the index")
if err := gitUpdateFiles(tmpIndexPath); err != nil {
return err
}
if s.conf.AddUntracked {
s.log.Infof("adding untracked files")
if err := gitAddUntrackedFiles(tmpIndexPath); err != nil {
@ -209,7 +224,7 @@ func (s *GitSave) Save(tmpIndexPath, message, branchName string) error {
s.log.Infof("tree: %s", treeSHA)
s.log.Infof("committing tree")
commitSHA, err := gitCommitTree("git-save", treeSHA)
commitSHA, err := gitCommitTree(message, treeSHA)
if err != nil {
return err
}

110
internal/services/gateway/webhook.go
View File

@ -23,6 +23,7 @@ import (
"github.com/sorintlab/agola/internal/config"
gitsource "github.com/sorintlab/agola/internal/gitsources"
"github.com/sorintlab/agola/internal/gitsources/agolagit"
"github.com/sorintlab/agola/internal/runconfig"
csapi "github.com/sorintlab/agola/internal/services/configstore/api"
"github.com/sorintlab/agola/internal/services/gateway/common"
@ -107,49 +108,80 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
ctx := r.Context()
projectID := r.URL.Query().Get("projectid")
isUserBuild := false
if projectID == "" {
isUserBuild = true
}
defer r.Body.Close()
var webhookData *types.WebhookData
var sshPrivKey string
var cloneURL string
var skipSSHHostKeyCheck bool
var runType types.RunType
var userID string
var gitSource gitsource.GitSource
project, _, err := h.configstoreClient.GetProject(ctx, projectID)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
}
h.log.Debugf("project: %s", util.Dump(project))
if !isUserBuild {
project, _, err := h.configstoreClient.GetProject(ctx, projectID)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
}
h.log.Debugf("project: %s", util.Dump(project))
user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID)
}
la := user.LinkedAccounts[project.LinkedAccountID]
h.log.Infof("la: %s", util.Dump(la))
if la == nil {
return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName)
}
rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID)
}
user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get user by linked account %q", project.LinkedAccountID)
}
la := user.LinkedAccounts[project.LinkedAccountID]
h.log.Infof("la: %s", util.Dump(la))
if la == nil {
return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.UserName)
}
rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID)
}
gitSource, err = common.GetGitSource(rs, la)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client")
}
gitSource, err = common.GetGitSource(rs, la)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create gitea client")
}
sshPrivKey := project.SSHPrivateKey
cloneURL := project.CloneURL
skipSSHHostKeyCheck := project.SkipSSHHostKeyCheck
runType := types.RunTypeProject
webhookData, err := gitSource.ParseWebhook(r)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
sshPrivKey = project.SSHPrivateKey
cloneURL = project.CloneURL
skipSSHHostKeyCheck = project.SkipSSHHostKeyCheck
runType = types.RunTypeProject
webhookData, err = gitSource.ParseWebhook(r)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
}
webhookData.ProjectID = projectID
} else {
gitSource = agolagit.New(h.apiExposedURL + "/repos")
var err error
webhookData, err = gitSource.ParseWebhook(r)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to parse webhook")
}
user, _, err := h.configstoreClient.GetUserByName(ctx, webhookData.Repo.Owner)
if err != nil {
return http.StatusBadRequest, "", errors.Wrapf(err, "failed to get project %s", projectID)
}
h.log.Debugf("user: %s", util.Dump(user))
userID = user.ID
cloneURL = fmt.Sprintf("%s/%s/%s", h.apiExposedURL+"/repos", webhookData.Repo.Owner, webhookData.Repo.Name)
runType = types.RunTypeUser
}
webhookData.ProjectID = projectID
h.log.Infof("webhookData: %s", util.Dump(webhookData))
var data []byte
err = util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
var err error
data, err = gitSource.GetFile(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, agolaDefaultConfigPath)
if err == nil {
@ -169,7 +201,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
}
env := map[string]string{
"CI": "true",
"CI": "true",
"AGOLA_SSHPRIVKEY": sshPrivKey,
"AGOLA_REPOSITORY_URL": cloneURL,
"AGOLA_GIT_HOST": gitURL.Host,
@ -182,7 +214,6 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
}
annotations := map[string]string{
AnnotationProjectID: webhookData.ProjectID,
AnnotationRunType: string(runType),
AnnotationEventType: string(webhookData.Event),
AnnotationVirtualBranch: genAnnotationVirtualBranch(webhookData),
@ -194,6 +225,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
AnnotationCompareLink: webhookData.CompareLink,
}
if !isUserBuild {
annotations[AnnotationProjectID] = webhookData.ProjectID
} else {
annotations[AnnotationUserID] = userID
}
if webhookData.Event == types.WebhookEventPush {
annotations[AnnotationBranch] = webhookData.Branch
annotations[AnnotationBranchLink] = webhookData.BranchLink
@ -207,7 +244,12 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
annotations[AnnotationPullRequestLink] = webhookData.PullRequestLink
}
group := genGroup(webhookData.ProjectID, webhookData)
var group string
if !isUserBuild {
group = genGroup(webhookData.ProjectID, webhookData)
} else {
group = genGroup(userID, webhookData)
}
if err := h.createRuns(ctx, data, group, annotations, env); err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")