agola/internal/services/gateway/action/run.go

612 lines
17 KiB
Go

// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package action
import (
"context"
"encoding/json"
"net/http"
"path"
"regexp"
"agola.io/agola/internal/config"
gitsource "agola.io/agola/internal/gitsources"
"agola.io/agola/internal/runconfig"
"agola.io/agola/internal/services/common"
itypes "agola.io/agola/internal/services/types"
"agola.io/agola/internal/util"
cstypes "agola.io/agola/services/configstore/types"
rsapitypes "agola.io/agola/services/runservice/api/types"
rstypes "agola.io/agola/services/runservice/types"
"agola.io/agola/services/types"
errors "golang.org/x/xerrors"
)
const (
defaultSSHPort = "22"
agolaDefaultConfigDir = ".agola"
agolaDefaultJsonnetConfigFile = "config.jsonnet"
agolaDefaultJsonConfigFile = "config.json"
agolaDefaultYamlConfigFile = "config.yml"
// List of runs annotations
AnnotationRunType = "run_type"
AnnotationRefType = "ref_type"
AnnotationProjectID = "projectid"
AnnotationUserID = "userid"
AnnotationRunCreationTrigger = "run_creation_trigger"
AnnotationWebhookEvent = "webhook_event"
AnnotationWebhookSender = "webhook_sender"
AnnotationCommitSHA = "commit_sha"
AnnotationRef = "ref"
AnnotationMessage = "message"
AnnotationCommitLink = "commit_link"
AnnotationCompareLink = "compare_link"
AnnotationBranch = "branch"
AnnotationBranchLink = "branch_link"
AnnotationTag = "tag"
AnnotationTagLink = "tag_link"
AnnotationPullRequestID = "pull_request_id"
AnnotationPullRequestLink = "pull_request_link"
)
var (
SkipRunMessage = regexp.MustCompile(`.*\[ci skip\].*`)
)
func (h *ActionHandler) GetRun(ctx context.Context, runID string) (*rsapitypes.RunResponse, error) {
runResp, resp, err := h.runserviceClient.GetRun(ctx, runID, nil)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
canGetRun, err := h.CanGetRun(ctx, runResp.RunConfig.Group)
if err != nil {
return nil, errors.Errorf("failed to determine permissions: %w", err)
}
if !canGetRun {
return nil, util.NewErrForbidden(errors.Errorf("user not authorized"))
}
return runResp, nil
}
type GetRunsRequest struct {
PhaseFilter []string
ResultFilter []string
Group string
LastRun bool
ChangeGroups []string
StartRunID string
Limit int
Asc bool
}
func (h *ActionHandler) GetRuns(ctx context.Context, req *GetRunsRequest) (*rsapitypes.GetRunsResponse, error) {
canGetRun, err := h.CanGetRun(ctx, req.Group)
if err != nil {
return nil, errors.Errorf("failed to determine permissions: %w", err)
}
if !canGetRun {
return nil, util.NewErrForbidden(errors.Errorf("user not authorized"))
}
groups := []string{req.Group}
runsResp, resp, err := h.runserviceClient.GetRuns(ctx, req.PhaseFilter, req.ResultFilter, groups, req.LastRun, req.ChangeGroups, req.StartRunID, req.Limit, req.Asc)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
return runsResp, nil
}
type GetLogsRequest struct {
RunID string
TaskID string
Setup bool
Step int
Follow bool
}
func (h *ActionHandler) GetLogs(ctx context.Context, req *GetLogsRequest) (*http.Response, error) {
runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
canGetRun, err := h.CanGetRun(ctx, runResp.RunConfig.Group)
if err != nil {
return nil, errors.Errorf("failed to determine permissions: %w", err)
}
if !canGetRun {
return nil, util.NewErrForbidden(errors.Errorf("user not authorized"))
}
resp, err = h.runserviceClient.GetLogs(ctx, req.RunID, req.TaskID, req.Setup, req.Step, req.Follow)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
return resp, nil
}
type DeleteLogsRequest struct {
RunID string
TaskID string
Setup bool
Step int
}
func (h *ActionHandler) DeleteLogs(ctx context.Context, req *DeleteLogsRequest) error {
runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
if err != nil {
return ErrFromRemote(resp, err)
}
canDoRunActions, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group)
if err != nil {
return errors.Errorf("failed to determine permissions: %w", err)
}
if !canDoRunActions {
return util.NewErrForbidden(errors.Errorf("user not authorized"))
}
resp, err = h.runserviceClient.DeleteLogs(ctx, req.RunID, req.TaskID, req.Setup, req.Step)
if err != nil {
return ErrFromRemote(resp, err)
}
return nil
}
type RunActionType string
const (
RunActionTypeRestart RunActionType = "restart"
RunActionTypeCancel RunActionType = "cancel"
RunActionTypeStop RunActionType = "stop"
)
type RunActionsRequest struct {
RunID string
ActionType RunActionType
// Restart
FromStart bool
}
func (h *ActionHandler) RunAction(ctx context.Context, req *RunActionsRequest) (*rsapitypes.RunResponse, error) {
runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
canGetRun, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group)
if err != nil {
return nil, errors.Errorf("failed to determine permissions: %w", err)
}
if !canGetRun {
return nil, util.NewErrForbidden(errors.Errorf("user not authorized"))
}
switch req.ActionType {
case RunActionTypeRestart:
rsreq := &rsapitypes.RunCreateRequest{
RunID: req.RunID,
FromStart: req.FromStart,
}
runResp, resp, err = h.runserviceClient.CreateRun(ctx, rsreq)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
case RunActionTypeCancel:
rsreq := &rsapitypes.RunActionsRequest{
ActionType: rsapitypes.RunActionTypeChangePhase,
Phase: rstypes.RunPhaseCancelled,
}
resp, err = h.runserviceClient.RunActions(ctx, req.RunID, rsreq)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
case RunActionTypeStop:
rsreq := &rsapitypes.RunActionsRequest{
ActionType: rsapitypes.RunActionTypeStop,
}
resp, err = h.runserviceClient.RunActions(ctx, req.RunID, rsreq)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
default:
return nil, util.NewErrBadRequest(errors.Errorf("wrong run action type %q", req.ActionType))
}
return runResp, nil
}
type RunTaskActionType string
const (
RunTaskActionTypeApprove RunTaskActionType = "approve"
)
type RunTaskActionsRequest struct {
RunID string
TaskID string
ActionType RunTaskActionType
}
func (h *ActionHandler) RunTaskAction(ctx context.Context, req *RunTaskActionsRequest) error {
runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
if err != nil {
return ErrFromRemote(resp, err)
}
canDoRunAction, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group)
if err != nil {
return errors.Errorf("failed to determine permissions: %w", err)
}
if !canDoRunAction {
return util.NewErrForbidden(errors.Errorf("user not authorized"))
}
curUserID := h.CurrentUserID(ctx)
if curUserID == "" {
return util.NewErrBadRequest(errors.Errorf("no logged in user"))
}
switch req.ActionType {
case RunTaskActionTypeApprove:
rt, ok := runResp.Run.Tasks[req.TaskID]
if !ok {
return util.NewErrBadRequest(errors.Errorf("run %q doesn't have task %q", req.RunID, req.TaskID))
}
approvers := []string{}
annotations := map[string]string{}
if rt.Annotations != nil {
annotations = rt.Annotations
}
approversAnnotation, ok := annotations[common.ApproversAnnotation]
if ok {
if err := json.Unmarshal([]byte(approversAnnotation), &approvers); err != nil {
return errors.Errorf("failed to unmarshal run task approvers annotation: %w", err)
}
}
for _, approver := range approvers {
if approver == curUserID {
return util.NewErrBadRequest(errors.Errorf("user %q alredy approved the task", approver))
}
}
approvers = append(approvers, curUserID)
approversj, err := json.Marshal(approvers)
if err != nil {
return errors.Errorf("failed to marshal run task approvers annotation: %w", err)
}
annotations[common.ApproversAnnotation] = string(approversj)
rsreq := &rsapitypes.RunTaskActionsRequest{
ActionType: rsapitypes.RunTaskActionTypeSetAnnotations,
Annotations: annotations,
ChangeGroupsUpdateToken: runResp.ChangeGroupsUpdateToken,
}
resp, err := h.runserviceClient.RunTaskActions(ctx, req.RunID, req.TaskID, rsreq)
if err != nil {
return ErrFromRemote(resp, err)
}
default:
return util.NewErrBadRequest(errors.Errorf("wrong run task action type %q", req.ActionType))
}
return nil
}
type CreateRunRequest struct {
RunType itypes.RunType
RefType itypes.RunRefType
RunCreationTrigger itypes.RunCreationTriggerType
Project *cstypes.Project
User *cstypes.User
RepoPath string
GitSource gitsource.GitSource
CommitSHA string
Message string
Branch string
Tag string
Ref string
PullRequestID string
PRFromSameRepo bool
SSHPrivKey string
SSHHostKey string
SkipSSHHostKeyCheck bool
CloneURL string
WebhookEvent string
WebhookSender string
CommitLink string
BranchLink string
TagLink string
PullRequestLink string
// CompareLink is provided only when triggered by a webhook and contains the
// commit compare link
CompareLink string
// fields only used with user direct runs
UserRunRepoUUID string
Variables map[string]string
}
func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) error {
setupErrors := []string{}
if req.CommitSHA == "" {
return util.NewErrBadRequest(errors.Errorf("empty commit SHA"))
}
if req.Message == "" {
return util.NewErrBadRequest(errors.Errorf("empty message"))
}
var baseGroupType common.GroupType
var baseGroupID string
var groupType common.GroupType
var group string
if req.RunType == itypes.RunTypeProject {
baseGroupType = common.GroupTypeProject
baseGroupID = req.Project.ID
} else {
baseGroupType = common.GroupTypeUser
baseGroupID = req.User.ID
}
switch req.RefType {
case itypes.RunRefTypeBranch:
groupType = common.GroupTypeBranch
group = req.Branch
case itypes.RunRefTypeTag:
groupType = common.GroupTypeTag
group = req.Tag
case itypes.RunRefTypePullRequest:
groupType = common.GroupTypePullRequest
group = req.PullRequestID
}
runGroup := common.GenRunGroup(baseGroupType, baseGroupID, groupType, group)
gitURL, err := util.ParseGitURL(req.CloneURL)
if err != nil {
return errors.Errorf("failed to parse clone url: %w", err)
}
gitHost := gitURL.Hostname()
gitPort := gitURL.Port()
if gitPort == "" {
gitPort = defaultSSHPort
}
// this env vars overrides other env vars
env := map[string]string{
"CI": "true",
"AGOLA_SSHPRIVKEY": req.SSHPrivKey,
"AGOLA_REPOSITORY_URL": req.CloneURL,
"AGOLA_GIT_HOST": gitHost,
"AGOLA_GIT_PORT": gitPort,
"AGOLA_GIT_BRANCH": req.Branch,
"AGOLA_GIT_TAG": req.Tag,
"AGOLA_GIT_REF": req.Ref,
"AGOLA_GIT_COMMITSHA": req.CommitSHA,
}
if req.SSHHostKey != "" {
env["AGOLA_SSHHOSTKEY"] = req.SSHHostKey
}
if req.SkipSSHHostKeyCheck {
env["AGOLA_SKIPSSHHOSTKEYCHECK"] = "1"
}
var variables map[string]string
if req.RunType == itypes.RunTypeProject {
if req.RefType != itypes.RunRefTypePullRequest || req.PRFromSameRepo || req.Project.PassVarsToForkedPR {
var err error
variables, err = h.genRunVariables(ctx, req)
if err != nil {
return err
}
}
} else {
variables = req.Variables
}
annotations := map[string]string{
AnnotationRunType: string(req.RunType),
AnnotationRefType: string(req.RefType),
AnnotationRunCreationTrigger: string(req.RunCreationTrigger),
AnnotationWebhookEvent: req.WebhookEvent,
AnnotationWebhookSender: req.WebhookSender,
AnnotationCommitSHA: req.CommitSHA,
AnnotationRef: req.Ref,
AnnotationMessage: req.Message,
AnnotationCommitLink: req.CommitLink,
AnnotationCompareLink: req.CompareLink,
}
if req.RunType == itypes.RunTypeProject {
annotations[AnnotationProjectID] = req.Project.ID
} else {
annotations[AnnotationUserID] = req.User.ID
}
if req.Branch != "" {
annotations[AnnotationBranch] = req.Branch
annotations[AnnotationBranchLink] = req.BranchLink
}
if req.Tag != "" {
annotations[AnnotationTag] = req.Tag
annotations[AnnotationTagLink] = req.TagLink
}
if req.PullRequestID != "" {
annotations[AnnotationPullRequestID] = req.PullRequestID
annotations[AnnotationPullRequestLink] = req.PullRequestLink
}
// Since user belong to the same group (the user uuid) we needed another way to differentiate the cache. We'll use the user uuid + the user run repo uuid
var cacheGroup string
if req.RunType == itypes.RunTypeUser {
cacheGroup = req.User.ID + "-" + req.UserRunRepoUUID
}
data, filename, err := h.fetchConfigFiles(ctx, req.GitSource, req.RepoPath, req.CommitSHA)
if err != nil {
return util.NewErrInternal(errors.Errorf("failed to fetch config file: %w", err))
}
h.log.Debug("data: %s", data)
var configFormat config.ConfigFormat
switch path.Ext(filename) {
case ".jsonnet":
configFormat = config.ConfigFormatJsonnet
case ".json":
fallthrough
case ".yml":
configFormat = config.ConfigFormatJSON
}
config, err := config.ParseConfig([]byte(data), configFormat)
if err != nil {
h.log.Errorf("failed to parse config: %+v", err)
// create a run (per config file) with a generic error since we cannot parse
// it and know how many runs are defined
setupErrors = append(setupErrors, err.Error())
createRunReq := &rsapitypes.RunCreateRequest{
RunConfigTasks: nil,
Group: runGroup,
SetupErrors: setupErrors,
Name: rstypes.RunGenericSetupErrorName,
StaticEnvironment: env,
Annotations: annotations,
}
if _, _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil {
h.log.Errorf("failed to create run: %+v", err)
return err
}
return nil
}
for _, run := range config.Runs {
if SkipRunMessage.MatchString(req.Message) {
h.log.Debugf("skipping run since special commit message")
continue
}
if match := types.MatchWhen(run.When.ToWhen(), req.RefType, req.Branch, req.Tag, req.Ref); !match {
h.log.Debugf("skipping run since when condition doesn't match")
continue
}
rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, run.Name, variables, req.RefType, req.Branch, req.Tag, req.Ref)
createRunReq := &rsapitypes.RunCreateRequest{
RunConfigTasks: rcts,
Group: runGroup,
SetupErrors: setupErrors,
Name: run.Name,
StaticEnvironment: env,
Annotations: annotations,
CacheGroup: cacheGroup,
}
if _, _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil {
h.log.Errorf("failed to create run: %+v", err)
return err
}
}
return nil
}
func (h *ActionHandler) fetchConfigFiles(ctx context.Context, gitSource gitsource.GitSource, repopath, commitSHA string) ([]byte, string, error) {
var data []byte
var filename string
err := util.ExponentialBackoff(ctx, util.FetchFileBackoff, func() (bool, error) {
for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
var err error
data, err = gitSource.GetFile(repopath, commitSHA, path.Join(agolaDefaultConfigDir, filename))
if err == nil {
return true, nil
}
h.log.Errorf("get file err: %v", err)
}
return false, nil
})
if err != nil {
return nil, "", err
}
return data, filename, nil
}
func (h *ActionHandler) genRunVariables(ctx context.Context, req *CreateRunRequest) (map[string]string, error) {
variables := map[string]string{}
// get project variables
pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, req.Project.ID, true)
if err != nil {
return nil, errors.Errorf("failed to get project variables: %w", err)
}
// remove overriden variables
pvars = common.FilterOverriddenVariables(pvars)
// get project secrets
secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, req.Project.ID, true)
if err != nil {
return nil, errors.Errorf("failed to get project secrets: %w", err)
}
for _, pvar := range pvars {
// find the value match
var varval cstypes.VariableValue
for _, varval = range pvar.Values {
match := types.MatchWhen(varval.When, req.RefType, req.Branch, req.Tag, req.Ref)
if !match {
continue
}
// get the secret value referenced by the variable, it must be a secret at the same level or a lower level
secret := common.GetVarValueMatchingSecret(varval, pvar.ParentPath, secrets)
if secret != nil {
varValue, ok := secret.Data[varval.SecretVar]
if ok {
variables[pvar.Name] = varValue
}
}
break
}
}
return variables, nil
}