agola/internal/services/gateway/webhook.go

// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.

package gateway

import (
	"context"
	"fmt"
	"net/http"
	"path"

	"github.com/sorintlab/agola/internal/config"
	gitsource "github.com/sorintlab/agola/internal/gitsources"
	"github.com/sorintlab/agola/internal/gitsources/agolagit"
	"github.com/sorintlab/agola/internal/runconfig"
	"github.com/sorintlab/agola/internal/services/common"
	csapi "github.com/sorintlab/agola/internal/services/configstore/api"
	"github.com/sorintlab/agola/internal/services/gateway/action"
	rsapi "github.com/sorintlab/agola/internal/services/runservice/api"
	rstypes "github.com/sorintlab/agola/internal/services/runservice/types"
	"github.com/sorintlab/agola/internal/services/types"
	"github.com/sorintlab/agola/internal/util"

	"go.uber.org/zap"
	errors "golang.org/x/xerrors"
)

const (
	defaultSSHPort = "22"

	agolaDefaultConfigDir         = ".agola"
	agolaDefaultJsonnetConfigFile = "config.jsonnet"
	agolaDefaultJsonConfigFile    = "config.json"
	agolaDefaultYamlConfigFile    = "config.yml"

	// List of runs annotations
	AnnotationEventType = "event_type"
	AnnotationRunType   = "runtype"
	AnnotationProjectID = "projectid"
	AnnotationUserID    = "userid"
	// AnnotationVirtualBranch represent a "virtual branch": i.e a normal branch, a pr (with name pr-$prid), a tag (with name tag-tagname)
	AnnotationVirtualBranch = "virtual_branch"

	AnnotationCommitSHA   = "commit_sha"
	AnnotationRef         = "ref"
	AnnotationSender      = "sender"
	AnnotationMessage     = "message"
	AnnotationCommitLink  = "commit_link"
	AnnotationCompareLink = "compare_link"

	AnnotationBranch          = "branch"
	AnnotationBranchLink      = "branch_link"
	AnnotationTag             = "tag"
	AnnotationTagLink         = "tag_link"
	AnnotationPullRequestID   = "pull_request_id"
	AnnotationPullRequestLink = "pull_request_link"
)

func genAnnotationVirtualBranch(webhookData *types.WebhookData) string {
	switch webhookData.Event {
	case types.WebhookEventPush:
		return "branch-" + webhookData.Branch
	case types.WebhookEventTag:
		return "tag-" + webhookData.Tag
	case types.WebhookEventPullRequest:
		return "pr-" + webhookData.PullRequestID
	}

	panic(fmt.Errorf("invalid webhook event type: %q", webhookData.Event))
}

type webhooksHandler struct {
	log               *zap.SugaredLogger
	ah                *action.ActionHandler
	configstoreClient *csapi.Client
	runserviceClient  *rsapi.Client
	apiExposedURL     string
}

func (h *webhooksHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	code, userErr, err := h.handleWebhook(r)
	if err != nil {
		h.log.Errorf("err: %+v", err)
		http.Error(w, userErr, code)
	}
}

func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
	ctx := r.Context()

	projectID := r.URL.Query().Get("projectid")
	userID := r.URL.Query().Get("userid")
	if projectID == "" && userID == "" {
		return http.StatusBadRequest, "", errors.Errorf("bad webhook url %q. Missing projectid or userid", r.URL)
	}

	isUserBuild := false
	if projectID == "" {
		isUserBuild = true
	}

	defer r.Body.Close()

	var webhookData *types.WebhookData
	var sshPrivKey string
	var cloneURL string
	var sshHostKey string
	var skipSSHHostKeyCheck bool
	var runType types.RunType
	variables := map[string]string{}

	var gitSource gitsource.GitSource
	if !isUserBuild {
		project, _, err := h.configstoreClient.GetProject(ctx, projectID)
		if err != nil {
			return http.StatusBadRequest, "", errors.Errorf("failed to get project %s: %w", projectID, err)
		}
		h.log.Infof("project: %s", util.Dump(project))

		user, _, err := h.configstoreClient.GetUserByLinkedAccount(ctx, project.LinkedAccountID)
		if err != nil {
			return http.StatusInternalServerError, "", errors.Errorf("failed to get user by linked account %q: %w", project.LinkedAccountID, err)
		}
		la := user.LinkedAccounts[project.LinkedAccountID]
		h.log.Infof("la: %s", util.Dump(la))
		if la == nil {
			return http.StatusInternalServerError, "", errors.Errorf("linked account %q in user %q doesn't exist", project.LinkedAccountID, user.Name)
		}
		rs, _, err := h.configstoreClient.GetRemoteSource(ctx, la.RemoteSourceID)
		if err != nil {
			return http.StatusInternalServerError, "", errors.Errorf("failed to get remote source %q: %w", la.RemoteSourceID, err)
		}

		gitSource, err = h.ah.GetGitSource(ctx, rs, user.Name, la)
		if err != nil {
			return http.StatusInternalServerError, "", errors.Errorf("failed to create gitea client: %w", err)
		}

		sshPrivKey = project.SSHPrivateKey
		sshHostKey = rs.SSHHostKey
		// use remotesource skipSSHHostKeyCheck config and override with project config if set to true there
		skipSSHHostKeyCheck = rs.SkipSSHHostKeyCheck
		if project.SkipSSHHostKeyCheck {
			skipSSHHostKeyCheck = project.SkipSSHHostKeyCheck
		}
		runType = types.RunTypeProject
		webhookData, err = gitSource.ParseWebhook(r, project.WebhookSecret)
		if err != nil {
			return http.StatusBadRequest, "", errors.Errorf("failed to parse webhook: %w", err)
		}
		// skip nil webhook data
		// TODO(sgotti) report the reason of the skip
		if webhookData == nil {
			h.log.Infof("skipping webhook")
			return 0, "", nil
		}

		webhookData.ProjectID = projectID

		cloneURL = webhookData.SSHURL

		// get project variables
		pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, project.ID, true)
		if err != nil {
			return http.StatusInternalServerError, "", errors.Errorf("failed to get project variables: %w", err)
		}
		h.log.Infof("pvars: %v", util.Dump(pvars))

		// remove overriden variables
		pvars = common.FilterOverriddenVariables(pvars)
		h.log.Infof("pvars: %v", util.Dump(pvars))

		// get project secrets
		secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, project.ID, true)
		if err != nil {
			return http.StatusInternalServerError, "", errors.Errorf("failed to get project secrets: %w", err)
		}
		h.log.Infof("secrets: %v", util.Dump(secrets))
		for _, pvar := range pvars {
			// find the value match
			var varval types.VariableValue
			for _, varval = range pvar.Values {
				h.log.Infof("varval: %v", util.Dump(varval))
				match := types.MatchWhen(varval.When, webhookData.Branch, webhookData.Tag, webhookData.Ref)
				if !match {
					continue
				}
				// get the secret value referenced by the variable, it must be a secret at the same level or a lower level
				secret := common.GetVarValueMatchingSecret(varval, pvar.ParentPath, secrets)
				h.log.Infof("secret: %v", util.Dump(secret))
				if secret != nil {
					varValue, ok := secret.Data[varval.SecretVar]
					if ok {
						variables[pvar.Name] = varValue
					}
				}
				break
			}
		}
		h.log.Infof("variables: %v", util.Dump(variables))

	} else {
		gitSource = agolagit.New(h.apiExposedURL + "/repos")
		var err error
		webhookData, err = gitSource.ParseWebhook(r, "")
		if err != nil {
			return http.StatusBadRequest, "", errors.Errorf("failed to parse webhook: %w", err)
		}
		// skip nil webhook data
		// TODO(sgotti) report the reason of the skip
		if webhookData == nil {
			h.log.Infof("skipping webhook")
			return 0, "", nil
		}

		user, _, err := h.configstoreClient.GetUser(ctx, userID)
		if err != nil {
			return http.StatusBadRequest, "", errors.Errorf("failed to get user with id %q: %w", userID, err)
		}
		h.log.Debugf("user: %s", util.Dump(user))
		userID = user.ID

		cloneURL = fmt.Sprintf("%s/%s", h.apiExposedURL+"/repos", webhookData.Repo.Path)
		runType = types.RunTypeUser
	}

	h.log.Infof("webhookData: %s", util.Dump(webhookData))

	data, filename, err := h.fetchConfigFiles(gitSource, webhookData)
	if err != nil {
		return http.StatusInternalServerError, "", errors.Errorf("failed to fetch config file: %w", err)
	}
	h.log.Debug("data: %s", data)

	gitURL, err := util.ParseGitURL(cloneURL)
	if err != nil {
		return http.StatusInternalServerError, "", errors.Errorf("failed to parse clone url: %w", err)
	}
	gitHost := gitURL.Hostname()
	gitPort := gitURL.Port()
	if gitPort == "" {
		gitPort = defaultSSHPort
	}

	// this env vars ovverrides other env vars
	env := map[string]string{
		"CI":                   "true",
		"AGOLA_SSHPRIVKEY":     sshPrivKey,
		"AGOLA_REPOSITORY_URL": cloneURL,
		"AGOLA_GIT_HOST":       gitHost,
		"AGOLA_GIT_PORT":       gitPort,
		"AGOLA_GIT_BRANCH":     webhookData.Branch,
		"AGOLA_GIT_TAG":        webhookData.Tag,
		"AGOLA_GIT_REF":        webhookData.Ref,
		"AGOLA_GIT_COMMITSHA":  webhookData.CommitSHA,
	}

	if sshHostKey != "" {
		env["AGOLA_SSHHOSTKEY"] = sshHostKey
	}
	if skipSSHHostKeyCheck {
		env["AGOLA_SKIPSSHHOSTKEYCHECK"] = "1"
	}

	annotations := map[string]string{
		AnnotationRunType:       string(runType),
		AnnotationEventType:     string(webhookData.Event),
		AnnotationVirtualBranch: genAnnotationVirtualBranch(webhookData),
		AnnotationCommitSHA:     webhookData.CommitSHA,
		AnnotationRef:           webhookData.Ref,
		AnnotationSender:        webhookData.Sender,
		AnnotationMessage:       webhookData.Message,
		AnnotationCommitLink:    webhookData.CommitLink,
		AnnotationCompareLink:   webhookData.CompareLink,
	}

	if !isUserBuild {
		annotations[AnnotationProjectID] = webhookData.ProjectID
	} else {
		annotations[AnnotationUserID] = userID
	}

	if webhookData.Event == types.WebhookEventPush {
		annotations[AnnotationBranch] = webhookData.Branch
		annotations[AnnotationBranchLink] = webhookData.BranchLink
	}
	if webhookData.Event == types.WebhookEventTag {
		annotations[AnnotationTag] = webhookData.Tag
		annotations[AnnotationTagLink] = webhookData.TagLink
	}
	if webhookData.Event == types.WebhookEventPullRequest {
		annotations[AnnotationPullRequestID] = webhookData.PullRequestID
		annotations[AnnotationPullRequestLink] = webhookData.PullRequestLink
	}

	var group string
	if !isUserBuild {
		group = common.GenRunGroup(common.GroupTypeProject, webhookData.ProjectID, webhookData)
	} else {
		group = common.GenRunGroup(common.GroupTypeUser, userID, webhookData)
	}

	if err := h.createRuns(ctx, filename, data, group, annotations, env, variables, webhookData); err != nil {
		return http.StatusInternalServerError, "", errors.Errorf("failed to create run: %w", err)
	}
	//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
	//	h.log.Errorf("failed to update commit status: %v", err)
	//}

	return 0, "", nil
}

// fetchConfigFiles tries to fetch a config file in one of the supported formats. The precedence is for jsonnet, then json and then yml
// TODO(sgotti) For jsonnet, if we'll support custom import files inside the configdir, also fetch them.
func (h *webhooksHandler) fetchConfigFiles(gitSource gitsource.GitSource, webhookData *types.WebhookData) ([]byte, string, error) {
	var data []byte
	var filename string
	err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
		for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
			var err error
			data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, path.Join(agolaDefaultConfigDir, filename))
			if err == nil {
				return true, nil
			}
			h.log.Errorf("get file err: %v", err)
		}
		return false, nil
	})
	if err != nil {
		return nil, "", err
	}
	return data, filename, nil
}

func (h *webhooksHandler) createRuns(ctx context.Context, filename string, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
	setupErrors := []string{}

	var configFormat config.ConfigFormat
	switch path.Ext(filename) {
	case ".jsonnet":
		configFormat = config.ConfigFormatJsonnet
	case ".json":
		fallthrough
	case ".yml":
		configFormat = config.ConfigFormatJSON

	}
	config, err := config.ParseConfig([]byte(configData), configFormat)
	if err != nil {
		log.Errorf("failed to parse config: %+v", err)

		// create a run (per config file) with a generic error since we cannot parse
		// it and know how many runs are defined
		setupErrors = append(setupErrors, err.Error())
		createRunReq := &rsapi.RunCreateRequest{
			RunConfigTasks:    nil,
			Group:             group,
			SetupErrors:       setupErrors,
			Name:              rstypes.RunGenericSetupErrorName,
			StaticEnvironment: staticEnv,
			Annotations:       annotations,
		}

		if _, _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil {
			log.Errorf("failed to create run: %+v", err)
			return err
		}
		return nil
	}

	for _, run := range config.Runs {
		rcts := runconfig.GenRunConfigTasks(util.DefaultUUIDGenerator{}, config, run.Name, variables, webhookData.Branch, webhookData.Tag, webhookData.Ref)

		h.log.Debugf("rcts: %s", util.Dump(rcts))
		h.log.Infof("group: %s", group)
		createRunReq := &rsapi.RunCreateRequest{
			RunConfigTasks:    rcts,
			Group:             group,
			SetupErrors:       setupErrors,
			Name:              run.Name,
			StaticEnvironment: staticEnv,
			Annotations:       annotations,
		}

		if _, _, err := h.runserviceClient.CreateRun(ctx, createRunReq); err != nil {
			log.Errorf("failed to create run: %+v", err)
			return err
		}
	}

	return nil
}