Add initial git sources implementations

* gitea
* gitlab
This commit is contained in:
Simone Gotti 2019-02-21 16:06:34 +01:00
parent 57c756a5a4
commit ba00398009
5 changed files with 821 additions and 0 deletions

View File

@ -0,0 +1,218 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package gitea
import (
"crypto/tls"
"net/http"
"net/url"
"strconv"
gitsource "github.com/sorintlab/agola/internal/gitsources"
"code.gitea.io/sdk/gitea"
"github.com/pkg/errors"
)
const (
// TODO(sgotti) The gitea client doesn't provide an easy way to detect http response codes...
// we should probably use our own client implementation
ClientNotFound = "404 Not Found"
)
type Opts struct {
URL string
Token string
SkipVerify bool
}
type Client struct {
client *gitea.Client
}
// fromCommitStatus converts a gitsource commit status to a gitea commit status
func fromCommitStatus(status gitsource.CommitStatus) gitea.StatusState {
switch status {
case gitsource.CommitStatusPending:
return gitea.StatusPending
case gitsource.CommitStatusSuccess:
return gitea.StatusSuccess
case gitsource.CommitStatusFailed:
return gitea.StatusFailure
default:
return gitea.StatusError
}
}
func New(opts Opts) (*Client, error) {
client := gitea.NewClient(opts.URL, opts.Token)
if opts.SkipVerify {
httpClient := &http.Client{}
httpClient.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client.SetHTTPClient(httpClient)
}
return &Client{
client: client,
}, nil
}
func (c *Client) LoginPassword(username, password string) (string, error) {
// try to get agola access token if it already exists
var accessToken string
tokens, err := c.client.ListAccessTokens(username, password)
if err == nil {
for _, token := range tokens {
if token.Name == "agola" {
accessToken = token.Sha1
break
}
}
}
// create access token
if accessToken == "" {
token, terr := c.client.CreateAccessToken(
username,
password,
gitea.CreateAccessTokenOption{Name: "agola"},
)
if terr != nil {
return "", terr
}
accessToken = token.Sha1
}
return accessToken, nil
}
func (c *Client) GetUserInfo() (*gitsource.UserInfo, error) {
user, err := c.client.GetMyUserInfo()
if err != nil {
return nil, err
}
return &gitsource.UserInfo{
ID: strconv.FormatInt(user.ID, 10),
LoginName: user.UserName,
Email: user.Email,
}, nil
}
func (c *Client) GetFile(owner, repo, commit, file string) ([]byte, error) {
data, err := c.client.GetFile(owner, repo, commit, file)
return data, err
}
func (c *Client) CreateDeployKey(owner, repo, title, pubKey string, readonly bool) error {
_, err := c.client.CreateDeployKey(owner, repo, gitea.CreateKeyOption{
Title: title,
Key: pubKey,
ReadOnly: readonly,
})
return errors.Wrapf(err, "error creating deploy key")
}
func (c *Client) UpdateDeployKey(owner, repo, title, pubKey string, readonly bool) error {
// NOTE(sgotti) gitea has a bug where if we delete and remove the same key with
// the same value it is correctly readded and the admin must force a
// authorizec_keys regeneration on the server. To avoid this we update it only
// when the public key value has changed
keys, err := c.client.ListDeployKeys(owner, repo)
if err != nil {
return errors.Wrapf(err, "error retrieving existing deploy keys")
}
for _, key := range keys {
if key.Title == title {
if key.Key == pubKey {
return nil
}
if err := c.client.DeleteDeployKey(owner, repo, key.ID); err != nil {
return errors.Wrapf(err, "error removing existing deploy key")
}
}
}
if _, err := c.client.CreateDeployKey(owner, repo, gitea.CreateKeyOption{
Title: title,
Key: pubKey,
ReadOnly: readonly,
}); err != nil {
return errors.Wrapf(err, "error creating deploy key")
}
return nil
}
func (c *Client) DeleteDeployKey(owner, repo, title string) error {
keys, err := c.client.ListDeployKeys(owner, repo)
if err != nil {
return errors.Wrapf(err, "error retrieving existing deploy keys")
}
for _, key := range keys {
if key.Title == title {
if err := c.client.DeleteDeployKey(owner, repo, key.ID); err != nil {
return errors.Wrapf(err, "error removing existing deploy key")
}
}
}
return nil
}
func (c *Client) CreateRepoWebhook(owner, repo, url, secret string) error {
opts := gitea.CreateHookOption{
Type: "gitea",
Config: map[string]string{
"url": url,
"content_type": "json",
"secret": secret,
},
Events: []string{"push", "pull_request"},
Active: true,
}
_, err := c.client.CreateRepoHook(owner, repo, opts)
return errors.Wrapf(err, "error creating repository webhook")
}
func (c *Client) DeleteRepoWebhook(owner, repo, u string) error {
curURL, err := url.Parse(u)
if err != nil {
return errors.Wrapf(err, "failed to parse url")
}
hooks, err := c.client.ListRepoHooks(owner, repo)
if err != nil {
return errors.Wrapf(err, "error retrieving repository webhooks")
}
for _, hook := range hooks {
if hurl, ok := hook.Config["url"]; ok {
u, err := url.Parse(hurl)
if err == nil && u.Host == curURL.Host {
if err := c.client.DeleteRepoHook(owner, repo, hook.ID); err != nil {
return errors.Wrapf(err, "error deleting existing repository webhook")
}
}
}
}
return nil
}

View File

@ -0,0 +1,164 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package gitea
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/sorintlab/agola/internal/services/types"
"github.com/pkg/errors"
)
const (
hookEvent = "X-Gitea-Event"
hookPush = "push"
hookPullRequest = "pull_request"
prStateOpen = "open"
prActionOpen = "opened"
prActionSync = "synchronized"
)
func (c *Client) ParseWebhook(r *http.Request) (*types.WebhookData, error) {
return parseWebhook(r)
}
func parseWebhook(r *http.Request) (*types.WebhookData, error) {
switch r.Header.Get(hookEvent) {
case hookPush:
return parsePushHook(r.Body)
case hookPullRequest:
return parsePullRequestHook(r.Body)
default:
return nil, errors.Errorf("unknown webhook event type: %q", r.Header.Get(hookEvent))
}
}
func parsePush(r io.Reader) (*pushHook, error) {
push := new(pushHook)
err := json.NewDecoder(r).Decode(push)
return push, err
}
func parsePullRequest(r io.Reader) (*pullRequestHook, error) {
pr := new(pullRequestHook)
err := json.NewDecoder(r).Decode(pr)
return pr, err
}
func parsePushHook(payload io.Reader) (*types.WebhookData, error) {
push, err := parsePush(payload)
if err != nil {
return nil, err
}
return webhookDataFromPush(push)
}
func parsePullRequestHook(payload io.Reader) (*types.WebhookData, error) {
prhook, err := parsePullRequest(payload)
if err != nil {
return nil, err
}
// skip non open pull requests
if prhook.PullRequest.State != prStateOpen {
return nil, nil
}
// only accept actions that have new commits
if prhook.Action != prActionOpen && prhook.Action != prActionSync {
return nil, nil
}
return webhookDataFromPullRequest(prhook), nil
}
func webhookDataFromPush(hook *pushHook) (*types.WebhookData, error) {
sender := hook.Sender.Username
if sender == "" {
sender = hook.Sender.Login
}
// common data
whd := &types.WebhookData{
CommitSHA: hook.After,
Ref: hook.Ref,
CompareLink: hook.Compare,
CommitLink: fmt.Sprintf("%s/commit/%s", hook.Repo.URL, hook.After),
Sender: sender,
Repo: types.WebhookDataRepo{
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,
RepoURL: hook.Repo.URL,
},
}
switch {
case strings.HasPrefix(hook.Ref, "refs/heads/"):
whd.Event = types.WebhookEventPush
whd.Branch = strings.TrimPrefix(hook.Ref, "refs/heads/")
whd.BranchLink = fmt.Sprintf("%s/src/branch/%s", hook.Repo.URL, whd.Branch)
if len(hook.Commits) > 0 {
whd.Message = hook.Commits[0].Message
}
case strings.HasPrefix(hook.Ref, "refs/tags/"):
whd.Event = types.WebhookEventTag
whd.Tag = strings.TrimPrefix(hook.Ref, "refs/tags/")
whd.TagLink = fmt.Sprintf("%s/src/tag/%s", hook.Repo.URL, whd.Tag)
whd.Message = fmt.Sprintf("Tag %s", whd.Tag)
default:
// ignore received webhook since it doesn't have a ref we're interested in
return nil, fmt.Errorf("Unsupported webhook ref %q", hook.Ref)
}
return whd, nil
}
// helper function that extracts the Build data from a Gitea pull_request hook
func webhookDataFromPullRequest(hook *pullRequestHook) *types.WebhookData {
sender := hook.Sender.Username
if sender == "" {
sender = hook.Sender.Login
}
build := &types.WebhookData{
Event: types.WebhookEventPullRequest,
CommitSHA: hook.PullRequest.Head.Sha,
Ref: fmt.Sprintf("refs/pull/%d/head", hook.Number),
CommitLink: fmt.Sprintf("%s/commit/%s", hook.Repo.URL, hook.PullRequest.Head.Sha),
Branch: hook.PullRequest.Base.Ref,
Message: hook.PullRequest.Title,
Sender: sender,
PullRequestID: strconv.FormatInt(hook.PullRequest.ID, 10),
PullRequestLink: hook.PullRequest.URL,
Repo: types.WebhookDataRepo{
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,
RepoURL: hook.Repo.URL,
},
}
return build
}

View File

@ -0,0 +1,140 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package gitea
type pushHook struct {
Sha string `json:"sha"`
Ref string `json:"ref"`
Before string `json:"before"`
After string `json:"after"`
Compare string `json:"compare_url"`
RefType string `json:"ref_type"`
Pusher struct {
Name string `json:"name"`
Email string `json:"email"`
Login string `json:"login"`
Username string `json:"username"`
} `json:"pusher"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
Name string `json:"name"`
Email string `json:"email"`
Username string `json:"username"`
} `json:"owner"`
} `json:"repository"`
Commits []struct {
ID string `json:"id"`
Message string `json:"message"`
URL string `json:"url"`
} `json:"commits"`
Sender struct {
ID int64 `json:"id"`
Login string `json:"login"`
Username string `json:"username"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"sender"`
}
type pullRequestHook struct {
Action string `json:"action"`
Number int64 `json:"number"`
PullRequest struct {
ID int64 `json:"id"`
User struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"user"`
Title string `json:"title"`
Body string `json:"body"`
State string `json:"state"`
URL string `json:"html_url"`
Mergeable bool `json:"mergeable"`
Merged bool `json:"merged"`
MergeBase string `json:"merge_base"`
Base struct {
Label string `json:"label"`
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repo"`
} `json:"base"`
Head struct {
Label string `json:"label"`
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repo"`
} `json:"head"`
} `json:"pull_request"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repository"`
Sender struct {
ID int64 `json:"id"`
Login string `json:"login"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"sender"`
}

View File

@ -0,0 +1,235 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package gitlab
import (
"context"
"crypto/tls"
"encoding/base64"
"fmt"
"net"
"net/http"
"net/url"
"path"
"strconv"
"time"
"github.com/pkg/errors"
gitsource "github.com/sorintlab/agola/internal/gitsources"
"github.com/sorintlab/agola/internal/services/types"
gitlab "github.com/xanzy/go-gitlab"
"golang.org/x/oauth2"
)
const (
// TODO(sgotti) The gitea client doesn't provide an easy way to detect http response codes...
// we should probably use our own client implementation
ClientNotFound = "404 Not Found"
)
var (
GitlabOauth2Scopes = []string{"api"}
)
type Opts struct {
URL string
Token string
SkipVerify bool
Oauth2ClientID string
Oauth2Secret string
}
type Client struct {
client *gitlab.Client
URL string
oauth2ClientID string
oauth2Secret string
}
func New(opts Opts) (*Client, error) {
// copied from net/http until it has a clone function: https://github.com/golang/go/issues/26013
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: opts.SkipVerify},
}
httpClient := &http.Client{Transport: transport}
client := gitlab.NewOAuthClient(httpClient, opts.Token)
client.SetBaseURL(opts.URL)
return &Client{
client: client,
URL: opts.URL,
oauth2ClientID: opts.Oauth2ClientID,
oauth2Secret: opts.Oauth2Secret,
}, nil
}
func (c *Client) GetOauth2AuthorizationURL(callbackURL, state string) (string, error) {
var config = &oauth2.Config{
ClientID: c.oauth2ClientID,
ClientSecret: c.oauth2Secret,
Scopes: GitlabOauth2Scopes,
Endpoint: oauth2.Endpoint{
AuthURL: fmt.Sprintf("%s/oauth/authorize", c.URL),
TokenURL: fmt.Sprintf("%s/oauth/token", c.URL),
},
RedirectURL: callbackURL,
}
return config.AuthCodeURL(state), nil
}
func (c *Client) RequestOauth2Token(callbackURL, code string) (*oauth2.Token, error) {
var config = &oauth2.Config{
ClientID: c.oauth2ClientID,
ClientSecret: c.oauth2Secret,
Scopes: GitlabOauth2Scopes,
Endpoint: oauth2.Endpoint{
AuthURL: fmt.Sprintf("%s/oauth/authorize", c.URL),
TokenURL: fmt.Sprintf("%s/oauth/token", c.URL),
},
RedirectURL: callbackURL,
}
token, err := config.Exchange(context.TODO(), code)
if err != nil {
return nil, errors.Wrapf(err, "cannot get oauth2 token")
}
return token, nil
}
func (c *Client) GetUserInfo() (*gitsource.UserInfo, error) {
user, _, err := c.client.Users.CurrentUser()
if err != nil {
return nil, err
}
return &gitsource.UserInfo{
ID: strconv.Itoa(user.ID),
LoginName: user.Username,
Email: user.Email,
}, nil
}
func (c *Client) GetFile(owner, repo, commit, file string) ([]byte, error) {
f, _, err := c.client.RepositoryFiles.GetFile(path.Join(owner, repo), file, &gitlab.GetFileOptions{Ref: gitlab.String(commit)})
data, err := base64.StdEncoding.DecodeString(f.Content)
if err != nil {
return nil, err
}
return data, err
}
func (c *Client) CreateDeployKey(owner, repo, title, pubKey string, readonly bool) error {
_, _, err := c.client.DeployKeys.AddDeployKey(path.Join(owner, repo), &gitlab.AddDeployKeyOptions{
Title: gitlab.String(title),
Key: gitlab.String(pubKey),
})
return errors.Wrapf(err, "error creating deploy key")
}
func (c *Client) UpdateDeployKey(owner, repo, title, pubKey string, readonly bool) error {
keys, _, err := c.client.DeployKeys.ListProjectDeployKeys(path.Join(owner, repo), nil)
if err != nil {
return errors.Wrapf(err, "error retrieving existing deploy keys")
}
for _, key := range keys {
if key.Title == title {
if key.Key == pubKey {
return nil
}
if _, err := c.client.DeployKeys.DeleteDeployKey(path.Join(owner, repo), key.ID); err != nil {
return errors.Wrapf(err, "error removing existing deploy key")
}
}
}
if _, _, err := c.client.DeployKeys.AddDeployKey(path.Join(owner, repo), &gitlab.AddDeployKeyOptions{
Title: &title,
Key: &pubKey,
}); err != nil {
return errors.Wrapf(err, "error creating deploy key")
}
return nil
}
func (c *Client) DeleteDeployKey(owner, repo, title string) error {
keys, _, err := c.client.DeployKeys.ListProjectDeployKeys(path.Join(owner, repo), nil)
if err != nil {
return errors.Wrapf(err, "error retrieving existing deploy keys")
}
for _, key := range keys {
if key.Title == title {
if _, err := c.client.DeployKeys.DeleteDeployKey(path.Join(owner, repo), key.ID); err != nil {
return errors.Wrapf(err, "error removing existing deploy key")
}
}
}
return nil
}
func (c *Client) CreateRepoWebhook(owner, repo, url, secret string) error {
opts := &gitlab.AddProjectHookOptions{
URL: gitlab.String(url),
PushEvents: gitlab.Bool(true),
}
_, _, err := c.client.Projects.AddProjectHook(path.Join(owner, repo), opts)
return errors.Wrapf(err, "error creating repository webhook")
}
func (c *Client) DeleteRepoWebhook(owner, repo, u string) error {
curURL, err := url.Parse(u)
if err != nil {
return errors.Wrapf(err, "failed to parse url")
}
hooks, _, err := c.client.Projects.ListProjectHooks(path.Join(owner, repo), nil)
if err != nil {
return errors.Wrapf(err, "error retrieving repository webhooks")
}
for _, hook := range hooks {
u, err := url.Parse(hook.URL)
if err == nil && u.Host == curURL.Host {
if _, err := c.client.Projects.DeleteProjectHook(path.Join(owner, repo), hook.ID); err != nil {
return errors.Wrapf(err, "error deleting existing repository webhook")
}
}
}
return nil
}
func (c *Client) ParseWebhook(r *http.Request) (*types.WebhookData, error) {
hookEvent := "X-Gitea-Event"
return nil, errors.Errorf("unknown webhook event type: %q", r.Header.Get(hookEvent))
}

View File

@ -0,0 +1,64 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package gitsource
import (
"net/http"
"github.com/sorintlab/agola/internal/services/types"
"golang.org/x/oauth2"
)
type CommitStatus string
const (
CommitStatusPending CommitStatus = "pending"
CommitStatusSuccess CommitStatus = "success"
CommitStatusFailed CommitStatus = "failed"
)
type GitSource interface {
GetFile(owner, repo, commit, file string) ([]byte, error)
DeleteDeployKey(owner, repo, title string) error
CreateDeployKey(owner, repo, title, pubKey string, readonly bool) error
UpdateDeployKey(owner, repo, title, pubKey string, readonly bool) error
DeleteRepoWebhook(owner, repo, url string) error
CreateRepoWebhook(owner, repo, url, secret string) error
ParseWebhook(r *http.Request) (*types.WebhookData, error)
}
type UserSource interface {
GetUserInfo() (*UserInfo, error)
}
type PasswordSource interface {
UserSource
LoginPassword(username, password string) (string, error)
}
type Oauth2Source interface {
UserSource
// Oauth2AuthorizationRequest return the authorization request URL to the
// authorization server
GetOauth2AuthorizationURL(callbackURL, state string) (redirectURL string, err error)
// OauthTokenRequest requests the oauth2 access token to the authorization server
RequestOauth2Token(callbackURL, code string) (*oauth2.Token, error)
}
type UserInfo struct {
ID string
LoginName string
Email string
}