From b22c197fefacb937df064059f37eb381c343413a Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Wed, 15 May 2019 23:46:21 +0200 Subject: [PATCH] gitsources: add github gitsource --- cmd/agola/cmd/remotesourcecreate.go | 8 + go.mod | 3 +- go.sum | 14 +- internal/gitsources/gitea/gitea.go | 14 +- internal/gitsources/github/github.go | 387 ++++++++++++++++++++++++++ internal/gitsources/github/parse.go | 135 +++++++++ internal/gitsources/gitlab/gitlab.go | 19 +- internal/services/common/gitsource.go | 19 +- 8 files changed, 574 insertions(+), 25 deletions(-) create mode 100644 internal/gitsources/github/github.go create mode 100644 internal/gitsources/github/parse.go diff --git a/cmd/agola/cmd/remotesourcecreate.go b/cmd/agola/cmd/remotesourcecreate.go index e10f00e..d0bd215 100644 --- a/cmd/agola/cmd/remotesourcecreate.go +++ b/cmd/agola/cmd/remotesourcecreate.go @@ -18,7 +18,9 @@ import ( "context" "github.com/pkg/errors" + "github.com/sorintlab/agola/internal/gitsources/github" "github.com/sorintlab/agola/internal/services/gateway/api" + "github.com/sorintlab/agola/internal/services/types" "github.com/spf13/cobra" ) @@ -69,6 +71,12 @@ func init() { func remoteSourceCreate(cmd *cobra.Command, args []string) error { gwclient := api.NewClient(gatewayURL, token) + // for github remote source type, set defaults for github.com + if remoteSourceCreateOpts.rsType == string(types.RemoteSourceTypeGithub) { + remoteSourceCreateOpts.apiURL = github.GitHubAPIURL + remoteSourceCreateOpts.sshHostKey = github.GitHubSSHHostKey + } + req := &api.CreateRemoteSourceRequest{ Name: remoteSourceCreateOpts.name, Type: remoteSourceCreateOpts.rsType, diff --git a/go.mod b/go.mod index dfee576..c36a7fe 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/google/go-cmp v0.3.0 github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d + github.com/google/go-github/v25 v25.0.4 github.com/google/go-jsonnet v0.12.1 github.com/google/gofuzz v1.0.0 // indirect github.com/googleapis/gnostic v0.2.0 // indirect @@ -58,7 +59,7 @@ require ( github.com/xanzy/go-gitlab v0.14.1 go.etcd.io/etcd v0.0.0-20181128220305-dedae6eb7c25 go.uber.org/zap v1.9.1 - golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.42.0 // indirect diff --git a/go.sum b/go.sum index acdcf5c..d310f2d 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d h1:K8AF5hFHsOYRk0CG22FwQk3oCu7CbL2bNfiHoaGuW4Y= github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= +github.com/google/go-github/v25 v25.0.4 h1:i/JXg8Et3dm4eD/u5VFB0tO6e9ICQ0zcUQavk5eSoSs= +github.com/google/go-github/v25 v25.0.4/go.mod h1:6z5pC69qHtrPJ0sXPsj4BLnd82b+r6sLB7qcBoRZqpw= github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU= github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -216,13 +218,16 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9 h1:pfyU+l9dEu0vZzDDMsdAKa1gZbJYEn6urYXj/+Xkz7s= golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -230,13 +235,18 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/gitsources/gitea/gitea.go b/internal/gitsources/gitea/gitea.go index d8bd153..0430446 100644 --- a/internal/gitsources/gitea/gitea.go +++ b/internal/gitsources/gitea/gitea.go @@ -47,7 +47,7 @@ var ( ) type Opts struct { - URL string + APIURL string Token string SkipVerify bool Oauth2ClientID string @@ -57,7 +57,7 @@ type Opts struct { type Client struct { client *gitea.Client httpClient *http.Client - URL string + APIURL string oauth2ClientID string oauth2Secret string } @@ -103,13 +103,13 @@ func New(opts Opts) (*Client, error) { } httpClient := &http.Client{Transport: transport} - client := gitea.NewClient(opts.URL, opts.Token) + client := gitea.NewClient(opts.APIURL, opts.Token) client.SetHTTPClient(httpClient) return &Client{ client: client, httpClient: httpClient, - URL: opts.URL, + APIURL: opts.APIURL, oauth2ClientID: opts.Oauth2ClientID, oauth2Secret: opts.Oauth2Secret, }, nil @@ -121,8 +121,8 @@ func (c *Client) oauth2Config(callbackURL string) *oauth2.Config { ClientSecret: c.oauth2Secret, Scopes: GiteaOauth2Scopes, Endpoint: oauth2.Endpoint{ - AuthURL: fmt.Sprintf("%s/login/oauth/authorize", c.URL), - TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.URL), + AuthURL: fmt.Sprintf("%s/login/oauth/authorize", c.APIURL), + TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.APIURL), }, RedirectURL: callbackURL, } @@ -156,7 +156,7 @@ func (c *Client) LoginPassword(username, password, tokenName string) (string, er var accessToken string tokens := make([]*gitea.AccessToken, 0, 10) - req, err := http.NewRequest("GET", c.URL+"/api/v1"+fmt.Sprintf("/users/%s/tokens", username), nil) + req, err := http.NewRequest("GET", c.APIURL+"/api/v1"+fmt.Sprintf("/users/%s/tokens", username), nil) if err != nil { return "", err } diff --git a/internal/gitsources/github/github.go b/internal/gitsources/github/github.go new file mode 100644 index 0000000..78a97ae --- /dev/null +++ b/internal/gitsources/github/github.go @@ -0,0 +1,387 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "context" + "crypto/tls" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + gitsource "github.com/sorintlab/agola/internal/gitsources" + + "github.com/google/go-github/v25/github" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +var ( + GitHubOauth2Scopes = []string{"repo"} +) + +const ( + GitHubAPIURL = "https://api.github.com" + GitHubWebURL = "https://github.com" + + GitHubSSHHostKey = "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" +) + +type Opts struct { + APIURL string + WebURL string + Token string + SkipVerify bool + Oauth2ClientID string + Oauth2Secret string +} + +type Client struct { + client *github.Client + httpClient *http.Client + APIURL string + WebURL string + oauth2ClientID string + oauth2Secret string +} + +// fromCommitStatus converts a gitsource commit status to a github commit status +func fromCommitStatus(status gitsource.CommitStatus) string { + switch status { + case gitsource.CommitStatusPending: + return "pending" + case gitsource.CommitStatusSuccess: + return "success" + case gitsource.CommitStatusError: + return "error" + case gitsource.CommitStatusFailed: + return "failure" + default: + panic(fmt.Errorf("unknown commit status %q", status)) + } +} + +func parseRepoPath(repopath string) (string, string, error) { + parts := strings.Split(repopath, "/") + if len(parts) != 2 { + return "", "", errors.Errorf("wrong github repo path: %q", repopath) + } + return parts[0], parts[1], nil +} + +type TokenTransport struct { + token string + rt http.RoundTripper +} + +func (t *TokenTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if t.token != "" { + r.Header.Set("Authorization", "Bearer "+t.token) + } + return t.rt.RoundTrip(r) +} + +func New(opts Opts) (*Client, error) { + // copied from net/http until it has a clone function: https://github.com/golang/go/issues/26013 + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: opts.SkipVerify}, + } + httpClient := &http.Client{Transport: &TokenTransport{token: opts.Token, rt: transport}} + + if opts.APIURL == GitHubAPIURL { + opts.WebURL = GitHubWebURL + } else { + if opts.WebURL == "" { + opts.WebURL = opts.APIURL + } + } + + client := github.NewClient(httpClient) + client.BaseURL, _ = url.Parse(GitHubAPIURL + "/") + + return &Client{ + client: client, + httpClient: httpClient, + APIURL: opts.APIURL, + WebURL: opts.WebURL, + oauth2ClientID: opts.Oauth2ClientID, + oauth2Secret: opts.Oauth2Secret, + }, nil +} + +func (c *Client) oauth2Config(callbackURL string) *oauth2.Config { + return &oauth2.Config{ + ClientID: c.oauth2ClientID, + ClientSecret: c.oauth2Secret, + Scopes: GitHubOauth2Scopes, + Endpoint: oauth2.Endpoint{ + AuthURL: fmt.Sprintf("%s/login/oauth/authorize", c.WebURL), + TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.WebURL), + }, + RedirectURL: callbackURL, + } +} + +func (c *Client) GetOauth2AuthorizationURL(callbackURL, state string) (string, error) { + var config = c.oauth2Config(callbackURL) + return config.AuthCodeURL(state), nil +} + +func (c *Client) RequestOauth2Token(callbackURL, code string) (*oauth2.Token, error) { + var config = c.oauth2Config(callbackURL) + token, err := config.Exchange(context.TODO(), code) + if err != nil { + return nil, errors.Wrapf(err, "cannot get oauth2 token") + } + return token, nil +} + +func (c *Client) RefreshOauth2Token(refreshToken string) (*oauth2.Token, error) { + var config = c.oauth2Config("") + token := &oauth2.Token{RefreshToken: refreshToken} + ts := config.TokenSource(context.TODO(), token) + return ts.Token() +} + +func (c *Client) GetUserInfo() (*gitsource.UserInfo, error) { + user, _, err := c.client.Users.Get(context.TODO(), "") + if err != nil { + return nil, err + } + return &gitsource.UserInfo{ + ID: strconv.FormatInt(*user.ID, 10), + LoginName: *user.Login, + Email: *user.Email, + }, nil +} + +func (c *Client) GetRepoInfo(repopath string) (*gitsource.RepoInfo, error) { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return nil, err + } + rr, _, err := c.client.Repositories.Get(context.TODO(), owner, reponame) + if err != nil { + return nil, err + } + return fromGithubRepo(rr), nil +} + +func (c *Client) GetFile(repopath, commit, file string) ([]byte, error) { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return nil, err + } + r, err := c.client.Repositories.DownloadContents(context.TODO(), owner, reponame, file, &github.RepositoryContentGetOptions{Ref: commit}) + if err != nil { + return nil, err + } + defer r.Close() + + return ioutil.ReadAll(r) +} + +func (c *Client) CreateDeployKey(repopath, title, pubKey string, readonly bool) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + if _, _, err = c.client.Repositories.CreateKey(context.TODO(), owner, reponame, &github.Key{ + Title: github.String(title), + Key: github.String(pubKey), + ReadOnly: github.Bool(readonly), + }); err != nil { + return errors.Wrapf(err, "error creating deploy key") + } + return nil +} + +func (c *Client) UpdateDeployKey(repopath, title, pubKey string, readonly bool) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + // NOTE(sgotti) gitea has a bug where if we delete and remove the same key with + // the same value it is correctly readded and the admin must force a + // authorized_keys regeneration on the server. To avoid this we update it only + // when the public key value has changed + keys, _, err := c.client.Repositories.ListKeys(context.TODO(), owner, reponame, nil) + if err != nil { + return errors.Wrapf(err, "error retrieving existing deploy keys") + } + + for _, key := range keys { + if *key.Title == title { + if *key.Key == pubKey { + return nil + } + if _, err := c.client.Repositories.DeleteKey(context.TODO(), owner, reponame, *key.ID); err != nil { + return errors.Wrapf(err, "error removing existing deploy key") + } + } + } + + if _, _, err = c.client.Repositories.CreateKey(context.TODO(), owner, reponame, &github.Key{ + Title: github.String(title), + Key: github.String(pubKey), + ReadOnly: github.Bool(readonly), + }); err != nil { + return errors.Wrapf(err, "error creating deploy key") + } + + return nil +} + +func (c *Client) DeleteDeployKey(repopath, title string) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + keys, _, err := c.client.Repositories.ListKeys(context.TODO(), owner, reponame, nil) + if err != nil { + return errors.Wrapf(err, "error retrieving existing deploy keys") + } + + for _, key := range keys { + if *key.Title == title { + if _, err := c.client.Repositories.DeleteKey(context.TODO(), owner, reponame, *key.ID); err != nil { + return errors.Wrapf(err, "error removing existing deploy key") + } + } + } + + return nil +} + +func (c *Client) CreateRepoWebhook(repopath, url, secret string) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + + hook := &github.Hook{ + Config: map[string]interface{}{ + "url": url, + "content_type": "json", + "secret": secret, + }, + Events: []string{"push", "pull_request"}, + Active: github.Bool(true), + } + + if _, _, err = c.client.Repositories.CreateHook(context.TODO(), owner, reponame, hook); err != nil { + return errors.Wrapf(err, "error creating repository webhook") + } + + return nil +} + +func (c *Client) DeleteRepoWebhook(repopath, u string) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + + hooks := []*github.Hook{} + + opt := &github.ListOptions{} + for { + pHooks, resp, err := c.client.Repositories.ListHooks(context.TODO(), owner, reponame, opt) + if err != nil { + return errors.Wrapf(err, "error retrieving repository webhooks") + } + hooks = append(hooks, pHooks...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + // match the full url so we can have multiple webhooks for different agola + // projects + for _, hook := range hooks { + if hook.Config["url"] == u { + if _, err := c.client.Repositories.DeleteHook(context.TODO(), owner, reponame, *hook.ID); err != nil { + return errors.Wrapf(err, "error deleting existing repository webhook") + } + } + } + + return nil +} + +func (c *Client) CreateCommitStatus(repopath, commitSHA string, status gitsource.CommitStatus, targetURL, description, statusContext string) error { + owner, reponame, err := parseRepoPath(repopath) + if err != nil { + return err + } + _, _, err = c.client.Repositories.CreateStatus(context.TODO(), owner, reponame, commitSHA, &github.RepoStatus{ + State: github.String(fromCommitStatus(status)), + TargetURL: github.String(targetURL), + Description: github.String(description), + Context: github.String(statusContext), + }) + return err +} + +func (c *Client) ListUserRepos() ([]*gitsource.RepoInfo, error) { + remoteRepos := []*github.Repository{} + + opt := &github.RepositoryListOptions{} + for { + pRemoteRepos, resp, err := c.client.Repositories.List(context.TODO(), "", opt) + if err != nil { + return nil, err + } + remoteRepos = append(remoteRepos, pRemoteRepos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + repos := []*gitsource.RepoInfo{} + + for _, rr := range remoteRepos { + repos = append(repos, fromGithubRepo(rr)) + } + + return repos, nil +} + +func fromGithubRepo(rr *github.Repository) *gitsource.RepoInfo { + return &gitsource.RepoInfo{ + ID: strconv.FormatInt(*rr.ID, 10), + Path: path.Join(*rr.Owner.Login, *rr.Name), + SSHCloneURL: *rr.SSHURL, + HTTPCloneURL: *rr.CloneURL, + } +} diff --git a/internal/gitsources/github/parse.go b/internal/gitsources/github/parse.go new file mode 100644 index 0000000..8594dd0 --- /dev/null +++ b/internal/gitsources/github/parse.go @@ -0,0 +1,135 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package github + +import ( + "fmt" + "net/http" + "path" + "strconv" + "strings" + + "github.com/google/go-github/v25/github" + "github.com/sorintlab/agola/internal/services/types" + + "github.com/pkg/errors" +) + +const ( + hookPush = "push" + hookPullRequest = "pull_request" + + prStateOpen = "open" + + prActionOpen = "opened" + prActionSync = "synchronize" +) + +func (c *Client) ParseWebhook(r *http.Request, secret string) (*types.WebhookData, error) { + payload, err := github.ValidatePayload(r, []byte(secret)) + if err != nil { + return nil, errors.Wrapf(err, "wrong webhook signature") + } + webHookType := github.WebHookType(r) + event, err := github.ParseWebHook(webHookType, payload) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse webhook") + } + switch event := event.(type) { + case *github.PushEvent: + return webhookDataFromPush(event) + case *github.PullRequestEvent: + return webhookDataFromPullRequest(event) + default: + return nil, errors.Errorf("unknown webhook event type: %q", webHookType) + } +} + +func webhookDataFromPush(hook *github.PushEvent) (*types.WebhookData, error) { + sender := hook.Sender.Name + if sender == nil { + sender = hook.Sender.Login + } + + // common data + whd := &types.WebhookData{ + CommitSHA: *hook.After, + SSHURL: *hook.Repo.SSHURL, + Ref: *hook.Ref, + CompareLink: *hook.Compare, + CommitLink: fmt.Sprintf("%s/commit/%s", *hook.Repo.HTMLURL, *hook.After), + Sender: *sender, + + Repo: types.WebhookDataRepo{ + Path: path.Join(*hook.Repo.Owner.Name, *hook.Repo.Name), + WebURL: *hook.Repo.HTMLURL, + }, + } + + switch { + case strings.HasPrefix(*hook.Ref, "refs/heads/"): + whd.Event = types.WebhookEventPush + whd.Branch = strings.TrimPrefix(*hook.Ref, "refs/heads/") + whd.BranchLink = fmt.Sprintf("%s/tree/%s", *hook.Repo.HTMLURL, whd.Branch) + whd.Message = *hook.HeadCommit.Message + + case strings.HasPrefix(*hook.Ref, "refs/tags/"): + whd.Event = types.WebhookEventTag + whd.Tag = strings.TrimPrefix(*hook.Ref, "refs/tags/") + whd.TagLink = fmt.Sprintf("%s/tree/%s", *hook.Repo.HTMLURL, whd.Tag) + whd.Message = fmt.Sprintf("Tag %s", whd.Tag) + default: + // ignore received webhook since it doesn't have a ref we're interested in + return nil, fmt.Errorf("unsupported webhook ref %q", *hook.Ref) + } + + return whd, nil +} + +func webhookDataFromPullRequest(hook *github.PullRequestEvent) (*types.WebhookData, error) { + // skip non open pull requests + if *hook.PullRequest.State != prStateOpen { + return nil, nil + } + // only accept actions that have new commits + if *hook.Action != prActionOpen && *hook.Action != prActionSync { + return nil, nil + } + + sender := hook.Sender.Name + if sender == nil { + sender = hook.Sender.Login + } + + whd := &types.WebhookData{ + Event: types.WebhookEventPullRequest, + CommitSHA: *hook.PullRequest.Head.SHA, + SSHURL: *hook.Repo.SSHURL, + Ref: fmt.Sprintf("refs/pull/%d/head", *hook.Number), + CommitLink: fmt.Sprintf("%s/commit/%s", *hook.Repo.HTMLURL, *hook.PullRequest.Head.SHA), + Branch: *hook.PullRequest.Base.Ref, + Message: *hook.PullRequest.Title, + Sender: *sender, + PullRequestID: strconv.Itoa(*hook.PullRequest.Number), + PullRequestLink: *hook.PullRequest.HTMLURL, + + Repo: types.WebhookDataRepo{ + Path: path.Join(*hook.Repo.Owner.Login, *hook.Repo.Name), + WebURL: *hook.Repo.HTMLURL, + }, + } + + return whd, nil +} diff --git a/internal/gitsources/gitlab/gitlab.go b/internal/gitsources/gitlab/gitlab.go index dd008d3..66c1349 100644 --- a/internal/gitsources/gitlab/gitlab.go +++ b/internal/gitsources/gitlab/gitlab.go @@ -30,19 +30,12 @@ import ( "golang.org/x/oauth2" ) -const ( - // TODO(sgotti) The gitea client doesn't provide an easy way to detect http response codes... - // we should probably use our own client implementation - - ClientNotFound = "404 Not Found" -) - var ( GitlabOauth2Scopes = []string{"api"} ) type Opts struct { - URL string + APIURL string Token string SkipVerify bool Oauth2ClientID string @@ -51,7 +44,7 @@ type Opts struct { type Client struct { client *gitlab.Client - URL string + APIURL string oauth2ClientID string oauth2Secret string } @@ -89,11 +82,11 @@ func New(opts Opts) (*Client, error) { } httpClient := &http.Client{Transport: transport} client := gitlab.NewOAuthClient(httpClient, opts.Token) - client.SetBaseURL(opts.URL) + client.SetBaseURL(opts.APIURL) return &Client{ client: client, - URL: opts.URL, + APIURL: opts.APIURL, oauth2ClientID: opts.Oauth2ClientID, oauth2Secret: opts.Oauth2Secret, }, nil @@ -105,8 +98,8 @@ func (c *Client) oauth2Config(callbackURL string) *oauth2.Config { ClientSecret: c.oauth2Secret, Scopes: GitlabOauth2Scopes, Endpoint: oauth2.Endpoint{ - AuthURL: fmt.Sprintf("%s/oauth/authorize", c.URL), - TokenURL: fmt.Sprintf("%s/oauth/token", c.URL), + AuthURL: fmt.Sprintf("%s/oauth/authorize", c.APIURL), + TokenURL: fmt.Sprintf("%s/oauth/token", c.APIURL), }, RedirectURL: callbackURL, } diff --git a/internal/services/common/gitsource.go b/internal/services/common/gitsource.go index 135571c..80101a3 100644 --- a/internal/services/common/gitsource.go +++ b/internal/services/common/gitsource.go @@ -17,6 +17,7 @@ package common import ( gitsource "github.com/sorintlab/agola/internal/gitsources" "github.com/sorintlab/agola/internal/gitsources/gitea" + "github.com/sorintlab/agola/internal/gitsources/github" "github.com/sorintlab/agola/internal/gitsources/gitlab" "github.com/sorintlab/agola/internal/services/types" @@ -25,7 +26,7 @@ import ( func newGitea(rs *types.RemoteSource, accessToken string) (*gitea.Client, error) { return gitea.New(gitea.Opts{ - URL: rs.APIURL, + APIURL: rs.APIURL, SkipVerify: rs.SkipVerify, Token: accessToken, Oauth2ClientID: rs.Oauth2ClientID, @@ -35,7 +36,17 @@ func newGitea(rs *types.RemoteSource, accessToken string) (*gitea.Client, error) func newGitlab(rs *types.RemoteSource, accessToken string) (*gitlab.Client, error) { return gitlab.New(gitlab.Opts{ - URL: rs.APIURL, + APIURL: rs.APIURL, + SkipVerify: rs.SkipVerify, + Token: accessToken, + Oauth2ClientID: rs.Oauth2ClientID, + Oauth2Secret: rs.Oauth2ClientSecret, + }) +} + +func newGithub(rs *types.RemoteSource, accessToken string) (*github.Client, error) { + return github.New(github.Opts{ + APIURL: rs.APIURL, SkipVerify: rs.SkipVerify, Token: accessToken, Oauth2ClientID: rs.Oauth2ClientID, @@ -71,6 +82,8 @@ func GetGitSource(rs *types.RemoteSource, la *types.LinkedAccount) (gitsource.Gi gitSource, err = newGitea(rs, accessToken) case types.RemoteSourceTypeGitlab: gitSource, err = newGitlab(rs, accessToken) + case types.RemoteSourceTypeGithub: + gitSource, err = newGithub(rs, accessToken) default: return nil, errors.Errorf("remote source %s isn't a valid git source", rs.Name) } @@ -101,6 +114,8 @@ func GetOauth2Source(rs *types.RemoteSource, accessToken string) (gitsource.Oaut oauth2Source, err = newGitea(rs, accessToken) case types.RemoteSourceTypeGitlab: oauth2Source, err = newGitlab(rs, accessToken) + case types.RemoteSourceTypeGithub: + oauth2Source, err = newGithub(rs, accessToken) default: return nil, errors.Errorf("remote source %s isn't a valid oauth2 source", rs.Name) }