gitsources: add github gitsource

cmd/agola/cmd/remotesourcecreate.go

@@ -18,7 +18,9 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/gitsources/github"
 	"github.com/sorintlab/agola/internal/services/gateway/api"
+	"github.com/sorintlab/agola/internal/services/types"
 
 	"github.com/spf13/cobra"
 )
@@ -69,6 +71,12 @@ func init() {
 func remoteSourceCreate(cmd *cobra.Command, args []string) error {
 	gwclient := api.NewClient(gatewayURL, token)
 
+	// for github remote source type, set defaults for github.com
+	if remoteSourceCreateOpts.rsType == string(types.RemoteSourceTypeGithub) {
+		remoteSourceCreateOpts.apiURL = github.GitHubAPIURL
+		remoteSourceCreateOpts.sshHostKey = github.GitHubSSHHostKey
+	}
+
 	req := &api.CreateRemoteSourceRequest{
 		Name:                remoteSourceCreateOpts.name,
 		Type:                remoteSourceCreateOpts.rsType,

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d
+	github.com/google/go-github/v25 v25.0.4
 	github.com/google/go-jsonnet v0.12.1
 	github.com/google/gofuzz v1.0.0 // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
@@ -58,7 +59,7 @@ require (
 	github.com/xanzy/go-gitlab v0.14.1
 	go.etcd.io/etcd v0.0.0-20181128220305-dedae6eb7c25
 	go.uber.org/zap v1.9.1
-	golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 	golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect

go.sum

@@ -72,6 +72,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d h1:K8AF5hFHsOYRk0CG22FwQk3oCu7CbL2bNfiHoaGuW4Y=
 github.com/google/go-containerregistry v0.0.0-20190412005658-1d38b9cfdb9d/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk=
+github.com/google/go-github/v25 v25.0.4 h1:i/JXg8Et3dm4eD/u5VFB0tO6e9ICQ0zcUQavk5eSoSs=
+github.com/google/go-github/v25 v25.0.4/go.mod h1:6z5pC69qHtrPJ0sXPsj4BLnd82b+r6sLB7qcBoRZqpw=
 github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU=
 github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -216,13 +218,16 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9 h1:pfyU+l9dEu0vZzDDMsdAKa1gZbJYEn6urYXj/+Xkz7s=
 golang.org/x/oauth2 v0.0.0-20190220154721-9b3c75971fc9/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -230,13 +235,18 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

internal/gitsources/gitea/gitea.go

@@ -47,7 +47,7 @@ var (
 )
 
 type Opts struct {
-	URL            string
+	APIURL         string
 	Token          string
 	SkipVerify     bool
 	Oauth2ClientID string
@@ -57,7 +57,7 @@ type Opts struct {
 type Client struct {
 	client         *gitea.Client
 	httpClient     *http.Client
-	URL            string
+	APIURL         string
 	oauth2ClientID string
 	oauth2Secret   string
 }
@@ -103,13 +103,13 @@ func New(opts Opts) (*Client, error) {
 	}
 	httpClient := &http.Client{Transport: transport}
 
-	client := gitea.NewClient(opts.URL, opts.Token)
+	client := gitea.NewClient(opts.APIURL, opts.Token)
 	client.SetHTTPClient(httpClient)
 
 	return &Client{
 		client:         client,
 		httpClient:     httpClient,
-		URL:            opts.URL,
+		APIURL:         opts.APIURL,
 		oauth2ClientID: opts.Oauth2ClientID,
 		oauth2Secret:   opts.Oauth2Secret,
 	}, nil
@@ -121,8 +121,8 @@ func (c *Client) oauth2Config(callbackURL string) *oauth2.Config {
 		ClientSecret: c.oauth2Secret,
 		Scopes:       GiteaOauth2Scopes,
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  fmt.Sprintf("%s/login/oauth/authorize", c.URL),
+			AuthURL:  fmt.Sprintf("%s/login/oauth/authorize", c.APIURL),
-			TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.URL),
+			TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.APIURL),
 		},
 		RedirectURL: callbackURL,
 	}
@@ -156,7 +156,7 @@ func (c *Client) LoginPassword(username, password, tokenName string) (string, er
 	var accessToken string
 
 	tokens := make([]*gitea.AccessToken, 0, 10)
-	req, err := http.NewRequest("GET", c.URL+"/api/v1"+fmt.Sprintf("/users/%s/tokens", username), nil)
+	req, err := http.NewRequest("GET", c.APIURL+"/api/v1"+fmt.Sprintf("/users/%s/tokens", username), nil)
 	if err != nil {
 		return "", err
 	}

internal/gitsources/github/github.go

@@ -0,0 +1,387 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	gitsource "github.com/sorintlab/agola/internal/gitsources"
+
+	"github.com/google/go-github/v25/github"
+	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
+)
+
+var (
+	GitHubOauth2Scopes = []string{"repo"}
+)
+
+const (
+	GitHubAPIURL = "https://api.github.com"
+	GitHubWebURL = "https://github.com"
+
+	GitHubSSHHostKey = "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+)
+
+type Opts struct {
+	APIURL         string
+	WebURL         string
+	Token          string
+	SkipVerify     bool
+	Oauth2ClientID string
+	Oauth2Secret   string
+}
+
+type Client struct {
+	client         *github.Client
+	httpClient     *http.Client
+	APIURL         string
+	WebURL         string
+	oauth2ClientID string
+	oauth2Secret   string
+}
+
+// fromCommitStatus converts a gitsource commit status to a github commit status
+func fromCommitStatus(status gitsource.CommitStatus) string {
+	switch status {
+	case gitsource.CommitStatusPending:
+		return "pending"
+	case gitsource.CommitStatusSuccess:
+		return "success"
+	case gitsource.CommitStatusError:
+		return "error"
+	case gitsource.CommitStatusFailed:
+		return "failure"
+	default:
+		panic(fmt.Errorf("unknown commit status %q", status))
+	}
+}
+
+func parseRepoPath(repopath string) (string, string, error) {
+	parts := strings.Split(repopath, "/")
+	if len(parts) != 2 {
+		return "", "", errors.Errorf("wrong github repo path: %q", repopath)
+	}
+	return parts[0], parts[1], nil
+}
+
+type TokenTransport struct {
+	token string
+	rt    http.RoundTripper
+}
+
+func (t *TokenTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	if t.token != "" {
+		r.Header.Set("Authorization", "Bearer "+t.token)
+	}
+	return t.rt.RoundTrip(r)
+}
+
+func New(opts Opts) (*Client, error) {
+	// copied from net/http until it has a clone function: https://github.com/golang/go/issues/26013
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: opts.SkipVerify},
+	}
+	httpClient := &http.Client{Transport: &TokenTransport{token: opts.Token, rt: transport}}
+
+	if opts.APIURL == GitHubAPIURL {
+		opts.WebURL = GitHubWebURL
+	} else {
+		if opts.WebURL == "" {
+			opts.WebURL = opts.APIURL
+		}
+	}
+
+	client := github.NewClient(httpClient)
+	client.BaseURL, _ = url.Parse(GitHubAPIURL + "/")
+
+	return &Client{
+		client:         client,
+		httpClient:     httpClient,
+		APIURL:         opts.APIURL,
+		WebURL:         opts.WebURL,
+		oauth2ClientID: opts.Oauth2ClientID,
+		oauth2Secret:   opts.Oauth2Secret,
+	}, nil
+}
+
+func (c *Client) oauth2Config(callbackURL string) *oauth2.Config {
+	return &oauth2.Config{
+		ClientID:     c.oauth2ClientID,
+		ClientSecret: c.oauth2Secret,
+		Scopes:       GitHubOauth2Scopes,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  fmt.Sprintf("%s/login/oauth/authorize", c.WebURL),
+			TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.WebURL),
+		},
+		RedirectURL: callbackURL,
+	}
+}
+
+func (c *Client) GetOauth2AuthorizationURL(callbackURL, state string) (string, error) {
+	var config = c.oauth2Config(callbackURL)
+	return config.AuthCodeURL(state), nil
+}
+
+func (c *Client) RequestOauth2Token(callbackURL, code string) (*oauth2.Token, error) {
+	var config = c.oauth2Config(callbackURL)
+	token, err := config.Exchange(context.TODO(), code)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot get oauth2 token")
+	}
+	return token, nil
+}
+
+func (c *Client) RefreshOauth2Token(refreshToken string) (*oauth2.Token, error) {
+	var config = c.oauth2Config("")
+	token := &oauth2.Token{RefreshToken: refreshToken}
+	ts := config.TokenSource(context.TODO(), token)
+	return ts.Token()
+}
+
+func (c *Client) GetUserInfo() (*gitsource.UserInfo, error) {
+	user, _, err := c.client.Users.Get(context.TODO(), "")
+	if err != nil {
+		return nil, err
+	}
+	return &gitsource.UserInfo{
+		ID:        strconv.FormatInt(*user.ID, 10),
+		LoginName: *user.Login,
+		Email:     *user.Email,
+	}, nil
+}
+
+func (c *Client) GetRepoInfo(repopath string) (*gitsource.RepoInfo, error) {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return nil, err
+	}
+	rr, _, err := c.client.Repositories.Get(context.TODO(), owner, reponame)
+	if err != nil {
+		return nil, err
+	}
+	return fromGithubRepo(rr), nil
+}
+
+func (c *Client) GetFile(repopath, commit, file string) ([]byte, error) {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return nil, err
+	}
+	r, err := c.client.Repositories.DownloadContents(context.TODO(), owner, reponame, file, &github.RepositoryContentGetOptions{Ref: commit})
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	return ioutil.ReadAll(r)
+}
+
+func (c *Client) CreateDeployKey(repopath, title, pubKey string, readonly bool) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+	if _, _, err = c.client.Repositories.CreateKey(context.TODO(), owner, reponame, &github.Key{
+		Title:    github.String(title),
+		Key:      github.String(pubKey),
+		ReadOnly: github.Bool(readonly),
+	}); err != nil {
+		return errors.Wrapf(err, "error creating deploy key")
+	}
+	return nil
+}
+
+func (c *Client) UpdateDeployKey(repopath, title, pubKey string, readonly bool) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+	// NOTE(sgotti) gitea has a bug where if we delete and remove the same key with
+	// the same value it is correctly readded and the admin must force a
+	// authorized_keys regeneration on the server. To avoid this we update it only
+	// when the public key value has changed
+	keys, _, err := c.client.Repositories.ListKeys(context.TODO(), owner, reponame, nil)
+	if err != nil {
+		return errors.Wrapf(err, "error retrieving existing deploy keys")
+	}
+
+	for _, key := range keys {
+		if *key.Title == title {
+			if *key.Key == pubKey {
+				return nil
+			}
+			if _, err := c.client.Repositories.DeleteKey(context.TODO(), owner, reponame, *key.ID); err != nil {
+				return errors.Wrapf(err, "error removing existing deploy key")
+			}
+		}
+	}
+
+	if _, _, err = c.client.Repositories.CreateKey(context.TODO(), owner, reponame, &github.Key{
+		Title:    github.String(title),
+		Key:      github.String(pubKey),
+		ReadOnly: github.Bool(readonly),
+	}); err != nil {
+		return errors.Wrapf(err, "error creating deploy key")
+	}
+
+	return nil
+}
+
+func (c *Client) DeleteDeployKey(repopath, title string) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+	keys, _, err := c.client.Repositories.ListKeys(context.TODO(), owner, reponame, nil)
+	if err != nil {
+		return errors.Wrapf(err, "error retrieving existing deploy keys")
+	}
+
+	for _, key := range keys {
+		if *key.Title == title {
+			if _, err := c.client.Repositories.DeleteKey(context.TODO(), owner, reponame, *key.ID); err != nil {
+				return errors.Wrapf(err, "error removing existing deploy key")
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) CreateRepoWebhook(repopath, url, secret string) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+
+	hook := &github.Hook{
+		Config: map[string]interface{}{
+			"url":          url,
+			"content_type": "json",
+			"secret":       secret,
+		},
+		Events: []string{"push", "pull_request"},
+		Active: github.Bool(true),
+	}
+
+	if _, _, err = c.client.Repositories.CreateHook(context.TODO(), owner, reponame, hook); err != nil {
+		return errors.Wrapf(err, "error creating repository webhook")
+	}
+
+	return nil
+}
+
+func (c *Client) DeleteRepoWebhook(repopath, u string) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+
+	hooks := []*github.Hook{}
+
+	opt := &github.ListOptions{}
+	for {
+		pHooks, resp, err := c.client.Repositories.ListHooks(context.TODO(), owner, reponame, opt)
+		if err != nil {
+			return errors.Wrapf(err, "error retrieving repository webhooks")
+		}
+		hooks = append(hooks, pHooks...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	// match the full url so we can have multiple webhooks for different agola
+	// projects
+	for _, hook := range hooks {
+		if hook.Config["url"] == u {
+			if _, err := c.client.Repositories.DeleteHook(context.TODO(), owner, reponame, *hook.ID); err != nil {
+				return errors.Wrapf(err, "error deleting existing repository webhook")
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) CreateCommitStatus(repopath, commitSHA string, status gitsource.CommitStatus, targetURL, description, statusContext string) error {
+	owner, reponame, err := parseRepoPath(repopath)
+	if err != nil {
+		return err
+	}
+	_, _, err = c.client.Repositories.CreateStatus(context.TODO(), owner, reponame, commitSHA, &github.RepoStatus{
+		State:       github.String(fromCommitStatus(status)),
+		TargetURL:   github.String(targetURL),
+		Description: github.String(description),
+		Context:     github.String(statusContext),
+	})
+	return err
+}
+
+func (c *Client) ListUserRepos() ([]*gitsource.RepoInfo, error) {
+	remoteRepos := []*github.Repository{}
+
+	opt := &github.RepositoryListOptions{}
+	for {
+		pRemoteRepos, resp, err := c.client.Repositories.List(context.TODO(), "", opt)
+		if err != nil {
+			return nil, err
+		}
+		remoteRepos = append(remoteRepos, pRemoteRepos...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	repos := []*gitsource.RepoInfo{}
+
+	for _, rr := range remoteRepos {
+		repos = append(repos, fromGithubRepo(rr))
+	}
+
+	return repos, nil
+}
+
+func fromGithubRepo(rr *github.Repository) *gitsource.RepoInfo {
+	return &gitsource.RepoInfo{
+		ID:           strconv.FormatInt(*rr.ID, 10),
+		Path:         path.Join(*rr.Owner.Login, *rr.Name),
+		SSHCloneURL:  *rr.SSHURL,
+		HTTPCloneURL: *rr.CloneURL,
+	}
+}

internal/gitsources/github/parse.go

@@ -0,0 +1,135 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package github
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/google/go-github/v25/github"
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	hookPush        = "push"
+	hookPullRequest = "pull_request"
+
+	prStateOpen = "open"
+
+	prActionOpen = "opened"
+	prActionSync = "synchronize"
+)
+
+func (c *Client) ParseWebhook(r *http.Request, secret string) (*types.WebhookData, error) {
+	payload, err := github.ValidatePayload(r, []byte(secret))
+	if err != nil {
+		return nil, errors.Wrapf(err, "wrong webhook signature")
+	}
+	webHookType := github.WebHookType(r)
+	event, err := github.ParseWebHook(webHookType, payload)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse webhook")
+	}
+	switch event := event.(type) {
+	case *github.PushEvent:
+		return webhookDataFromPush(event)
+	case *github.PullRequestEvent:
+		return webhookDataFromPullRequest(event)
+	default:
+		return nil, errors.Errorf("unknown webhook event type: %q", webHookType)
+	}
+}
+
+func webhookDataFromPush(hook *github.PushEvent) (*types.WebhookData, error) {
+	sender := hook.Sender.Name
+	if sender == nil {
+		sender = hook.Sender.Login
+	}
+
+	// common data
+	whd := &types.WebhookData{
+		CommitSHA:   *hook.After,
+		SSHURL:      *hook.Repo.SSHURL,
+		Ref:         *hook.Ref,
+		CompareLink: *hook.Compare,
+		CommitLink:  fmt.Sprintf("%s/commit/%s", *hook.Repo.HTMLURL, *hook.After),
+		Sender:      *sender,
+
+		Repo: types.WebhookDataRepo{
+			Path:   path.Join(*hook.Repo.Owner.Name, *hook.Repo.Name),
+			WebURL: *hook.Repo.HTMLURL,
+		},
+	}
+
+	switch {
+	case strings.HasPrefix(*hook.Ref, "refs/heads/"):
+		whd.Event = types.WebhookEventPush
+		whd.Branch = strings.TrimPrefix(*hook.Ref, "refs/heads/")
+		whd.BranchLink = fmt.Sprintf("%s/tree/%s", *hook.Repo.HTMLURL, whd.Branch)
+		whd.Message = *hook.HeadCommit.Message
+
+	case strings.HasPrefix(*hook.Ref, "refs/tags/"):
+		whd.Event = types.WebhookEventTag
+		whd.Tag = strings.TrimPrefix(*hook.Ref, "refs/tags/")
+		whd.TagLink = fmt.Sprintf("%s/tree/%s", *hook.Repo.HTMLURL, whd.Tag)
+		whd.Message = fmt.Sprintf("Tag %s", whd.Tag)
+	default:
+		// ignore received webhook since it doesn't have a ref we're interested in
+		return nil, fmt.Errorf("unsupported webhook ref %q", *hook.Ref)
+	}
+
+	return whd, nil
+}
+
+func webhookDataFromPullRequest(hook *github.PullRequestEvent) (*types.WebhookData, error) {
+	// skip non open pull requests
+	if *hook.PullRequest.State != prStateOpen {
+		return nil, nil
+	}
+	// only accept actions that have new commits
+	if *hook.Action != prActionOpen && *hook.Action != prActionSync {
+		return nil, nil
+	}
+
+	sender := hook.Sender.Name
+	if sender == nil {
+		sender = hook.Sender.Login
+	}
+
+	whd := &types.WebhookData{
+		Event:           types.WebhookEventPullRequest,
+		CommitSHA:       *hook.PullRequest.Head.SHA,
+		SSHURL:          *hook.Repo.SSHURL,
+		Ref:             fmt.Sprintf("refs/pull/%d/head", *hook.Number),
+		CommitLink:      fmt.Sprintf("%s/commit/%s", *hook.Repo.HTMLURL, *hook.PullRequest.Head.SHA),
+		Branch:          *hook.PullRequest.Base.Ref,
+		Message:         *hook.PullRequest.Title,
+		Sender:          *sender,
+		PullRequestID:   strconv.Itoa(*hook.PullRequest.Number),
+		PullRequestLink: *hook.PullRequest.HTMLURL,
+
+		Repo: types.WebhookDataRepo{
+			Path:   path.Join(*hook.Repo.Owner.Login, *hook.Repo.Name),
+			WebURL: *hook.Repo.HTMLURL,
+		},
+	}
+
+	return whd, nil
+}

internal/gitsources/gitlab/gitlab.go

@@ -30,19 +30,12 @@ import (
 	"golang.org/x/oauth2"
 )
 
-const (
-	// TODO(sgotti) The gitea client doesn't provide an easy way to detect http response codes...
-	// we should probably use our own client implementation
-
-	ClientNotFound = "404 Not Found"
-)
-
 var (
 	GitlabOauth2Scopes = []string{"api"}
 )
 
 type Opts struct {
-	URL            string
+	APIURL         string
 	Token          string
 	SkipVerify     bool
 	Oauth2ClientID string
@@ -51,7 +44,7 @@ type Opts struct {
 
 type Client struct {
 	client         *gitlab.Client
-	URL            string
+	APIURL         string
 	oauth2ClientID string
 	oauth2Secret   string
 }
@@ -89,11 +82,11 @@ func New(opts Opts) (*Client, error) {
 	}
 	httpClient := &http.Client{Transport: transport}
 	client := gitlab.NewOAuthClient(httpClient, opts.Token)
-	client.SetBaseURL(opts.URL)
+	client.SetBaseURL(opts.APIURL)
 
 	return &Client{
 		client:         client,
-		URL:            opts.URL,
+		APIURL:         opts.APIURL,
 		oauth2ClientID: opts.Oauth2ClientID,
 		oauth2Secret:   opts.Oauth2Secret,
 	}, nil
@@ -105,8 +98,8 @@ func (c *Client) oauth2Config(callbackURL string) *oauth2.Config {
 		ClientSecret: c.oauth2Secret,
 		Scopes:       GitlabOauth2Scopes,
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  fmt.Sprintf("%s/oauth/authorize", c.URL),
+			AuthURL:  fmt.Sprintf("%s/oauth/authorize", c.APIURL),
-			TokenURL: fmt.Sprintf("%s/oauth/token", c.URL),
+			TokenURL: fmt.Sprintf("%s/oauth/token", c.APIURL),
 		},
 		RedirectURL: callbackURL,
 	}

internal/services/common/gitsource.go

@@ -17,6 +17,7 @@ package common
 import (
 	gitsource "github.com/sorintlab/agola/internal/gitsources"
 	"github.com/sorintlab/agola/internal/gitsources/gitea"
+	"github.com/sorintlab/agola/internal/gitsources/github"
 	"github.com/sorintlab/agola/internal/gitsources/gitlab"
 	"github.com/sorintlab/agola/internal/services/types"
 
@@ -25,7 +26,7 @@ import (
 
 func newGitea(rs *types.RemoteSource, accessToken string) (*gitea.Client, error) {
 	return gitea.New(gitea.Opts{
-		URL:            rs.APIURL,
+		APIURL:         rs.APIURL,
 		SkipVerify:     rs.SkipVerify,
 		Token:          accessToken,
 		Oauth2ClientID: rs.Oauth2ClientID,
@@ -35,7 +36,17 @@ func newGitea(rs *types.RemoteSource, accessToken string) (*gitea.Client, error)
 
 func newGitlab(rs *types.RemoteSource, accessToken string) (*gitlab.Client, error) {
 	return gitlab.New(gitlab.Opts{
-		URL:            rs.APIURL,
+		APIURL:         rs.APIURL,
+		SkipVerify:     rs.SkipVerify,
+		Token:          accessToken,
+		Oauth2ClientID: rs.Oauth2ClientID,
+		Oauth2Secret:   rs.Oauth2ClientSecret,
+	})
+}
+
+func newGithub(rs *types.RemoteSource, accessToken string) (*github.Client, error) {
+	return github.New(github.Opts{
+		APIURL:         rs.APIURL,
 		SkipVerify:     rs.SkipVerify,
 		Token:          accessToken,
 		Oauth2ClientID: rs.Oauth2ClientID,
@@ -71,6 +82,8 @@ func GetGitSource(rs *types.RemoteSource, la *types.LinkedAccount) (gitsource.Gi
 		gitSource, err = newGitea(rs, accessToken)
 	case types.RemoteSourceTypeGitlab:
 		gitSource, err = newGitlab(rs, accessToken)
+	case types.RemoteSourceTypeGithub:
+		gitSource, err = newGithub(rs, accessToken)
 	default:
 		return nil, errors.Errorf("remote source %s isn't a valid git source", rs.Name)
 	}
@@ -101,6 +114,8 @@ func GetOauth2Source(rs *types.RemoteSource, accessToken string) (gitsource.Oaut
 		oauth2Source, err = newGitea(rs, accessToken)
 	case types.RemoteSourceTypeGitlab:
 		oauth2Source, err = newGitlab(rs, accessToken)
+	case types.RemoteSourceTypeGithub:
+		oauth2Source, err = newGithub(rs, accessToken)
 	default:
 		return nil, errors.Errorf("remote source %s isn't a valid oauth2 source", rs.Name)
 	}