Add custom git server

cmd/agola/cmd/serve.go

@@ -22,6 +22,7 @@ import (
 	"github.com/sorintlab/agola/internal/services/config"
 	"github.com/sorintlab/agola/internal/services/configstore"
 	"github.com/sorintlab/agola/internal/services/gateway"
+	"github.com/sorintlab/agola/internal/services/gitserver"
 	"github.com/sorintlab/agola/internal/services/runservice/executor"
 	rsscheduler "github.com/sorintlab/agola/internal/services/runservice/scheduler"
 	"github.com/sorintlab/agola/internal/services/scheduler"
@@ -133,12 +134,18 @@ func serve(cmd *cobra.Command, args []string) error {
 		return errors.Wrapf(err, "failed to start gateway")
 	}
 
+	gitserver, err := gitserver.NewGitServer(&c.GitServer)
+	if err != nil {
+		return errors.Wrapf(err, "failed to start git server")
+	}
+
 	errCh := make(chan error)
 
 	go func() { errCh <- rsex1.Run(ctx) }()
 	go func() { errCh <- rssched1.Run(ctx) }()
 	go func() { errCh <- cs.Run(ctx) }()
 	go func() { errCh <- gateway.Run(ctx) }()
+	go func() { errCh <- gitserver.Run(ctx) }()
 	go func() { errCh <- sched1.Run(ctx) }()
 
 	return <-errCh

internal/git-handler/handler.go

@@ -0,0 +1,265 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package githandler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/sorintlab/agola/internal/util"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+)
+
+var (
+	InfoRefsRegExp    = regexp.MustCompile(`/(.+\.git)/info/refs$`)
+	UploadPackRegExp  = regexp.MustCompile(`/(.+\.git)/git-upload-pack$`)
+	ReceivePackRegExp = regexp.MustCompile(`/(.+\.git)/git-receive-pack$`)
+
+	FetchFileRegExp = regexp.MustCompile(`/(.+\.git)/raw/(.+?)/(.+)`)
+)
+
+type RequestType int
+
+const (
+	RequestTypeInfoRefs RequestType = iota
+	RequestTypeUploadPack
+	RequestTypeReceivePack
+)
+
+type FetchFileData struct {
+	RepoPath string
+	Ref      string
+	Path     string
+}
+
+func ParseFetchFilePath(path string) (*FetchFileData, error) {
+	matches := FetchFileRegExp.FindStringSubmatch(path)
+	if len(matches) != 4 {
+		return nil, errors.New("cannot get fetch file data from url")
+	}
+	return &FetchFileData{
+		RepoPath: matches[1],
+		Ref:      matches[2],
+		Path:     matches[3],
+	}, nil
+}
+
+func MatchPath(path string) (string, RequestType, error) {
+	var matchedRegExp *regexp.Regexp
+	var reqType RequestType
+	for i, regExp := range []*regexp.Regexp{InfoRefsRegExp, UploadPackRegExp, ReceivePackRegExp} {
+		if regExp.MatchString(path) {
+			matchedRegExp = regExp
+			reqType = RequestType(i)
+		}
+	}
+	if matchedRegExp == nil {
+		return "", 0, errors.New("wrong request path")
+	}
+
+	matches := matchedRegExp.FindStringSubmatch(path)
+	if len(matches) != 2 {
+		return "", 0, errors.New("cannot get repository path from url")
+	}
+	return matches[1], reqType, nil
+}
+
+func gitServiceName(r *http.Request) (string, error) {
+	service := r.URL.Query().Get("service")
+	if !strings.HasPrefix(service, "git-") {
+		return "", errors.Errorf("wrong git service %q", service)
+	}
+	return strings.TrimPrefix(service, "git-"), nil
+}
+
+func writePacketLine(w io.Writer, line string) {
+	fmt.Fprintf(w, "%.4x%s\n", len(line)+5, line)
+}
+
+func writeFlushPacket(w io.Writer) {
+	fmt.Fprintf(w, "0000")
+}
+
+func InfoRefsResponse(ctx context.Context, repoPath, serviceName string) ([]byte, error) {
+	buf := &bytes.Buffer{}
+
+	writePacketLine(buf, "# service=git-"+serviceName)
+	writeFlushPacket(buf)
+
+	git := &util.Git{}
+	out, err := git.Output(ctx, nil, serviceName, "--stateless-rpc", "--advertise-refs", repoPath)
+	if err != nil {
+		return nil, err
+	}
+	buf.Write(out)
+
+	return buf.Bytes(), err
+}
+
+func gitService(ctx context.Context, w io.Writer, r io.Reader, repoPath, serviceName string) error {
+	git := &util.Git{GitDir: repoPath}
+	return git.Pipe(ctx, w, r, serviceName, "--stateless-rpc", repoPath)
+}
+
+func gitFetchFile(ctx context.Context, w io.Writer, r io.Reader, repoPath, ref, path string) error {
+	git := &util.Git{GitDir: repoPath}
+	return git.Pipe(ctx, w, r, "show", fmt.Sprintf("%s:%s", ref, path))
+}
+
+var ErrWrongRepoPath = errors.New("wrong repository path")
+
+// RepoAbsPathFunc is a user defined functions that, given the repo path
+// provided in the url request, will return the file system absolute repo path
+// and if it exists.
+// This function should also do path validation and return ErrWrongRepoPath if
+// path validation failed.
+type RepoAbsPathFunc func(reposDir, path string) (absPath string, exists bool, err error)
+
+type RepoPostCreateFunc func(repoPath, repoAbsPath string) error
+
+type GitSmartHandler struct {
+	log                *zap.SugaredLogger
+	reposDir           string
+	createRepo         bool
+	repoAbsPathFunc    RepoAbsPathFunc
+	repoPostCreateFunc RepoPostCreateFunc
+}
+
+func NewGitSmartHandler(logger *zap.Logger, reposDir string, createRepo bool, repoAbsPathFunc RepoAbsPathFunc, repoPostCreateFunc RepoPostCreateFunc) *GitSmartHandler {
+	return &GitSmartHandler{
+		log:                logger.Sugar(),
+		reposDir:           reposDir,
+		createRepo:         createRepo,
+		repoAbsPathFunc:    repoAbsPathFunc,
+		repoPostCreateFunc: repoPostCreateFunc,
+	}
+}
+
+func (h *GitSmartHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	repoPath, reqType, err := MatchPath(r.URL.Path)
+	h.log.Infof("repoPath: %s", repoPath)
+	repoAbsPath, exists, err := h.repoAbsPathFunc(h.reposDir, repoPath)
+	if err != nil {
+		if err == ErrWrongRepoPath {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	h.log.Infof("repoAbsPath: %s", repoAbsPath)
+	h.log.Infof("repo exists: %t", exists)
+
+	git := &util.Git{GitDir: repoAbsPath}
+
+	switch reqType {
+	case RequestTypeInfoRefs:
+		if h.createRepo && !exists {
+			if output, err := git.Output(ctx, nil, "init", "--bare", repoAbsPath); err != nil {
+				h.log.Infof("git error %v, output: %s", err, output)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			if h.repoPostCreateFunc != nil {
+				if err := h.repoPostCreateFunc(repoPath, repoAbsPath); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+			}
+		}
+
+		serviceName, err := gitServiceName(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		res, err := InfoRefsResponse(ctx, repoAbsPath, serviceName)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/x-git-"+serviceName+"-advertisement")
+		w.Write(res)
+
+	case RequestTypeUploadPack:
+		w.Header().Set("Content-Type", "application/x-git-upload-pack-result")
+
+		if err := gitService(ctx, w, r.Body, repoAbsPath, "upload-pack"); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			// we cannot return any http error since the http header has already been written
+			h.log.Infof("git command error: %v", err)
+		}
+	case RequestTypeReceivePack:
+		w.Header().Set("Content-Type", "application/x-git-receive-pack-result")
+
+		if err := gitService(ctx, w, r.Body, repoAbsPath, "receive-pack"); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			// we cannot return any http error since the http header has already been written
+			h.log.Infof("git command error: %v", err)
+		}
+	}
+}
+
+type FetchFileHandler struct {
+	log             *zap.SugaredLogger
+	reposDir        string
+	repoAbsPathFunc RepoAbsPathFunc
+}
+
+func NewFetchFileHandler(logger *zap.Logger, reposDir string, repoAbsPathFunc RepoAbsPathFunc) *FetchFileHandler {
+	return &FetchFileHandler{
+		log:             logger.Sugar(),
+		reposDir:        reposDir,
+		repoAbsPathFunc: repoAbsPathFunc,
+	}
+}
+
+func (h *FetchFileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	fetchData, err := ParseFetchFilePath(r.URL.Path)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	h.log.Infof("fetchData: %v", fetchData)
+
+	repoAbsPath, _, err := h.repoAbsPathFunc(h.reposDir, fetchData.RepoPath)
+	if err != nil {
+		if err == ErrWrongRepoPath {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := gitFetchFile(ctx, w, r.Body, repoAbsPath, fetchData.Ref, fetchData.Path); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		// we cannot return any http error since the http header has already been written
+		h.log.Infof("git command error: %v", err)
+	}
+}

internal/services/gitserver/main.go

@@ -0,0 +1,218 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gitserver
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	handlers "github.com/sorintlab/agola/internal/git-handler"
+	slog "github.com/sorintlab/agola/internal/log"
+	"github.com/sorintlab/agola/internal/services/config"
+	"github.com/sorintlab/agola/internal/util"
+
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
+var logger = slog.New(level)
+var log = logger.Sugar()
+
+const (
+	gitSuffix = ".git"
+)
+
+func repoPathIsValid(reposDir, repoPath string) (bool, error) {
+	// a parent cannot end with .git
+	parts := strings.Split(repoPath, "/")
+	for _, part := range parts[:len(parts)-1] {
+		if strings.HasSuffix(part, gitSuffix) {
+			return false, errors.Errorf("path %q contains a parent directory with .git suffix", repoPath)
+		}
+	}
+
+	// check that a subdirectory doesn't exists
+	reposDir, err := filepath.Abs(reposDir)
+	if err != nil {
+		return false, err
+	}
+
+	path := repoPath
+	_, err = os.Stat(path)
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	if !os.IsNotExist(err) {
+		// if it exists assume it's valid
+		return true, nil
+	}
+
+	for {
+		path = filepath.Dir(path)
+		if len(path) <= len(reposDir) {
+			break
+		}
+
+		_, err := os.Stat(path)
+		if err != nil && !os.IsNotExist(err) {
+			return false, err
+		}
+		// a parent path cannot end with .git
+		if strings.HasSuffix(path, gitSuffix) {
+			return false, nil
+		}
+		if !os.IsNotExist(err) {
+			// if a parent exists return not valid
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func repoExists(repoAbsPath string) (bool, error) {
+	_, err := os.Stat(repoAbsPath)
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	return !os.IsNotExist(err), nil
+}
+
+func repoAbsPath(reposDir, repoPath string) (string, bool, error) {
+	valid, err := repoPathIsValid(reposDir, repoPath)
+	if err != nil {
+		return "", false, err
+	}
+	if !valid {
+		return "", false, handlers.ErrWrongRepoPath
+	}
+
+	repoFSPath, err := filepath.Abs(filepath.Join(reposDir, repoPath))
+	if err != nil {
+		return "", false, err
+	}
+
+	exists, err := repoExists(repoFSPath)
+	if err != nil {
+		return "", false, err
+	}
+
+	return repoFSPath, exists, nil
+}
+
+func Matcher(matchRegexp *regexp.Regexp) mux.MatcherFunc {
+	return func(r *http.Request, rm *mux.RouteMatch) bool {
+		return matchRegexp.MatchString(r.URL.Path)
+	}
+}
+
+func (s *GitServer) repoPostCreateFunc(githookPath, gatewayURL string) handlers.RepoPostCreateFunc {
+	return func(repoPath, repoAbsPath string) error {
+		f, err := os.OpenFile(filepath.Join(repoAbsPath, "hooks/post-receive"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0760)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		log.Infof("creating post-receive hook: %s", repoAbsPath)
+		f.WriteString("#/bin/bash\n")
+		f.WriteString("while read oval nval ref; do\n")
+		f.WriteString(githookPath + " $oval $nval $ref\n")
+		f.WriteString("done\n")
+
+		git := &util.Git{GitDir: repoAbsPath}
+		git.ConfigSet(context.Background(), "agola.repo", repoPath)
+		git.ConfigSet(context.Background(), "agola.webhookURL", gatewayURL+"/webhooks")
+
+		return nil
+	}
+}
+
+type GitServer struct {
+	c *config.GitServer
+}
+
+func NewGitServer(c *config.GitServer) (*GitServer, error) {
+	if c.Debug {
+		level.SetLevel(zapcore.DebugLevel)
+	}
+
+	var err error
+	c.GithookPath, err = filepath.Abs(c.GithookPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot find agola-git-hook absolute path")
+	}
+	if c.GithookPath == "" {
+		path, err := exec.LookPath("agola-git-hook")
+		if err != nil {
+			return nil, errors.Errorf("cannot find \"agola-git-hook\" binaries in PATH, agola-githook path must be explicitly provided")
+		}
+		c.GithookPath = path
+	}
+
+	return &GitServer{
+		c: c,
+	}, nil
+}
+
+func (s *GitServer) Run(ctx context.Context) error {
+	gitSmartHandler := handlers.NewGitSmartHandler(logger, s.c.DataDir, true, repoAbsPath, s.repoPostCreateFunc(s.c.GithookPath, s.c.GatewayURL))
+	fetchFileHandler := handlers.NewFetchFileHandler(logger, s.c.DataDir, repoAbsPath)
+
+	router := mux.NewRouter()
+	router.MatcherFunc(Matcher(handlers.InfoRefsRegExp)).Handler(gitSmartHandler)
+	router.MatcherFunc(Matcher(handlers.UploadPackRegExp)).Handler(gitSmartHandler)
+	router.MatcherFunc(Matcher(handlers.ReceivePackRegExp)).Handler(gitSmartHandler)
+	router.MatcherFunc(Matcher(handlers.FetchFileRegExp)).Handler(fetchFileHandler)
+
+	var tlsConfig *tls.Config
+	if s.c.Web.TLS {
+		var err error
+		tlsConfig, err = util.NewTLSConfig(s.c.Web.TLSCertFile, s.c.Web.TLSKeyFile, "", false)
+		if err != nil {
+			log.Errorf("err: %+v")
+			return err
+		}
+	}
+
+	httpServer := http.Server{
+		Addr:      s.c.Web.ListenAddress,
+		Handler:   router,
+		TLSConfig: tlsConfig,
+	}
+
+	lerrCh := make(chan error)
+	go func() {
+		lerrCh <- httpServer.ListenAndServe()
+	}()
+
+	select {
+	case <-ctx.Done():
+		log.Infof("gitserver exiting")
+		httpServer.Close()
+		return nil
+	case err := <-lerrCh:
+		log.Errorf("http server listen error: %v", err)
+		return err
+	}
+}