From 203a41eaf7ade0480c120df281427addc0a9e368 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Thu, 21 Feb 2019 16:03:17 +0100 Subject: [PATCH] Add custom git server --- cmd/agola/cmd/serve.go | 7 + internal/git-handler/handler.go | 265 ++++++++++++++++++++++++++++ internal/services/gitserver/main.go | 218 +++++++++++++++++++++++ 3 files changed, 490 insertions(+) create mode 100644 internal/git-handler/handler.go create mode 100644 internal/services/gitserver/main.go diff --git a/cmd/agola/cmd/serve.go b/cmd/agola/cmd/serve.go index c3ed7eb..be0383a 100644 --- a/cmd/agola/cmd/serve.go +++ b/cmd/agola/cmd/serve.go @@ -22,6 +22,7 @@ import ( "github.com/sorintlab/agola/internal/services/config" "github.com/sorintlab/agola/internal/services/configstore" "github.com/sorintlab/agola/internal/services/gateway" + "github.com/sorintlab/agola/internal/services/gitserver" "github.com/sorintlab/agola/internal/services/runservice/executor" rsscheduler "github.com/sorintlab/agola/internal/services/runservice/scheduler" "github.com/sorintlab/agola/internal/services/scheduler" @@ -133,12 +134,18 @@ func serve(cmd *cobra.Command, args []string) error { return errors.Wrapf(err, "failed to start gateway") } + gitserver, err := gitserver.NewGitServer(&c.GitServer) + if err != nil { + return errors.Wrapf(err, "failed to start git server") + } + errCh := make(chan error) go func() { errCh <- rsex1.Run(ctx) }() go func() { errCh <- rssched1.Run(ctx) }() go func() { errCh <- cs.Run(ctx) }() go func() { errCh <- gateway.Run(ctx) }() + go func() { errCh <- gitserver.Run(ctx) }() go func() { errCh <- sched1.Run(ctx) }() return <-errCh diff --git a/internal/git-handler/handler.go b/internal/git-handler/handler.go new file mode 100644 index 0000000..8255ead --- /dev/null +++ b/internal/git-handler/handler.go @@ -0,0 +1,265 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package githandler + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "github.com/sorintlab/agola/internal/util" + + "github.com/pkg/errors" + "go.uber.org/zap" +) + +var ( + InfoRefsRegExp = regexp.MustCompile(`/(.+\.git)/info/refs$`) + UploadPackRegExp = regexp.MustCompile(`/(.+\.git)/git-upload-pack$`) + ReceivePackRegExp = regexp.MustCompile(`/(.+\.git)/git-receive-pack$`) + + FetchFileRegExp = regexp.MustCompile(`/(.+\.git)/raw/(.+?)/(.+)`) +) + +type RequestType int + +const ( + RequestTypeInfoRefs RequestType = iota + RequestTypeUploadPack + RequestTypeReceivePack +) + +type FetchFileData struct { + RepoPath string + Ref string + Path string +} + +func ParseFetchFilePath(path string) (*FetchFileData, error) { + matches := FetchFileRegExp.FindStringSubmatch(path) + if len(matches) != 4 { + return nil, errors.New("cannot get fetch file data from url") + } + return &FetchFileData{ + RepoPath: matches[1], + Ref: matches[2], + Path: matches[3], + }, nil +} + +func MatchPath(path string) (string, RequestType, error) { + var matchedRegExp *regexp.Regexp + var reqType RequestType + for i, regExp := range []*regexp.Regexp{InfoRefsRegExp, UploadPackRegExp, ReceivePackRegExp} { + if regExp.MatchString(path) { + matchedRegExp = regExp + reqType = RequestType(i) + } + } + if matchedRegExp == nil { + return "", 0, errors.New("wrong request path") + } + + matches := matchedRegExp.FindStringSubmatch(path) + if len(matches) != 2 { + return "", 0, errors.New("cannot get repository path from url") + } + return matches[1], reqType, nil +} + +func gitServiceName(r *http.Request) (string, error) { + service := r.URL.Query().Get("service") + if !strings.HasPrefix(service, "git-") { + return "", errors.Errorf("wrong git service %q", service) + } + return strings.TrimPrefix(service, "git-"), nil +} + +func writePacketLine(w io.Writer, line string) { + fmt.Fprintf(w, "%.4x%s\n", len(line)+5, line) +} + +func writeFlushPacket(w io.Writer) { + fmt.Fprintf(w, "0000") +} + +func InfoRefsResponse(ctx context.Context, repoPath, serviceName string) ([]byte, error) { + buf := &bytes.Buffer{} + + writePacketLine(buf, "# service=git-"+serviceName) + writeFlushPacket(buf) + + git := &util.Git{} + out, err := git.Output(ctx, nil, serviceName, "--stateless-rpc", "--advertise-refs", repoPath) + if err != nil { + return nil, err + } + buf.Write(out) + + return buf.Bytes(), err +} + +func gitService(ctx context.Context, w io.Writer, r io.Reader, repoPath, serviceName string) error { + git := &util.Git{GitDir: repoPath} + return git.Pipe(ctx, w, r, serviceName, "--stateless-rpc", repoPath) +} + +func gitFetchFile(ctx context.Context, w io.Writer, r io.Reader, repoPath, ref, path string) error { + git := &util.Git{GitDir: repoPath} + return git.Pipe(ctx, w, r, "show", fmt.Sprintf("%s:%s", ref, path)) +} + +var ErrWrongRepoPath = errors.New("wrong repository path") + +// RepoAbsPathFunc is a user defined functions that, given the repo path +// provided in the url request, will return the file system absolute repo path +// and if it exists. +// This function should also do path validation and return ErrWrongRepoPath if +// path validation failed. +type RepoAbsPathFunc func(reposDir, path string) (absPath string, exists bool, err error) + +type RepoPostCreateFunc func(repoPath, repoAbsPath string) error + +type GitSmartHandler struct { + log *zap.SugaredLogger + reposDir string + createRepo bool + repoAbsPathFunc RepoAbsPathFunc + repoPostCreateFunc RepoPostCreateFunc +} + +func NewGitSmartHandler(logger *zap.Logger, reposDir string, createRepo bool, repoAbsPathFunc RepoAbsPathFunc, repoPostCreateFunc RepoPostCreateFunc) *GitSmartHandler { + return &GitSmartHandler{ + log: logger.Sugar(), + reposDir: reposDir, + createRepo: createRepo, + repoAbsPathFunc: repoAbsPathFunc, + repoPostCreateFunc: repoPostCreateFunc, + } +} + +func (h *GitSmartHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + repoPath, reqType, err := MatchPath(r.URL.Path) + h.log.Infof("repoPath: %s", repoPath) + repoAbsPath, exists, err := h.repoAbsPathFunc(h.reposDir, repoPath) + if err != nil { + if err == ErrWrongRepoPath { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + h.log.Infof("repoAbsPath: %s", repoAbsPath) + h.log.Infof("repo exists: %t", exists) + + git := &util.Git{GitDir: repoAbsPath} + + switch reqType { + case RequestTypeInfoRefs: + if h.createRepo && !exists { + if output, err := git.Output(ctx, nil, "init", "--bare", repoAbsPath); err != nil { + h.log.Infof("git error %v, output: %s", err, output) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if h.repoPostCreateFunc != nil { + if err := h.repoPostCreateFunc(repoPath, repoAbsPath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + + serviceName, err := gitServiceName(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + res, err := InfoRefsResponse(ctx, repoAbsPath, serviceName) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/x-git-"+serviceName+"-advertisement") + w.Write(res) + + case RequestTypeUploadPack: + w.Header().Set("Content-Type", "application/x-git-upload-pack-result") + + if err := gitService(ctx, w, r.Body, repoAbsPath, "upload-pack"); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + // we cannot return any http error since the http header has already been written + h.log.Infof("git command error: %v", err) + } + case RequestTypeReceivePack: + w.Header().Set("Content-Type", "application/x-git-receive-pack-result") + + if err := gitService(ctx, w, r.Body, repoAbsPath, "receive-pack"); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + // we cannot return any http error since the http header has already been written + h.log.Infof("git command error: %v", err) + } + } +} + +type FetchFileHandler struct { + log *zap.SugaredLogger + reposDir string + repoAbsPathFunc RepoAbsPathFunc +} + +func NewFetchFileHandler(logger *zap.Logger, reposDir string, repoAbsPathFunc RepoAbsPathFunc) *FetchFileHandler { + return &FetchFileHandler{ + log: logger.Sugar(), + reposDir: reposDir, + repoAbsPathFunc: repoAbsPathFunc, + } +} + +func (h *FetchFileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + fetchData, err := ParseFetchFilePath(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + h.log.Infof("fetchData: %v", fetchData) + + repoAbsPath, _, err := h.repoAbsPathFunc(h.reposDir, fetchData.RepoPath) + if err != nil { + if err == ErrWrongRepoPath { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := gitFetchFile(ctx, w, r.Body, repoAbsPath, fetchData.Ref, fetchData.Path); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + // we cannot return any http error since the http header has already been written + h.log.Infof("git command error: %v", err) + } +} diff --git a/internal/services/gitserver/main.go b/internal/services/gitserver/main.go new file mode 100644 index 0000000..fe36a05 --- /dev/null +++ b/internal/services/gitserver/main.go @@ -0,0 +1,218 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitserver + +import ( + "context" + "crypto/tls" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + handlers "github.com/sorintlab/agola/internal/git-handler" + slog "github.com/sorintlab/agola/internal/log" + "github.com/sorintlab/agola/internal/services/config" + "github.com/sorintlab/agola/internal/util" + + "github.com/gorilla/mux" + "github.com/pkg/errors" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) +var logger = slog.New(level) +var log = logger.Sugar() + +const ( + gitSuffix = ".git" +) + +func repoPathIsValid(reposDir, repoPath string) (bool, error) { + // a parent cannot end with .git + parts := strings.Split(repoPath, "/") + for _, part := range parts[:len(parts)-1] { + if strings.HasSuffix(part, gitSuffix) { + return false, errors.Errorf("path %q contains a parent directory with .git suffix", repoPath) + } + } + + // check that a subdirectory doesn't exists + reposDir, err := filepath.Abs(reposDir) + if err != nil { + return false, err + } + + path := repoPath + _, err = os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return false, err + } + if !os.IsNotExist(err) { + // if it exists assume it's valid + return true, nil + } + + for { + path = filepath.Dir(path) + if len(path) <= len(reposDir) { + break + } + + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return false, err + } + // a parent path cannot end with .git + if strings.HasSuffix(path, gitSuffix) { + return false, nil + } + if !os.IsNotExist(err) { + // if a parent exists return not valid + return false, nil + } + } + + return true, nil +} + +func repoExists(repoAbsPath string) (bool, error) { + _, err := os.Stat(repoAbsPath) + if err != nil && !os.IsNotExist(err) { + return false, err + } + return !os.IsNotExist(err), nil +} + +func repoAbsPath(reposDir, repoPath string) (string, bool, error) { + valid, err := repoPathIsValid(reposDir, repoPath) + if err != nil { + return "", false, err + } + if !valid { + return "", false, handlers.ErrWrongRepoPath + } + + repoFSPath, err := filepath.Abs(filepath.Join(reposDir, repoPath)) + if err != nil { + return "", false, err + } + + exists, err := repoExists(repoFSPath) + if err != nil { + return "", false, err + } + + return repoFSPath, exists, nil +} + +func Matcher(matchRegexp *regexp.Regexp) mux.MatcherFunc { + return func(r *http.Request, rm *mux.RouteMatch) bool { + return matchRegexp.MatchString(r.URL.Path) + } +} + +func (s *GitServer) repoPostCreateFunc(githookPath, gatewayURL string) handlers.RepoPostCreateFunc { + return func(repoPath, repoAbsPath string) error { + f, err := os.OpenFile(filepath.Join(repoAbsPath, "hooks/post-receive"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0760) + if err != nil { + return err + } + defer f.Close() + log.Infof("creating post-receive hook: %s", repoAbsPath) + f.WriteString("#/bin/bash\n") + f.WriteString("while read oval nval ref; do\n") + f.WriteString(githookPath + " $oval $nval $ref\n") + f.WriteString("done\n") + + git := &util.Git{GitDir: repoAbsPath} + git.ConfigSet(context.Background(), "agola.repo", repoPath) + git.ConfigSet(context.Background(), "agola.webhookURL", gatewayURL+"/webhooks") + + return nil + } +} + +type GitServer struct { + c *config.GitServer +} + +func NewGitServer(c *config.GitServer) (*GitServer, error) { + if c.Debug { + level.SetLevel(zapcore.DebugLevel) + } + + var err error + c.GithookPath, err = filepath.Abs(c.GithookPath) + if err != nil { + return nil, errors.Wrapf(err, "cannot find agola-git-hook absolute path") + } + if c.GithookPath == "" { + path, err := exec.LookPath("agola-git-hook") + if err != nil { + return nil, errors.Errorf("cannot find \"agola-git-hook\" binaries in PATH, agola-githook path must be explicitly provided") + } + c.GithookPath = path + } + + return &GitServer{ + c: c, + }, nil +} + +func (s *GitServer) Run(ctx context.Context) error { + gitSmartHandler := handlers.NewGitSmartHandler(logger, s.c.DataDir, true, repoAbsPath, s.repoPostCreateFunc(s.c.GithookPath, s.c.GatewayURL)) + fetchFileHandler := handlers.NewFetchFileHandler(logger, s.c.DataDir, repoAbsPath) + + router := mux.NewRouter() + router.MatcherFunc(Matcher(handlers.InfoRefsRegExp)).Handler(gitSmartHandler) + router.MatcherFunc(Matcher(handlers.UploadPackRegExp)).Handler(gitSmartHandler) + router.MatcherFunc(Matcher(handlers.ReceivePackRegExp)).Handler(gitSmartHandler) + router.MatcherFunc(Matcher(handlers.FetchFileRegExp)).Handler(fetchFileHandler) + + var tlsConfig *tls.Config + if s.c.Web.TLS { + var err error + tlsConfig, err = util.NewTLSConfig(s.c.Web.TLSCertFile, s.c.Web.TLSKeyFile, "", false) + if err != nil { + log.Errorf("err: %+v") + return err + } + } + + httpServer := http.Server{ + Addr: s.c.Web.ListenAddress, + Handler: router, + TLSConfig: tlsConfig, + } + + lerrCh := make(chan error) + go func() { + lerrCh <- httpServer.ListenAndServe() + }() + + select { + case <-ctx.Done(): + log.Infof("gitserver exiting") + httpServer.Close() + return nil + case err := <-lerrCh: + log.Errorf("http server listen error: %v", err) + return err + } +}