runservice: build and use multiple toolboxes per architecture

This commit is contained in:
Simone Gotti 2019-05-09 12:36:30 +02:00
parent 28d31c0802
commit 4e785e4851
9 changed files with 113 additions and 44 deletions

View File

@ -2,7 +2,7 @@
####### Build the backend ####### Build the backend
####### #######
# Base build image # base build image
FROM golang:1.11 AS build_base FROM golang:1.11 AS build_base
WORKDIR /agola WORKDIR /agola
@ -10,7 +10,7 @@ WORKDIR /agola
# use go modules # use go modules
ENV GO111MODULE=on ENV GO111MODULE=on
# Only copy go.mod and go.sum # only copy go.mod and go.sum
COPY go.mod . COPY go.mod .
COPY go.sum . COPY go.sum .
@ -19,10 +19,10 @@ RUN go mod download
# This image builds the weavaite server # This image builds the weavaite server
FROM build_base AS server_builder FROM build_base AS server_builder
# Copy all the source # copy all the source
COPY . . COPY . .
# Copy the agola-web dist # copy the agola-web dist
COPY --from=agola-web /agola-web/dist/ /agola-web/dist/ COPY --from=agola-web /agola-web/dist/ /agola-web/dist/
RUN make WEBBUNDLE=1 WEBDISTPATH=/agola-web/dist RUN make WEBBUNDLE=1 WEBDISTPATH=/agola-web/dist
@ -35,8 +35,8 @@ FROM debian:stable AS agola
WORKDIR / WORKDIR /
# Finally we copy the statically compiled Go binary. # copy to agola binaries
COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox /bin/ COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox-* /bin/
ENTRYPOINT ["/bin/agola"] ENTRYPOINT ["/bin/agola"]
@ -54,7 +54,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
git \ git \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Copy the example config # copy the example config
COPY examples/agolademo/config.yml . COPY examples/agolademo/config.yml .
ENTRYPOINT ["/bin/agola"] ENTRYPOINT ["/bin/agola"]

View File

@ -29,40 +29,43 @@ AGOLA_DEPS = $(AGOLA_WEBBUNDLE_DEPS)
AGOLA_TAGS += $(AGOLA_WEBBUNDLE_TAGS) AGOLA_TAGS += $(AGOLA_WEBBUNDLE_TAGS)
endif endif
TOOLBOX_OSES=linux
TOOLBOX_ARCHS=amd64 arm64
.PHONY: all .PHONY: all
all: build all: build
.PHONY: build .PHONY: build
build: bin/agola bin/agola-toolbox bin/agola-git-hook build: agola agola-toolbox agola-git-hook
.PHONY: test .PHONY: test
test: tools/bin/gocovmerge test: gocovmerge
@scripts/test.sh @scripts/test.sh
# don't use existing file names and track go sources, let's do this to the go tool # don't use existing file names and track go sources, let's do this to the go tool
.PHONY: bin/agola .PHONY: agola
bin/agola: $(AGOLA_DEPS) agola: $(AGOLA_DEPS)
GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola $(REPO_PATH)/cmd/agola GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola $(REPO_PATH)/cmd/agola
# toolbox MUST be statically compiled so it can be used in any image for that arch # toolbox MUST be statically compiled so it can be used in any image for that arch
# TODO(sgotti) cross compile to multiple archs .PHONY: agola-toolbox
.PHONY: bin/agola-toolbox agola-toolbox:
bin/agola-toolbox: $(foreach GOOS, $(TOOLBOX_OSES),\
CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox $(REPO_PATH)/cmd/toolbox $(foreach GOARCH, $(TOOLBOX_ARCHS), $(shell GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox-$(GOOS)-$(GOARCH) $(REPO_PATH)/cmd/toolbox)))
.PHONY: tools/bin/go-bindata .PHONY: go-bindata
tools/bin/go-bindata: go-bindata:
GOBIN=$(PROJDIR)/tools/bin go install github.com/go-bindata/go-bindata/go-bindata GOBIN=$(PROJDIR)/tools/bin go install github.com/go-bindata/go-bindata/go-bindata
.PHONY: bin/agola-git-hook .PHONY: agola-git-hook
bin/agola-git-hook: agola-git-hook:
CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-git-hook $(REPO_PATH)/cmd/agola-git-hook CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-git-hook $(REPO_PATH)/cmd/agola-git-hook
.PHONY: tools/bin/gocovmerge .PHONY: gocovmerge
tools/bin/gocovmerge: gocovmerge:
GOBIN=$(PROJDIR)/tools/bin go install github.com/wadey/gocovmerge GOBIN=$(PROJDIR)/tools/bin go install github.com/wadey/gocovmerge
webbundle/bindata.go: tools/bin/go-bindata $(WEBDISTPATH) webbundle/bindata.go: go-bindata $(WEBDISTPATH)
./tools/bin/go-bindata -o webbundle/bindata.go -tags webbundle -pkg webbundle -prefix "$(WEBDISTPATH)" -nocompress=true "$(WEBDISTPATH)/..." ./tools/bin/go-bindata -o webbundle/bindata.go -tags webbundle -pkg webbundle -prefix "$(WEBDISTPATH)" -nocompress=true "$(WEBDISTPATH)/..."
.PHONY: docker-agola .PHONY: docker-agola

View File

@ -43,7 +43,8 @@ runservice:
executor: executor:
dataDir: /tmp/agola/executor dataDir: /tmp/agola/executor
toolboxPath: ./bin/agola-toolbox # The directory containing the toolbox compiled for the various supported architectures
toolboxPath: ./bin
runserviceURL: "http://localhost:4000" runserviceURL: "http://localhost:4000"
web: web:
listenAddress: ":4001" listenAddress: ":4001"

View File

@ -121,7 +121,8 @@ data:
executor: executor:
dataDir: /mnt/agola/local/executor dataDir: /mnt/agola/local/executor
toolboxPath: ./bin/agola-toolbox # The directory containing the toolbox compiled for the various supported architectures
toolboxPath: ./bin
runserviceURL: "http://agola-runservice:4000" runserviceURL: "http://agola-runservice:4000"
web: web:
listenAddress: ":4001" listenAddress: ":4001"

View File

@ -110,7 +110,8 @@ data:
executor: executor:
dataDir: /mnt/agola/local/executor dataDir: /mnt/agola/local/executor
toolboxPath: ./bin/agola-toolbox # The directory containing the toolbox compiled for the various supported architectures
toolboxPath: ./bin
runserviceURL: "http://agola-internal:4000" runserviceURL: "http://agola-internal:4000"
web: web:
listenAddress: ":4001" listenAddress: ":4001"

View File

@ -46,6 +46,7 @@ type DockerDriver struct {
initVolumeHostDir string initVolumeHostDir string
toolboxPath string toolboxPath string
executorID string executorID string
arch common.Arch
} }
func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxPath string) (*DockerDriver, error) { func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxPath string) (*DockerDriver, error) {
@ -53,12 +54,14 @@ func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxP
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &DockerDriver{ return &DockerDriver{
logger: logger, logger: logger,
client: cli, client: cli,
initVolumeHostDir: initVolumeHostDir, initVolumeHostDir: initVolumeHostDir,
toolboxPath: toolboxPath, toolboxPath: toolboxPath,
executorID: executorID, executorID: executorID,
arch: common.ArchFromString(runtime.GOARCH),
}, nil }, nil
} }
@ -95,10 +98,15 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error {
return err return err
} }
srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false) toolboxExecPath, err := toolboxExecPath(d.toolboxPath, d.arch)
if err != nil {
return errors.Wrapf(err, "failed to get toolbox path for arch %q", d.arch)
}
srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false)
if err != nil { if err != nil {
return err return err
} }
srcInfo.RebaseName = "agola-toolbox"
srcArchive, err := archive.TarResource(srcInfo) srcArchive, err := archive.TarResource(srcInfo)
if err != nil { if err != nil {
@ -123,7 +131,7 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error {
func (d *DockerDriver) Archs(ctx context.Context) ([]common.Arch, error) { func (d *DockerDriver) Archs(ctx context.Context) ([]common.Arch, error) {
// since we are using the local docker driver we can return our go arch information // since we are using the local docker driver we can return our go arch information
return []common.Arch{common.ArchFromString(runtime.GOARCH)}, nil return []common.Arch{d.arch}, nil
} }
func (d *DockerDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Writer) (Pod, error) { func (d *DockerDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Writer) (Pod, error) {

View File

@ -16,13 +16,18 @@ package driver
import ( import (
"context" "context"
"fmt"
"io" "io"
"os"
"path/filepath"
"github.com/sorintlab/agola/internal/common" "github.com/sorintlab/agola/internal/common"
"github.com/sorintlab/agola/internal/services/executor/registry" "github.com/sorintlab/agola/internal/services/executor/registry"
) )
const ( const (
toolboxPrefix = "agola-toolbox"
labelPrefix = "agola.io/" labelPrefix = "agola.io/"
agolaLabelKey = labelPrefix + "agola" agolaLabelKey = labelPrefix + "agola"
@ -99,3 +104,12 @@ type ExecConfig struct {
Stderr io.Writer Stderr io.Writer
Tty bool Tty bool
} }
func toolboxExecPath(toolboxDir string, arch common.Arch) (string, error) {
toolboxPath := filepath.Join(toolboxDir, fmt.Sprintf("%s-linux-%s", toolboxPrefix, arch))
_, err := os.Stat(toolboxPath)
if err != nil {
return "", err
}
return toolboxPath, nil
}

View File

@ -15,6 +15,7 @@
package driver package driver
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
@ -424,23 +425,69 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
fmt.Fprintf(out, "init container ready\n") fmt.Fprintf(out, "init container ready\n")
srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false) coreclient, err := corev1client.NewForConfig(d.restconfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// get the pod arch
req := coreclient.RESTClient().
Post().
Namespace(pod.Namespace).
Resource("pods").
Name(pod.Name).
SubResource("exec").
VersionedParams(&corev1.PodExecOptions{
Container: "initcontainer",
Command: []string{"uname", "-m"},
Stdout: true,
Stderr: true,
TTY: false,
}, scheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL())
if err != nil {
return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL())
}
stdout := bytes.Buffer{}
err = exec.Stream(remotecommand.StreamOptions{
Stdout: &stdout,
Stderr: out,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to execute command on initcontainer")
}
osArch := strings.TrimSpace(stdout.String())
var arch common.Arch
switch osArch {
case "x86_64":
arch = common.ArchAMD64
case "aarch64":
arch = common.ArchARM64
default:
return nil, errors.Errorf("unsupported pod arch %q", osArch)
}
// copy the toolbox for the pod arch
toolboxExecPath, err := toolboxExecPath(d.toolboxPath, arch)
if err != nil {
return nil, errors.Wrapf(err, "failed to get toolbox path for arch %q", arch)
}
srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false)
if err != nil {
return nil, err
}
srcInfo.RebaseName = "agola-toolbox"
srcArchive, err := archive.TarResource(srcInfo) srcArchive, err := archive.TarResource(srcInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer srcArchive.Close() defer srcArchive.Close()
coreclient, err := corev1client.NewForConfig(d.restconfig) req = coreclient.RESTClient().
if err != nil {
return nil, err
}
req := coreclient.RESTClient().
Post(). Post().
Namespace(pod.Namespace). Namespace(pod.Namespace).
Resource("pods"). Resource("pods").
@ -455,11 +502,12 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
TTY: false, TTY: false,
}, scheme.ParameterCodec) }, scheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL()) exec, err = remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL())
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL()) return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL())
} }
fmt.Fprintf(out, "extracting toolbox\n")
err = exec.Stream(remotecommand.StreamOptions{ err = exec.Stream(remotecommand.StreamOptions{
Stdin: srcArchive, Stdin: srcArchive,
Stdout: out, Stdout: out,
@ -468,6 +516,7 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to execute command on initcontainer") return nil, errors.Wrapf(err, "failed to execute command on initcontainer")
} }
fmt.Fprintf(out, "extracting toolbox done\n")
req = coreclient.RESTClient(). req = coreclient.RESTClient().
Post(). Post().

View File

@ -25,7 +25,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"sync" "sync"
@ -1244,14 +1243,7 @@ func NewExecutor(c *config.Executor) (*Executor, error) {
var err error var err error
c.ToolboxPath, err = filepath.Abs(c.ToolboxPath) c.ToolboxPath, err = filepath.Abs(c.ToolboxPath)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "cannot find \"agola-toolbox\" absolute path") return nil, errors.Wrapf(err, "cannot determine \"agola-toolbox\" absolute path")
}
if c.ToolboxPath == "" {
path, err := exec.LookPath("agola-toolbox")
if err != nil {
return nil, errors.Errorf("cannot find \"agola-toolbox\" binaries in PATH, agola-toolbox path must be explicitly provided")
}
c.ToolboxPath = path
} }
e := &Executor{ e := &Executor{