diff --git a/Dockerfile b/Dockerfile index 3b81d4b..d163b22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ####### Build the backend ####### -# Base build image +# base build image FROM golang:1.11 AS build_base WORKDIR /agola @@ -10,7 +10,7 @@ WORKDIR /agola # use go modules ENV GO111MODULE=on -# Only copy go.mod and go.sum +# only copy go.mod and go.sum COPY go.mod . COPY go.sum . @@ -19,10 +19,10 @@ RUN go mod download # This image builds the weavaite server FROM build_base AS server_builder -# Copy all the source +# copy all the source COPY . . -# Copy the agola-web dist +# copy the agola-web dist COPY --from=agola-web /agola-web/dist/ /agola-web/dist/ RUN make WEBBUNDLE=1 WEBDISTPATH=/agola-web/dist @@ -35,8 +35,8 @@ FROM debian:stable AS agola WORKDIR / -# Finally we copy the statically compiled Go binary. -COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox /bin/ +# copy to agola binaries +COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox-* /bin/ ENTRYPOINT ["/bin/agola"] @@ -54,7 +54,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -# Copy the example config +# copy the example config COPY examples/agolademo/config.yml . ENTRYPOINT ["/bin/agola"] diff --git a/Makefile b/Makefile index 423091f..5d215e8 100644 --- a/Makefile +++ b/Makefile @@ -29,40 +29,43 @@ AGOLA_DEPS = $(AGOLA_WEBBUNDLE_DEPS) AGOLA_TAGS += $(AGOLA_WEBBUNDLE_TAGS) endif +TOOLBOX_OSES=linux +TOOLBOX_ARCHS=amd64 arm64 + .PHONY: all all: build .PHONY: build -build: bin/agola bin/agola-toolbox bin/agola-git-hook +build: agola agola-toolbox agola-git-hook .PHONY: test -test: tools/bin/gocovmerge +test: gocovmerge @scripts/test.sh # don't use existing file names and track go sources, let's do this to the go tool -.PHONY: bin/agola -bin/agola: $(AGOLA_DEPS) +.PHONY: agola +agola: $(AGOLA_DEPS) GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola $(REPO_PATH)/cmd/agola # toolbox MUST be statically compiled so it can be used in any image for that arch -# TODO(sgotti) cross compile to multiple archs -.PHONY: bin/agola-toolbox -bin/agola-toolbox: - CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox $(REPO_PATH)/cmd/toolbox +.PHONY: agola-toolbox +agola-toolbox: + $(foreach GOOS, $(TOOLBOX_OSES),\ + $(foreach GOARCH, $(TOOLBOX_ARCHS), $(shell GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox-$(GOOS)-$(GOARCH) $(REPO_PATH)/cmd/toolbox))) -.PHONY: tools/bin/go-bindata -tools/bin/go-bindata: +.PHONY: go-bindata +go-bindata: GOBIN=$(PROJDIR)/tools/bin go install github.com/go-bindata/go-bindata/go-bindata -.PHONY: bin/agola-git-hook -bin/agola-git-hook: +.PHONY: agola-git-hook +agola-git-hook: CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-git-hook $(REPO_PATH)/cmd/agola-git-hook -.PHONY: tools/bin/gocovmerge -tools/bin/gocovmerge: +.PHONY: gocovmerge +gocovmerge: GOBIN=$(PROJDIR)/tools/bin go install github.com/wadey/gocovmerge -webbundle/bindata.go: tools/bin/go-bindata $(WEBDISTPATH) +webbundle/bindata.go: go-bindata $(WEBDISTPATH) ./tools/bin/go-bindata -o webbundle/bindata.go -tags webbundle -pkg webbundle -prefix "$(WEBDISTPATH)" -nocompress=true "$(WEBDISTPATH)/..." .PHONY: docker-agola diff --git a/examples/agolademo/config.yml b/examples/agolademo/config.yml index 01d6019..c0f21bc 100644 --- a/examples/agolademo/config.yml +++ b/examples/agolademo/config.yml @@ -43,7 +43,8 @@ runservice: executor: dataDir: /tmp/agola/executor - toolboxPath: ./bin/agola-toolbox + # The directory containing the toolbox compiled for the various supported architectures + toolboxPath: ./bin runserviceURL: "http://localhost:4000" web: listenAddress: ":4001" diff --git a/examples/kubernetes/distributed/agola.yml b/examples/kubernetes/distributed/agola.yml index 888af58..bff2a82 100644 --- a/examples/kubernetes/distributed/agola.yml +++ b/examples/kubernetes/distributed/agola.yml @@ -121,7 +121,8 @@ data: executor: dataDir: /mnt/agola/local/executor - toolboxPath: ./bin/agola-toolbox + # The directory containing the toolbox compiled for the various supported architectures + toolboxPath: ./bin runserviceURL: "http://agola-runservice:4000" web: listenAddress: ":4001" diff --git a/examples/kubernetes/simple/agola.yml b/examples/kubernetes/simple/agola.yml index eebf5e4..1019dcb 100644 --- a/examples/kubernetes/simple/agola.yml +++ b/examples/kubernetes/simple/agola.yml @@ -110,7 +110,8 @@ data: executor: dataDir: /mnt/agola/local/executor - toolboxPath: ./bin/agola-toolbox + # The directory containing the toolbox compiled for the various supported architectures + toolboxPath: ./bin runserviceURL: "http://agola-internal:4000" web: listenAddress: ":4001" diff --git a/internal/services/executor/driver/docker.go b/internal/services/executor/driver/docker.go index 4c73cbd..5c90fe6 100644 --- a/internal/services/executor/driver/docker.go +++ b/internal/services/executor/driver/docker.go @@ -46,6 +46,7 @@ type DockerDriver struct { initVolumeHostDir string toolboxPath string executorID string + arch common.Arch } func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxPath string) (*DockerDriver, error) { @@ -53,12 +54,14 @@ func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxP if err != nil { return nil, err } + return &DockerDriver{ logger: logger, client: cli, initVolumeHostDir: initVolumeHostDir, toolboxPath: toolboxPath, executorID: executorID, + arch: common.ArchFromString(runtime.GOARCH), }, nil } @@ -95,10 +98,15 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error { return err } - srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false) + toolboxExecPath, err := toolboxExecPath(d.toolboxPath, d.arch) + if err != nil { + return errors.Wrapf(err, "failed to get toolbox path for arch %q", d.arch) + } + srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false) if err != nil { return err } + srcInfo.RebaseName = "agola-toolbox" srcArchive, err := archive.TarResource(srcInfo) if err != nil { @@ -123,7 +131,7 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error { func (d *DockerDriver) Archs(ctx context.Context) ([]common.Arch, error) { // since we are using the local docker driver we can return our go arch information - return []common.Arch{common.ArchFromString(runtime.GOARCH)}, nil + return []common.Arch{d.arch}, nil } func (d *DockerDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Writer) (Pod, error) { diff --git a/internal/services/executor/driver/driver.go b/internal/services/executor/driver/driver.go index d8279da..8fde764 100644 --- a/internal/services/executor/driver/driver.go +++ b/internal/services/executor/driver/driver.go @@ -16,13 +16,18 @@ package driver import ( "context" + "fmt" "io" + "os" + "path/filepath" "github.com/sorintlab/agola/internal/common" "github.com/sorintlab/agola/internal/services/executor/registry" ) const ( + toolboxPrefix = "agola-toolbox" + labelPrefix = "agola.io/" agolaLabelKey = labelPrefix + "agola" @@ -99,3 +104,12 @@ type ExecConfig struct { Stderr io.Writer Tty bool } + +func toolboxExecPath(toolboxDir string, arch common.Arch) (string, error) { + toolboxPath := filepath.Join(toolboxDir, fmt.Sprintf("%s-linux-%s", toolboxPrefix, arch)) + _, err := os.Stat(toolboxPath) + if err != nil { + return "", err + } + return toolboxPath, nil +} diff --git a/internal/services/executor/driver/k8s.go b/internal/services/executor/driver/k8s.go index fb91c6d..1d434c2 100644 --- a/internal/services/executor/driver/k8s.go +++ b/internal/services/executor/driver/k8s.go @@ -15,6 +15,7 @@ package driver import ( + "bytes" "context" "encoding/json" "fmt" @@ -424,23 +425,69 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri fmt.Fprintf(out, "init container ready\n") - srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false) + coreclient, err := corev1client.NewForConfig(d.restconfig) if err != nil { return nil, err } + // get the pod arch + req := coreclient.RESTClient(). + Post(). + Namespace(pod.Namespace). + Resource("pods"). + Name(pod.Name). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "initcontainer", + Command: []string{"uname", "-m"}, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL()) + if err != nil { + return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL()) + } + + stdout := bytes.Buffer{} + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: out, + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to execute command on initcontainer") + } + osArch := strings.TrimSpace(stdout.String()) + + var arch common.Arch + switch osArch { + case "x86_64": + arch = common.ArchAMD64 + case "aarch64": + arch = common.ArchARM64 + default: + return nil, errors.Errorf("unsupported pod arch %q", osArch) + } + + // copy the toolbox for the pod arch + toolboxExecPath, err := toolboxExecPath(d.toolboxPath, arch) + if err != nil { + return nil, errors.Wrapf(err, "failed to get toolbox path for arch %q", arch) + } + srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false) + if err != nil { + return nil, err + } + srcInfo.RebaseName = "agola-toolbox" + srcArchive, err := archive.TarResource(srcInfo) if err != nil { return nil, err } defer srcArchive.Close() - coreclient, err := corev1client.NewForConfig(d.restconfig) - if err != nil { - return nil, err - } - - req := coreclient.RESTClient(). + req = coreclient.RESTClient(). Post(). Namespace(pod.Namespace). Resource("pods"). @@ -455,11 +502,12 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri TTY: false, }, scheme.ParameterCodec) - exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL()) + exec, err = remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL()) if err != nil { return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL()) } + fmt.Fprintf(out, "extracting toolbox\n") err = exec.Stream(remotecommand.StreamOptions{ Stdin: srcArchive, Stdout: out, @@ -468,6 +516,7 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri if err != nil { return nil, errors.Wrapf(err, "failed to execute command on initcontainer") } + fmt.Fprintf(out, "extracting toolbox done\n") req = coreclient.RESTClient(). Post(). diff --git a/internal/services/executor/executor.go b/internal/services/executor/executor.go index be456d7..ac7e794 100644 --- a/internal/services/executor/executor.go +++ b/internal/services/executor/executor.go @@ -25,7 +25,6 @@ import ( "net/http" "net/url" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -1244,14 +1243,7 @@ func NewExecutor(c *config.Executor) (*Executor, error) { var err error c.ToolboxPath, err = filepath.Abs(c.ToolboxPath) if err != nil { - return nil, errors.Wrapf(err, "cannot find \"agola-toolbox\" absolute path") - } - if c.ToolboxPath == "" { - path, err := exec.LookPath("agola-toolbox") - if err != nil { - return nil, errors.Errorf("cannot find \"agola-toolbox\" binaries in PATH, agola-toolbox path must be explicitly provided") - } - c.ToolboxPath = path + return nil, errors.Wrapf(err, "cannot determine \"agola-toolbox\" absolute path") } e := &Executor{