fd26e617b3
Since they're not types common to all the services but belongs to the configstore. Next step will be to make them local to the configstore and not directly used by other services since these types are also stored.
555 lines
15 KiB
Go
555 lines
15 KiB
Go
// Copyright 2019 Sorint.lab
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package tests
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
slog "agola.io/agola/internal/log"
|
|
"agola.io/agola/internal/services/config"
|
|
"agola.io/agola/internal/services/configstore"
|
|
cstypes "agola.io/agola/internal/services/configstore/types"
|
|
"agola.io/agola/internal/services/executor"
|
|
"agola.io/agola/internal/services/gateway"
|
|
gwapi "agola.io/agola/internal/services/gateway/api"
|
|
"agola.io/agola/internal/services/gitserver"
|
|
"agola.io/agola/internal/services/notification"
|
|
rsscheduler "agola.io/agola/internal/services/runservice"
|
|
rstypes "agola.io/agola/internal/services/runservice/types"
|
|
"agola.io/agola/internal/services/scheduler"
|
|
"agola.io/agola/internal/testutil"
|
|
"agola.io/agola/internal/util"
|
|
|
|
gtypes "code.gitea.io/gitea/modules/structs"
|
|
"code.gitea.io/sdk/gitea"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zapcore"
|
|
errors "golang.org/x/xerrors"
|
|
"gopkg.in/src-d/go-billy.v4/memfs"
|
|
"gopkg.in/src-d/go-git.v4"
|
|
gitconfig "gopkg.in/src-d/go-git.v4/config"
|
|
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
|
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
|
|
"gopkg.in/src-d/go-git.v4/storage/memory"
|
|
)
|
|
|
|
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
|
|
var logger = slog.New(level)
|
|
|
|
const (
|
|
giteaUser01 = "user01"
|
|
agolaUser01 = "user01"
|
|
)
|
|
|
|
func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
|
|
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
if err := tetcd.Start(); err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
if err := tetcd.WaitUp(30 * time.Second); err != nil {
|
|
t.Fatalf("error waiting on etcd up: %v", err)
|
|
}
|
|
return tetcd
|
|
}
|
|
|
|
func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) {
|
|
if tetcd.Etcd != nil {
|
|
_ = tetcd.Kill()
|
|
}
|
|
}
|
|
|
|
func setupGitea(t *testing.T, dir string) *testutil.TestGitea {
|
|
tgitea, err := testutil.NewTestGitea(t, logger, dir)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
if err := tgitea.Start(); err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
|
|
cmd := exec.Command(tgitea.GiteaPath, "admin", "create-user", "--name", giteaUser01, "--email", giteaUser01+"@example.com", "--password", "password", "--admin", "--config", tgitea.ConfigPath)
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("unexpected err: %v, out: %s", err, out)
|
|
}
|
|
|
|
return tgitea
|
|
}
|
|
|
|
func shutdownGitea(tgitea *testutil.TestGitea) {
|
|
tgitea.Kill()
|
|
}
|
|
|
|
func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config) (<-chan error, error) {
|
|
rs, err := rsscheduler.NewRunservice(ctx, &c.Runservice)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start run service scheduler: %w", err)
|
|
}
|
|
|
|
ex, err := executor.NewExecutor(&c.Executor)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start run service executor: %w", err)
|
|
}
|
|
|
|
cs, err := configstore.NewConfigstore(ctx, &c.Configstore)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start config store: %w", err)
|
|
}
|
|
|
|
sched, err := scheduler.NewScheduler(&c.Scheduler)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start scheduler: %w", err)
|
|
}
|
|
|
|
ns, err := notification.NewNotificationService(c)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start notification service: %w", err)
|
|
}
|
|
|
|
gw, err := gateway.NewGateway(c)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start gateway: %w", err)
|
|
}
|
|
|
|
gs, err := gitserver.NewGitserver(&c.Gitserver)
|
|
if err != nil {
|
|
return nil, errors.Errorf("failed to start git server: %w", err)
|
|
}
|
|
|
|
errCh := make(chan error)
|
|
|
|
go func() { errCh <- rs.Run(ctx) }()
|
|
go func() { errCh <- ex.Run(ctx) }()
|
|
go func() { errCh <- cs.Run(ctx) }()
|
|
go func() { errCh <- sched.Run(ctx) }()
|
|
go func() { errCh <- ns.Run(ctx) }()
|
|
go func() { errCh <- gw.Run(ctx) }()
|
|
go func() { errCh <- gs.Run(ctx) }()
|
|
|
|
// TODO(sgotti) find a better way to test that all is ready instead of sleeping
|
|
time.Sleep(5 * time.Second)
|
|
|
|
return errCh, nil
|
|
}
|
|
|
|
func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbeddedEtcd, *testutil.TestGitea, *config.Config) {
|
|
toolboxPath := os.Getenv("AGOLA_TOOLBOX_PATH")
|
|
if toolboxPath == "" {
|
|
t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined")
|
|
}
|
|
|
|
c := &config.Config{
|
|
ID: "agola",
|
|
Gateway: config.Gateway{
|
|
Debug: false,
|
|
APIExposedURL: "",
|
|
WebExposedURL: "",
|
|
RunserviceURL: "",
|
|
ConfigstoreURL: "",
|
|
GitserverURL: "",
|
|
Web: config.Web{
|
|
ListenAddress: "",
|
|
TLS: false,
|
|
},
|
|
TokenSigning: config.TokenSigning{
|
|
Duration: 12 * time.Hour,
|
|
Method: "hmac",
|
|
Key: "supersecretsigningkey",
|
|
},
|
|
AdminToken: "admintoken",
|
|
},
|
|
Scheduler: config.Scheduler{
|
|
Debug: false,
|
|
RunserviceURL: "",
|
|
},
|
|
Notification: config.Notification{
|
|
Debug: false,
|
|
WebExposedURL: "",
|
|
RunserviceURL: "",
|
|
ConfigstoreURL: "",
|
|
Etcd: config.Etcd{
|
|
Endpoints: "",
|
|
},
|
|
},
|
|
Runservice: config.Runservice{
|
|
Debug: false,
|
|
DataDir: filepath.Join(dir, "runservice"),
|
|
Web: config.Web{
|
|
ListenAddress: ":4000",
|
|
TLS: false,
|
|
},
|
|
Etcd: config.Etcd{
|
|
Endpoints: "",
|
|
},
|
|
ObjectStorage: config.ObjectStorage{
|
|
Type: "posix",
|
|
Path: filepath.Join(dir, "runservice/ost"),
|
|
},
|
|
RunCacheExpireInterval: 604800000000000,
|
|
},
|
|
Executor: config.Executor{
|
|
Debug: false,
|
|
DataDir: filepath.Join(dir, "executor"),
|
|
RunserviceURL: "",
|
|
ToolboxPath: toolboxPath,
|
|
Web: config.Web{
|
|
ListenAddress: ":4001",
|
|
TLS: false,
|
|
},
|
|
Driver: config.Driver{
|
|
Type: "docker",
|
|
},
|
|
Labels: map[string]string{},
|
|
ActiveTasksLimit: 2,
|
|
},
|
|
Configstore: config.Configstore{
|
|
Debug: false,
|
|
DataDir: filepath.Join(dir, "configstore"),
|
|
Web: config.Web{
|
|
ListenAddress: ":4002",
|
|
TLS: false,
|
|
},
|
|
Etcd: config.Etcd{
|
|
Endpoints: "",
|
|
},
|
|
ObjectStorage: config.ObjectStorage{
|
|
Type: "posix",
|
|
Path: filepath.Join(dir, "configstore/ost"),
|
|
},
|
|
},
|
|
Gitserver: config.Gitserver{
|
|
Debug: false,
|
|
DataDir: filepath.Join(dir, "gitserver"),
|
|
Web: config.Web{
|
|
ListenAddress: ":4003",
|
|
TLS: false,
|
|
},
|
|
Etcd: config.Etcd{
|
|
Endpoints: "",
|
|
},
|
|
},
|
|
}
|
|
|
|
tgitea := setupGitea(t, dir)
|
|
|
|
etcdDir := filepath.Join(dir, "etcd")
|
|
tetcd := setupEtcd(t, etcdDir)
|
|
|
|
c.Runservice.Etcd.Endpoints = tetcd.Endpoint
|
|
c.Configstore.Etcd.Endpoints = tetcd.Endpoint
|
|
|
|
_, gwPort, err := testutil.GetFreePort(true, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
_, csPort, err := testutil.GetFreePort(true, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
_, rsPort, err := testutil.GetFreePort(true, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
_, exPort, err := testutil.GetFreePort(true, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
listenAddress, gitServerPort, err := testutil.GetFreePort(true, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
gwURL := fmt.Sprintf("http://%s:%s", listenAddress, gwPort)
|
|
csURL := fmt.Sprintf("http://%s:%s", listenAddress, csPort)
|
|
rsURL := fmt.Sprintf("http://%s:%s", listenAddress, rsPort)
|
|
gitServerURL := fmt.Sprintf("http://%s:%s", listenAddress, gitServerPort)
|
|
|
|
c.Gateway.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, gwPort)
|
|
c.Configstore.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, csPort)
|
|
c.Runservice.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, rsPort)
|
|
c.Executor.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, exPort)
|
|
c.Gitserver.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, gitServerPort)
|
|
|
|
c.Gateway.APIExposedURL = gwURL
|
|
c.Gateway.WebExposedURL = gwURL
|
|
c.Gateway.RunserviceURL = rsURL
|
|
c.Gateway.ConfigstoreURL = csURL
|
|
c.Gateway.GitserverURL = gitServerURL
|
|
|
|
c.Scheduler.RunserviceURL = rsURL
|
|
|
|
c.Notification.WebExposedURL = gwURL
|
|
c.Notification.RunserviceURL = rsURL
|
|
c.Notification.ConfigstoreURL = csURL
|
|
|
|
c.Executor.RunserviceURL = rsURL
|
|
|
|
errCh, err := startAgola(ctx, t, dir, c)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
go func() {
|
|
err := <-errCh
|
|
if err != nil {
|
|
panic(fmt.Errorf("agola component returned error: %+v", err))
|
|
}
|
|
}()
|
|
|
|
return tetcd, tgitea, c
|
|
}
|
|
|
|
func TestCreateLinkedAccount(t *testing.T) {
|
|
dir, err := ioutil.TempDir("", "agola")
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tetcd, tgitea, c := setup(ctx, t, dir)
|
|
defer shutdownGitea(tgitea)
|
|
defer shutdownEtcd(tetcd)
|
|
|
|
createLinkedAccount(ctx, t, tgitea, c)
|
|
}
|
|
|
|
func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.TestGitea, c *config.Config) (string, string) {
|
|
giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort)
|
|
giteaClient := gitea.NewClient(giteaAPIURL, "")
|
|
|
|
giteaToken, err := giteaClient.CreateAccessToken(giteaUser01, "password", gtypes.CreateAccessTokenOption{Name: "token01"})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created gitea user token: %s", giteaToken.Token)
|
|
|
|
gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, "admintoken")
|
|
user, _, err := gwClient.CreateUser(ctx, &gwapi.CreateUserRequest{UserName: agolaUser01})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created agola user: %s", user.UserName)
|
|
|
|
token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapi.CreateUserTokenRequest{TokenName: "token01"})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created agola user token: %s", token.Token)
|
|
|
|
rs, _, err := gwClient.CreateRemoteSource(ctx, &gwapi.CreateRemoteSourceRequest{
|
|
Name: "gitea",
|
|
APIURL: giteaAPIURL,
|
|
Type: "gitea",
|
|
AuthType: "password",
|
|
SkipSSHHostKeyCheck: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created agola remote source: %s", rs.Name)
|
|
|
|
// From now use the user token
|
|
gwClient = gwapi.NewClient(c.Gateway.APIExposedURL, token.Token)
|
|
|
|
la, _, err := gwClient.CreateUserLA(ctx, agolaUser01, &gwapi.CreateUserLARequest{
|
|
RemoteSourceName: "gitea",
|
|
RemoteSourceLoginName: giteaUser01,
|
|
RemoteSourceLoginPassword: "password",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created user linked account: %s", util.Dump(la))
|
|
|
|
return giteaToken.Token, token.Token
|
|
}
|
|
|
|
func TestCreateProject(t *testing.T) {
|
|
dir, err := ioutil.TempDir("", "agola")
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tetcd, tgitea, c := setup(ctx, t, dir)
|
|
defer shutdownGitea(tgitea)
|
|
defer shutdownEtcd(tetcd)
|
|
|
|
giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort)
|
|
|
|
giteaToken, token := createLinkedAccount(ctx, t, tgitea, c)
|
|
|
|
giteaClient := gitea.NewClient(giteaAPIURL, giteaToken)
|
|
gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, token)
|
|
|
|
createProject(ctx, t, giteaClient, gwClient)
|
|
}
|
|
|
|
func createProject(ctx context.Context, t *testing.T, giteaClient *gitea.Client, gwClient *gwapi.Client) (*gtypes.Repository, *gwapi.ProjectResponse) {
|
|
giteaRepo, err := giteaClient.CreateRepo(gtypes.CreateRepoOption{
|
|
Name: "repo01",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
t.Logf("created gitea repo: %s", giteaRepo.Name)
|
|
|
|
project, _, err := gwClient.CreateProject(ctx, &gwapi.CreateProjectRequest{
|
|
Name: "project01",
|
|
ParentRef: path.Join("user", agolaUser01),
|
|
RemoteSourceName: "gitea",
|
|
RepoPath: path.Join(giteaUser01, "repo01"),
|
|
Visibility: cstypes.VisibilityPublic,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
return giteaRepo, project
|
|
}
|
|
|
|
func TestRun(t *testing.T) {
|
|
dir, err := ioutil.TempDir("", "agola")
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tetcd, tgitea, c := setup(ctx, t, dir)
|
|
defer shutdownGitea(tgitea)
|
|
defer shutdownEtcd(tetcd)
|
|
|
|
giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort)
|
|
|
|
giteaToken, token := createLinkedAccount(ctx, t, tgitea, c)
|
|
|
|
giteaClient := gitea.NewClient(giteaAPIURL, giteaToken)
|
|
gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, token)
|
|
|
|
giteaRepo, project := createProject(ctx, t, giteaClient, gwClient)
|
|
|
|
gitfs := memfs.New()
|
|
f, err := gitfs.Create(".agola/config.jsonnet")
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
_, err = f.Write([]byte(
|
|
`{
|
|
runs: [
|
|
{
|
|
name: 'run01',
|
|
tasks: [
|
|
{
|
|
name: 'task01',
|
|
runtime: {
|
|
containers: [
|
|
{
|
|
image: 'busybox',
|
|
},
|
|
],
|
|
},
|
|
steps: [
|
|
{ type: 'run', command: 'env' },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
}
|
|
`))
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
r, err := git.Init(memory.NewStorage(), gitfs)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
if _, err := r.CreateRemote(&gitconfig.RemoteConfig{
|
|
Name: "origin",
|
|
URLs: []string{giteaRepo.CloneURL},
|
|
}); err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
wt, err := r.Worktree()
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
if _, err := wt.Add(".agola/config.jsonnet"); err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
_, err = wt.Commit("commit", &git.CommitOptions{
|
|
Author: &object.Signature{
|
|
Name: "user01",
|
|
Email: "user01@example.com",
|
|
When: time.Now(),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
t.Logf("sshurl: %s", giteaRepo.CloneURL)
|
|
if err := r.Push(&git.PushOptions{
|
|
RemoteName: "origin",
|
|
Auth: &http.BasicAuth{
|
|
Username: giteaUser01,
|
|
Password: giteaToken,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
// TODO(sgotti) add an util to wait for a run phase
|
|
time.Sleep(10 * time.Second)
|
|
|
|
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/project", project.ID)}, nil, "", 0, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
t.Logf("runs: %s", util.Dump(runs))
|
|
|
|
if len(runs) != 1 {
|
|
t.Fatalf("expected 1 run got: %d", len(runs))
|
|
}
|
|
|
|
run := runs[0]
|
|
if run.Phase != rstypes.RunPhaseFinished {
|
|
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
|
|
}
|
|
}
|