From 2de91549a3461717a46bd92c3c364d4509cd7b01 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Wed, 15 Jan 2020 12:01:46 +0100 Subject: [PATCH] tests: improve services logging During tests provide a zaptest Logger so all services output will be redirected to golang testing logger. When multiple services of the same type are provided add a unique name field to distinguish them. --- cmd/agola/cmd/serve.go | 14 ++--- internal/datamanager/datamanager_test.go | 60 +++++++++++------- internal/services/configstore/configstore.go | 6 +- .../services/configstore/configstore_test.go | 63 ++++++++++++------- internal/services/executor/driver/docker.go | 8 ++- .../services/executor/driver/docker_test.go | 12 ++-- internal/services/executor/driver/k8s_test.go | 4 ++ internal/services/executor/executor.go | 6 +- internal/services/gateway/gateway.go | 7 ++- internal/services/gitserver/main.go | 6 +- .../services/notification/notification.go | 7 ++- internal/services/runservice/runservice.go | 12 +++- internal/services/runservice/scheduler.go | 7 --- internal/services/scheduler/scheduler.go | 6 +- internal/testutil/utils.go | 3 +- tests/setup_test.go | 32 +++++----- 16 files changed, 160 insertions(+), 93 deletions(-) diff --git a/cmd/agola/cmd/serve.go b/cmd/agola/cmd/serve.go index c672257..6318a21 100644 --- a/cmd/agola/cmd/serve.go +++ b/cmd/agola/cmd/serve.go @@ -142,7 +142,7 @@ func serve(cmd *cobra.Command, args []string) error { var rs *rsscheduler.Runservice if isComponentEnabled("runservice") { - rs, err = rsscheduler.NewRunservice(ctx, &c.Runservice) + rs, err = rsscheduler.NewRunservice(ctx, nil, &c.Runservice) if err != nil { return errors.Errorf("failed to start run service scheduler: %w", err) } @@ -150,7 +150,7 @@ func serve(cmd *cobra.Command, args []string) error { var ex *rsexecutor.Executor if isComponentEnabled("executor") { - ex, err = executor.NewExecutor(&c.Executor) + ex, err = executor.NewExecutor(ctx, nil, &c.Executor) if err != nil { return errors.Errorf("failed to start run service executor: %w", err) } @@ -158,7 +158,7 @@ func serve(cmd *cobra.Command, args []string) error { var cs *configstore.Configstore if isComponentEnabled("configstore") { - cs, err = configstore.NewConfigstore(ctx, &c.Configstore) + cs, err = configstore.NewConfigstore(ctx, nil, &c.Configstore) if err != nil { return errors.Errorf("failed to start config store: %w", err) } @@ -166,7 +166,7 @@ func serve(cmd *cobra.Command, args []string) error { var sched *scheduler.Scheduler if isComponentEnabled("scheduler") { - sched, err = scheduler.NewScheduler(&c.Scheduler) + sched, err = scheduler.NewScheduler(ctx, nil, &c.Scheduler) if err != nil { return errors.Errorf("failed to start scheduler: %w", err) } @@ -174,7 +174,7 @@ func serve(cmd *cobra.Command, args []string) error { var ns *notification.NotificationService if isComponentEnabled("notification") { - ns, err = notification.NewNotificationService(c) + ns, err = notification.NewNotificationService(ctx, nil, c) if err != nil { return errors.Errorf("failed to start notification service: %w", err) } @@ -182,7 +182,7 @@ func serve(cmd *cobra.Command, args []string) error { var gw *gateway.Gateway if isComponentEnabled("gateway") { - gw, err = gateway.NewGateway(c) + gw, err = gateway.NewGateway(ctx, nil, c) if err != nil { return errors.Errorf("failed to start gateway: %w", err) } @@ -190,7 +190,7 @@ func serve(cmd *cobra.Command, args []string) error { var gs *gitserver.Gitserver if isComponentEnabled("gitserver") { - gs, err = gitserver.NewGitserver(&c.Gitserver) + gs, err = gitserver.NewGitserver(ctx, nil, &c.Gitserver) if err != nil { return errors.Errorf("failed to start git server: %w", err) } diff --git a/internal/datamanager/datamanager_test.go b/internal/datamanager/datamanager_test.go index 5d056c9..fe3baf1 100644 --- a/internal/datamanager/datamanager_test.go +++ b/internal/datamanager/datamanager_test.go @@ -29,21 +29,17 @@ import ( "testing" "time" - slog "agola.io/agola/internal/log" "agola.io/agola/internal/objectstorage" "agola.io/agola/internal/testutil" "agola.io/agola/internal/util" "github.com/google/go-cmp/cmp" "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" errors "golang.org/x/xerrors" ) -var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) -var logger = slog.New(level) - -func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd { +func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd { tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir) if err != nil { t.Fatalf("unexpected err: %v", err) @@ -52,7 +48,7 @@ func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd { t.Fatalf("unexpected err: %v", err) } if err := tetcd.WaitUp(30 * time.Second); err != nil { - t.Fatalf("error waiting on store up: %v", err) + t.Fatalf("error waiting on etcd up: %v", err) } return tetcd } @@ -70,11 +66,13 @@ func TestEtcdReset(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) ctx, cancel := context.WithCancel(context.Background()) @@ -134,7 +132,7 @@ func TestEtcdReset(t *testing.T) { t.Logf("resetting etcd") os.RemoveAll(etcdDir) t.Logf("starting etcd") - tetcd = setupEtcd(t, etcdDir) + tetcd = setupEtcd(t, logger, etcdDir) if err := tetcd.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } @@ -177,11 +175,13 @@ func TestEtcdResetWalsGap(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) ctx, cancel := context.WithCancel(context.Background()) @@ -241,7 +241,7 @@ func TestEtcdResetWalsGap(t *testing.T) { t.Logf("resetting etcd") os.RemoveAll(etcdDir) t.Logf("starting etcd") - tetcd = setupEtcd(t, etcdDir) + tetcd = setupEtcd(t, logger, etcdDir) if err := tetcd.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } @@ -303,11 +303,13 @@ func TestConcurrentUpdate(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -394,11 +396,13 @@ func TestEtcdWalCleaner(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -468,11 +472,13 @@ func TestReadObject(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -761,11 +767,13 @@ func testCheckpoint(t *testing.T, basePath string) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -977,11 +985,13 @@ func TestRead(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -1091,11 +1101,13 @@ func testClean(t *testing.T, basePath string) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -1209,11 +1221,13 @@ func testCleanConcurrentCheckpoint(t *testing.T, basePath string) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -1338,11 +1352,13 @@ func testStorageWalCleaner(t *testing.T, basePath string) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) ctx := context.Background() @@ -1478,11 +1494,13 @@ func TestExportImport(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) ctx, cancel := context.WithCancel(context.Background()) @@ -1586,7 +1604,7 @@ func TestExportImport(t *testing.T) { t.Logf("resetting etcd") os.RemoveAll(etcdDir) t.Logf("starting etcd") - tetcd = setupEtcd(t, etcdDir) + tetcd = setupEtcd(t, logger, etcdDir) if err := tetcd.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } diff --git a/internal/services/configstore/configstore.go b/internal/services/configstore/configstore.go index 2e6ea6b..406899e 100644 --- a/internal/services/configstore/configstore.go +++ b/internal/services/configstore/configstore.go @@ -120,10 +120,14 @@ type Configstore struct { maintenanceMode bool } -func NewConfigstore(ctx context.Context, c *config.Configstore) (*Configstore, error) { +func NewConfigstore(ctx context.Context, l *zap.Logger, c *config.Configstore) (*Configstore, error) { + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() ost, err := scommon.NewObjectStorage(&c.ObjectStorage) if err != nil { diff --git a/internal/services/configstore/configstore_test.go b/internal/services/configstore/configstore_test.go index 0a09bfd..9be837a 100644 --- a/internal/services/configstore/configstore_test.go +++ b/internal/services/configstore/configstore_test.go @@ -29,15 +29,17 @@ import ( "agola.io/agola/internal/db" "agola.io/agola/internal/services/config" - action "agola.io/agola/internal/services/configstore/action" + "agola.io/agola/internal/services/configstore/action" "agola.io/agola/internal/testutil" "agola.io/agola/internal/util" "agola.io/agola/services/configstore/types" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) -func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd { +func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd { tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir) if err != nil { t.Fatalf("unexpected err: %v", err) @@ -57,12 +59,12 @@ func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) { } } -func setupConfigstore(ctx context.Context, t *testing.T, dir string) (*Configstore, *testutil.TestEmbeddedEtcd) { +func setupConfigstore(ctx context.Context, t *testing.T, logger *zap.Logger, dir string) (*Configstore, *testutil.TestEmbeddedEtcd) { etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) listenAddress, port, err := testutil.GetFreePort(true, false) if err != nil { @@ -92,7 +94,7 @@ func setupConfigstore(ctx context.Context, t *testing.T, dir string) (*Configsto csConfig.DataDir = csDir csConfig.Web.ListenAddress = net.JoinHostPort(listenAddress, port) - cs, err := NewConfigstore(ctx, &csConfig) + cs, err := NewConfigstore(ctx, logger, &csConfig) if err != nil { t.Fatalf("err: %v", err) } @@ -127,11 +129,13 @@ func TestResync(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) listenAddress1, port1, err := testutil.GetFreePort(true, false) @@ -184,11 +188,11 @@ func TestResync(t *testing.T) { cs2Config.DataDir = csDir2 cs2Config.Web.ListenAddress = net.JoinHostPort(listenAddress2, port2) - cs1, err := NewConfigstore(ctx, &cs1Config) + cs1, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs1")), &cs1Config) if err != nil { t.Fatalf("err: %v", err) } - cs2, err := NewConfigstore(ctx, &cs2Config) + cs2, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config) if err != nil { t.Fatalf("err: %v", err) } @@ -233,7 +237,7 @@ func TestResync(t *testing.T) { // start cs2 // it should resync from wals since the etcd revision as been compacted - cs2, err = NewConfigstore(ctx, &cs2Config) + cs2, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config) if err != nil { t.Fatalf("err: %v", err) } @@ -271,7 +275,7 @@ func TestResync(t *testing.T) { cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3) log.Infof("starting cs3") - cs3, err := NewConfigstore(ctx, &cs3Config) + cs3, err := NewConfigstore(ctx, logger, &cs3Config) if err != nil { t.Fatalf("err: %v", err) } @@ -301,11 +305,13 @@ func TestExportImport(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + etcdDir, err := ioutil.TempDir(dir, "etcd") if err != nil { t.Fatalf("unexpected err: %v", err) } - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) defer shutdownEtcd(tetcd) listenAddress1, port1, err := testutil.GetFreePort(true, false) @@ -362,15 +368,15 @@ func TestExportImport(t *testing.T) { cs3Config.DataDir = csDir3 cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3) - cs1, err := NewConfigstore(ctx, &cs1Config) + cs1, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs1")), &cs1Config) if err != nil { t.Fatalf("err: %v", err) } - cs2, err := NewConfigstore(ctx, &cs2Config) + cs2, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config) if err != nil { t.Fatalf("err: %v", err) } - cs3, err := NewConfigstore(ctx, &cs3Config) + cs3, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs3")), &cs3Config) if err != nil { t.Fatalf("err: %v", err) } @@ -459,7 +465,7 @@ func TestExportImport(t *testing.T) { // start cs2 // it should do a full resync since we have imported new data and there's now wal in etcd - cs2, err = NewConfigstore(ctx, &cs2Config) + cs2, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config) if err != nil { t.Fatalf("err: %v", err) } @@ -503,7 +509,7 @@ func TestExportImport(t *testing.T) { // start cs3 // it should do a full resync since we have imported new data and there're some wals with a different epoch - cs3, err = NewConfigstore(ctx, &cs3Config) + cs3, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs3")), &cs3Config) if err != nil { t.Fatalf("err: %v", err) } @@ -549,8 +555,9 @@ func TestUser(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -616,8 +623,9 @@ func TestProjectGroupsAndProjectsCreate(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -763,8 +771,9 @@ func TestProjectUpdate(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -839,8 +848,9 @@ func TestProjectGroupUpdate(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -989,8 +999,9 @@ func TestProjectGroupDelete(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -1045,8 +1056,9 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -1180,8 +1192,9 @@ func TestOrgMembers(t *testing.T) { defer os.RemoveAll(dir) ctx := context.Background() + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") @@ -1269,6 +1282,8 @@ func TestRemoteSource(t *testing.T) { } defer os.RemoveAll(dir) + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + tests := []struct { name string f func(ctx context.Context, t *testing.T, cs *Configstore) @@ -1417,7 +1432,7 @@ func TestRemoteSource(t *testing.T) { } ctx := context.Background() - cs, tetcd := setupConfigstore(ctx, t, dir) + cs, tetcd := setupConfigstore(ctx, t, logger, dir) defer shutdownEtcd(tetcd) t.Logf("starting cs") diff --git a/internal/services/executor/driver/docker.go b/internal/services/executor/driver/docker.go index 7fbc3df..9c2fd2c 100644 --- a/internal/services/executor/driver/docker.go +++ b/internal/services/executor/driver/docker.go @@ -15,13 +15,13 @@ package driver import ( + "bufio" "context" "encoding/base64" "encoding/json" "fmt" "io" "io/ioutil" - "os" "path/filepath" "runtime" "sort" @@ -76,7 +76,11 @@ func (d *DockerDriver) createToolboxVolume(ctx context.Context, podID string) (* if err != nil { return nil, err } - if _, err := io.Copy(os.Stdout, reader); err != nil { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + d.log.Infof("create toolbox volume image pull output: %s", scanner.Text()) + } + if err := scanner.Err(); err != nil { return nil, err } diff --git a/internal/services/executor/driver/docker_test.go b/internal/services/executor/driver/docker_test.go index 1312530..fecf4e4 100644 --- a/internal/services/executor/driver/docker_test.go +++ b/internal/services/executor/driver/docker_test.go @@ -22,19 +22,15 @@ import ( "testing" "time" - slog "agola.io/agola/internal/log" "agola.io/agola/internal/testutil" + "github.com/docker/docker/api/types" - uuid "github.com/satori/go.uuid" - "github.com/google/go-cmp/cmp" + uuid "github.com/satori/go.uuid" "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" ) -var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) -var logger = slog.New(level) - func TestDockerPod(t *testing.T) { if os.Getenv("SKIP_DOCKER_TESTS") == "1" { t.Skip("skipping since env var SKIP_DOCKER_TESTS is 1") @@ -44,6 +40,8 @@ func TestDockerPod(t *testing.T) { t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined") } + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + d, err := NewDockerDriver(logger, "executorid01", toolboxPath) if err != nil { t.Fatalf("unexpected err: %v", err) diff --git a/internal/services/executor/driver/k8s_test.go b/internal/services/executor/driver/k8s_test.go index 8ffd0af..f42f507 100644 --- a/internal/services/executor/driver/k8s_test.go +++ b/internal/services/executor/driver/k8s_test.go @@ -26,6 +26,8 @@ import ( "agola.io/agola/internal/testutil" uuid "github.com/satori/go.uuid" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) func TestK8sPod(t *testing.T) { @@ -37,6 +39,8 @@ func TestK8sPod(t *testing.T) { t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined") } + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + d, err := NewK8sDriver(logger, "executorid01", toolboxPath) if err != nil { t.Fatalf("unexpected err: %v", err) diff --git a/internal/services/executor/executor.go b/internal/services/executor/executor.go index b616d4f..0bfa469 100644 --- a/internal/services/executor/executor.go +++ b/internal/services/executor/executor.go @@ -1339,10 +1339,14 @@ type Executor struct { dynamic bool } -func NewExecutor(c *config.Executor) (*Executor, error) { +func NewExecutor(ctx context.Context, l *zap.Logger, c *config.Executor) (*Executor, error) { + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() var err error c.ToolboxPath, err = filepath.Abs(c.ToolboxPath) diff --git a/internal/services/gateway/gateway.go b/internal/services/gateway/gateway.go index 9632479..626ea32 100644 --- a/internal/services/gateway/gateway.go +++ b/internal/services/gateway/gateway.go @@ -58,11 +58,16 @@ type Gateway struct { sd *common.TokenSigningData } -func NewGateway(gc *config.Config) (*Gateway, error) { +func NewGateway(ctx context.Context, l *zap.Logger, gc *config.Config) (*Gateway, error) { c := &gc.Gateway + + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() if c.Web.ListenAddress == "" { return nil, errors.Errorf("listen address undefined") diff --git a/internal/services/gitserver/main.go b/internal/services/gitserver/main.go index 9e99c61..23f6858 100644 --- a/internal/services/gitserver/main.go +++ b/internal/services/gitserver/main.go @@ -130,10 +130,14 @@ type Gitserver struct { c *config.Gitserver } -func NewGitserver(c *config.Gitserver) (*Gitserver, error) { +func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Gitserver, error) { + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() return &Gitserver{ c: c, diff --git a/internal/services/notification/notification.go b/internal/services/notification/notification.go index d3000f2..a2d6831 100644 --- a/internal/services/notification/notification.go +++ b/internal/services/notification/notification.go @@ -42,11 +42,16 @@ type NotificationService struct { configstoreClient *csclient.Client } -func NewNotificationService(gc *config.Config) (*NotificationService, error) { +func NewNotificationService(ctx context.Context, l *zap.Logger, gc *config.Config) (*NotificationService, error) { c := &gc.Notification + + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() e, err := common.NewEtcd(&c.Etcd, logger, "notification") if err != nil { diff --git a/internal/services/runservice/runservice.go b/internal/services/runservice/runservice.go index bfdcbf1..9805afa 100644 --- a/internal/services/runservice/runservice.go +++ b/internal/services/runservice/runservice.go @@ -25,6 +25,7 @@ import ( scommon "agola.io/agola/internal/common" "agola.io/agola/internal/datamanager" "agola.io/agola/internal/etcd" + slog "agola.io/agola/internal/log" "agola.io/agola/internal/objectstorage" "agola.io/agola/internal/services/config" "agola.io/agola/internal/services/runservice/action" @@ -37,9 +38,14 @@ import ( "github.com/gorilla/mux" etcdclientv3 "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/zap" "go.uber.org/zap/zapcore" ) +var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) +var logger = slog.New(level) +var log = logger.Sugar() + // etcdPingerLoop periodically updates a key. // This is used by watchers to inform the client of the current revision // this is needed since if other users are updating other unwatched keys on @@ -141,10 +147,14 @@ type Runservice struct { maintenanceMode bool } -func NewRunservice(ctx context.Context, c *config.Runservice) (*Runservice, error) { +func NewRunservice(ctx context.Context, l *zap.Logger, c *config.Runservice) (*Runservice, error) { + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() ost, err := scommon.NewObjectStorage(&c.ObjectStorage) if err != nil { diff --git a/internal/services/runservice/scheduler.go b/internal/services/runservice/scheduler.go index 744bfe9..9816ea4 100644 --- a/internal/services/runservice/scheduler.go +++ b/internal/services/runservice/scheduler.go @@ -25,7 +25,6 @@ import ( "agola.io/agola/internal/datamanager" "agola.io/agola/internal/etcd" - slog "agola.io/agola/internal/log" "agola.io/agola/internal/objectstorage" "agola.io/agola/internal/runconfig" "agola.io/agola/internal/services/runservice/common" @@ -35,8 +34,6 @@ import ( etcdclientv3 "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/clientv3/concurrency" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" errors "golang.org/x/xerrors" ) @@ -47,10 +44,6 @@ const ( defaultExecutorNotAliveInterval = 60 * time.Second ) -var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) -var logger = slog.New(level) -var log = logger.Sugar() - func (s *Runservice) runActiveExecutorTasks(ctx context.Context, runID string) ([]*types.ExecutorTask, error) { // the real source of active tasks is the number of executor tasks in etcd // we can't rely on RunTask.Status since it's only updated when receiveing diff --git a/internal/services/scheduler/scheduler.go b/internal/services/scheduler/scheduler.go index 3fcda2c..833325a 100644 --- a/internal/services/scheduler/scheduler.go +++ b/internal/services/scheduler/scheduler.go @@ -197,10 +197,14 @@ type Scheduler struct { runserviceClient *rsclient.Client } -func NewScheduler(c *config.Scheduler) (*Scheduler, error) { +func NewScheduler(ctx context.Context, l *zap.Logger, c *config.Scheduler) (*Scheduler, error) { + if l != nil { + logger = l + } if c.Debug { level.SetLevel(zapcore.DebugLevel) } + log = logger.Sugar() return &Scheduler{ c: c, diff --git a/internal/testutil/utils.go b/internal/testutil/utils.go index a2bba44..0faeb39 100644 --- a/internal/testutil/utils.go +++ b/internal/testutil/utils.go @@ -167,6 +167,7 @@ func NewTestEmbeddedEtcd(t *testing.T, logger *zap.Logger, dir string, a ...stri cfg.Name = uid cfg.Dir = dataDir cfg.Logger = "zap" + cfg.LogLevel = "fatal" cfg.LogOutputs = []string{"stdout"} lcurl, _ := url.Parse(fmt.Sprintf("http://%s:%s", listenAddress, port)) lpurl, _ := url.Parse(fmt.Sprintf("http://%s:%s", listenAddress2, port2)) @@ -451,7 +452,7 @@ type TestGitea struct { SSHPort string } -func NewTestGitea(t *testing.T, logger *zap.Logger, dir, dockerBridgeAddress string, a ...string) (*TestGitea, error) { +func NewTestGitea(t *testing.T, dir, dockerBridgeAddress string, a ...string) (*TestGitea, error) { u := uuid.NewV4() uid := fmt.Sprintf("%x", u[:4]) diff --git a/tests/setup_test.go b/tests/setup_test.go index e67ce8e..2017aac 100644 --- a/tests/setup_test.go +++ b/tests/setup_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - slog "agola.io/agola/internal/log" "agola.io/agola/internal/services/config" "agola.io/agola/internal/services/configstore" "agola.io/agola/internal/services/executor" @@ -44,7 +43,7 @@ import ( "code.gitea.io/sdk/gitea" "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" errors "golang.org/x/xerrors" "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-billy.v4/osfs" @@ -57,15 +56,12 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" ) -var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) -var logger = slog.New(level) - const ( giteaUser01 = "user01" agolaUser01 = "user01" ) -func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd { +func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd { tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir) if err != nil { t.Fatalf("unexpected err: %v", err) @@ -86,7 +82,7 @@ func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) { } func setupGitea(t *testing.T, dir, dockerBridgeAddress string) *testutil.TestGitea { - tgitea, err := testutil.NewTestGitea(t, logger, dir, dockerBridgeAddress) + tgitea, err := testutil.NewTestGitea(t, dir, dockerBridgeAddress) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -128,38 +124,38 @@ func shutdownGitea(tgitea *testutil.TestGitea) { tgitea.Kill() } -func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config) (<-chan error, error) { - rs, err := rsscheduler.NewRunservice(ctx, &c.Runservice) +func startAgola(ctx context.Context, t *testing.T, logger *zap.Logger, dir string, c *config.Config) (<-chan error, error) { + rs, err := rsscheduler.NewRunservice(ctx, logger, &c.Runservice) if err != nil { return nil, errors.Errorf("failed to start run service scheduler: %w", err) } - ex, err := executor.NewExecutor(&c.Executor) + ex, err := executor.NewExecutor(ctx, logger, &c.Executor) if err != nil { return nil, errors.Errorf("failed to start run service executor: %w", err) } - cs, err := configstore.NewConfigstore(ctx, &c.Configstore) + cs, err := configstore.NewConfigstore(ctx, logger, &c.Configstore) if err != nil { return nil, errors.Errorf("failed to start config store: %w", err) } - sched, err := scheduler.NewScheduler(&c.Scheduler) + sched, err := scheduler.NewScheduler(ctx, logger, &c.Scheduler) if err != nil { return nil, errors.Errorf("failed to start scheduler: %w", err) } - ns, err := notification.NewNotificationService(c) + ns, err := notification.NewNotificationService(ctx, logger, c) if err != nil { return nil, errors.Errorf("failed to start notification service: %w", err) } - gw, err := gateway.NewGateway(c) + gw, err := gateway.NewGateway(ctx, logger, c) if err != nil { return nil, errors.Errorf("failed to start gateway: %w", err) } - gs, err := gitserver.NewGitserver(&c.Gitserver) + gs, err := gitserver.NewGitserver(ctx, logger, &c.Gitserver) if err != nil { return nil, errors.Errorf("failed to start git server: %w", err) } @@ -181,6 +177,8 @@ func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config) } func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbeddedEtcd, *testutil.TestGitea, *config.Config) { + logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel)) + dockerBridgeAddress := os.Getenv("DOCKER_BRIDGE_ADDRESS") if dockerBridgeAddress == "" { dockerBridgeAddress = "172.17.0.1" @@ -285,7 +283,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde tgitea := setupGitea(t, dir, dockerBridgeAddress) etcdDir := filepath.Join(dir, "etcd") - tetcd := setupEtcd(t, etcdDir) + tetcd := setupEtcd(t, logger, etcdDir) c.Runservice.Etcd.Endpoints = tetcd.Endpoint c.Configstore.Etcd.Endpoints = tetcd.Endpoint @@ -336,7 +334,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde c.Executor.RunserviceURL = rsURL - errCh, err := startAgola(ctx, t, dir, c) + errCh, err := startAgola(ctx, t, logger, dir, c) if err != nil { t.Fatalf("unexpected err: %v", err) }