a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #194 from sgotti/tests_improve_services_logging 

tests: improve services logging
This commit is contained in:
Simone Gotti 2020-01-15 15:01:19 +01:00 committed by GitHub
parent 180d8dd819 2de91549a3
commit fd9a69bbb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 160 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/agola/cmd/serve.go
View File

@ -142,7 +142,7 @@ func serve(cmd *cobra.Command, args []string) error {
var rs *rsscheduler.Runservice
if isComponentEnabled("runservice") {
rs, err = rsscheduler.NewRunservice(ctx, &c.Runservice)
rs, err = rsscheduler.NewRunservice(ctx, nil, &c.Runservice)
if err != nil {
return errors.Errorf("failed to start run service scheduler: %w", err)
}
@ -150,7 +150,7 @@ func serve(cmd *cobra.Command, args []string) error {
var ex *rsexecutor.Executor
if isComponentEnabled("executor") {
ex, err = executor.NewExecutor(&c.Executor)
ex, err = executor.NewExecutor(ctx, nil, &c.Executor)
if err != nil {
return errors.Errorf("failed to start run service executor: %w", err)
}
@ -158,7 +158,7 @@ func serve(cmd *cobra.Command, args []string) error {
var cs *configstore.Configstore
if isComponentEnabled("configstore") {
cs, err = configstore.NewConfigstore(ctx, &c.Configstore)
cs, err = configstore.NewConfigstore(ctx, nil, &c.Configstore)
if err != nil {
return errors.Errorf("failed to start config store: %w", err)
}
@ -166,7 +166,7 @@ func serve(cmd *cobra.Command, args []string) error {
var sched *scheduler.Scheduler
if isComponentEnabled("scheduler") {
sched, err = scheduler.NewScheduler(&c.Scheduler)
sched, err = scheduler.NewScheduler(ctx, nil, &c.Scheduler)
if err != nil {
return errors.Errorf("failed to start scheduler: %w", err)
}
@ -174,7 +174,7 @@ func serve(cmd *cobra.Command, args []string) error {
var ns *notification.NotificationService
if isComponentEnabled("notification") {
ns, err = notification.NewNotificationService(c)
ns, err = notification.NewNotificationService(ctx, nil, c)
if err != nil {
return errors.Errorf("failed to start notification service: %w", err)
}
@ -182,7 +182,7 @@ func serve(cmd *cobra.Command, args []string) error {
var gw *gateway.Gateway
if isComponentEnabled("gateway") {
gw, err = gateway.NewGateway(c)
gw, err = gateway.NewGateway(ctx, nil, c)
if err != nil {
return errors.Errorf("failed to start gateway: %w", err)
}
@ -190,7 +190,7 @@ func serve(cmd *cobra.Command, args []string) error {
var gs *gitserver.Gitserver
if isComponentEnabled("gitserver") {
gs, err = gitserver.NewGitserver(&c.Gitserver)
gs, err = gitserver.NewGitserver(ctx, nil, &c.Gitserver)
if err != nil {
return errors.Errorf("failed to start git server: %w", err)
}

60
internal/datamanager/datamanager_test.go
View File

@ -29,21 +29,17 @@ import (
"testing"
"time"
slog "agola.io/agola/internal/log"
"agola.io/agola/internal/objectstorage"
"agola.io/agola/internal/testutil"
"agola.io/agola/internal/util"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest"
errors "golang.org/x/xerrors"
)
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd {
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
@ -52,7 +48,7 @@ func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
t.Fatalf("unexpected err: %v", err)
}
if err := tetcd.WaitUp(30 * time.Second); err != nil {
t.Fatalf("error waiting on store up: %v", err)
t.Fatalf("error waiting on etcd up: %v", err)
}
return tetcd
}
@ -70,11 +66,13 @@ func TestEtcdReset(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
ctx, cancel := context.WithCancel(context.Background())
@ -134,7 +132,7 @@ func TestEtcdReset(t *testing.T) {
t.Logf("resetting etcd")
os.RemoveAll(etcdDir)
t.Logf("starting etcd")
tetcd = setupEtcd(t, etcdDir)
tetcd = setupEtcd(t, logger, etcdDir)
if err := tetcd.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
@ -177,11 +175,13 @@ func TestEtcdResetWalsGap(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
ctx, cancel := context.WithCancel(context.Background())
@ -241,7 +241,7 @@ func TestEtcdResetWalsGap(t *testing.T) {
t.Logf("resetting etcd")
os.RemoveAll(etcdDir)
t.Logf("starting etcd")
tetcd = setupEtcd(t, etcdDir)
tetcd = setupEtcd(t, logger, etcdDir)
if err := tetcd.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
@ -303,11 +303,13 @@ func TestConcurrentUpdate(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -394,11 +396,13 @@ func TestEtcdWalCleaner(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -468,11 +472,13 @@ func TestReadObject(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -761,11 +767,13 @@ func testCheckpoint(t *testing.T, basePath string) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -977,11 +985,13 @@ func TestRead(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -1091,11 +1101,13 @@ func testClean(t *testing.T, basePath string) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -1209,11 +1221,13 @@ func testCleanConcurrentCheckpoint(t *testing.T, basePath string) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -1338,11 +1352,13 @@ func testStorageWalCleaner(t *testing.T, basePath string) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
ctx := context.Background()
@ -1478,11 +1494,13 @@ func TestExportImport(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
ctx, cancel := context.WithCancel(context.Background())
@ -1586,7 +1604,7 @@ func TestExportImport(t *testing.T) {
t.Logf("resetting etcd")
os.RemoveAll(etcdDir)
t.Logf("starting etcd")
tetcd = setupEtcd(t, etcdDir)
tetcd = setupEtcd(t, logger, etcdDir)
if err := tetcd.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}

6
internal/services/configstore/configstore.go
View File

@ -120,10 +120,14 @@ type Configstore struct {
maintenanceMode bool
}
func NewConfigstore(ctx context.Context, c *config.Configstore) (*Configstore, error) {
func NewConfigstore(ctx context.Context, l *zap.Logger, c *config.Configstore) (*Configstore, error) {
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
ost, err := scommon.NewObjectStorage(&c.ObjectStorage)
if err != nil {

63
internal/services/configstore/configstore_test.go
View File

@ -29,15 +29,17 @@ import (
"agola.io/agola/internal/db"
"agola.io/agola/internal/services/config"
action "agola.io/agola/internal/services/configstore/action"
"agola.io/agola/internal/services/configstore/action"
"agola.io/agola/internal/testutil"
"agola.io/agola/internal/util"
"agola.io/agola/services/configstore/types"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd {
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
@ -57,12 +59,12 @@ func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) {
}
}
func setupConfigstore(ctx context.Context, t *testing.T, dir string) (*Configstore, *testutil.TestEmbeddedEtcd) {
func setupConfigstore(ctx context.Context, t *testing.T, logger *zap.Logger, dir string) (*Configstore, *testutil.TestEmbeddedEtcd) {
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
listenAddress, port, err := testutil.GetFreePort(true, false)
if err != nil {
@ -92,7 +94,7 @@ func setupConfigstore(ctx context.Context, t *testing.T, dir string) (*Configsto
csConfig.DataDir = csDir
csConfig.Web.ListenAddress = net.JoinHostPort(listenAddress, port)
cs, err := NewConfigstore(ctx, &csConfig)
cs, err := NewConfigstore(ctx, logger, &csConfig)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -127,11 +129,13 @@ func TestResync(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
listenAddress1, port1, err := testutil.GetFreePort(true, false)
@ -184,11 +188,11 @@ func TestResync(t *testing.T) {
cs2Config.DataDir = csDir2
cs2Config.Web.ListenAddress = net.JoinHostPort(listenAddress2, port2)
cs1, err := NewConfigstore(ctx, &cs1Config)
cs1, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs1")), &cs1Config)
if err != nil {
t.Fatalf("err: %v", err)
}
cs2, err := NewConfigstore(ctx, &cs2Config)
cs2, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -233,7 +237,7 @@ func TestResync(t *testing.T) {
// start cs2
// it should resync from wals since the etcd revision as been compacted
cs2, err = NewConfigstore(ctx, &cs2Config)
cs2, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -271,7 +275,7 @@ func TestResync(t *testing.T) {
cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3)
log.Infof("starting cs3")
cs3, err := NewConfigstore(ctx, &cs3Config)
cs3, err := NewConfigstore(ctx, logger, &cs3Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -301,11 +305,13 @@ func TestExportImport(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
etcdDir, err := ioutil.TempDir(dir, "etcd")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
defer shutdownEtcd(tetcd)
listenAddress1, port1, err := testutil.GetFreePort(true, false)
@ -362,15 +368,15 @@ func TestExportImport(t *testing.T) {
cs3Config.DataDir = csDir3
cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3)
cs1, err := NewConfigstore(ctx, &cs1Config)
cs1, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs1")), &cs1Config)
if err != nil {
t.Fatalf("err: %v", err)
}
cs2, err := NewConfigstore(ctx, &cs2Config)
cs2, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
cs3, err := NewConfigstore(ctx, &cs3Config)
cs3, err := NewConfigstore(ctx, logger.With(zap.String("name", "cs3")), &cs3Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -459,7 +465,7 @@ func TestExportImport(t *testing.T) {
// start cs2
// it should do a full resync since we have imported new data and there's now wal in etcd
cs2, err = NewConfigstore(ctx, &cs2Config)
cs2, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs2")), &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -503,7 +509,7 @@ func TestExportImport(t *testing.T) {
// start cs3
// it should do a full resync since we have imported new data and there're some wals with a different epoch
cs3, err = NewConfigstore(ctx, &cs3Config)
cs3, err = NewConfigstore(ctx, logger.With(zap.String("name", "cs3")), &cs3Config)
if err != nil {
t.Fatalf("err: %v", err)
}
@ -549,8 +555,9 @@ func TestUser(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -616,8 +623,9 @@ func TestProjectGroupsAndProjectsCreate(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -763,8 +771,9 @@ func TestProjectUpdate(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -839,8 +848,9 @@ func TestProjectGroupUpdate(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -989,8 +999,9 @@ func TestProjectGroupDelete(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -1045,8 +1056,9 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -1180,8 +1192,9 @@ func TestOrgMembers(t *testing.T) {
defer os.RemoveAll(dir)
ctx := context.Background()
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
@ -1269,6 +1282,8 @@ func TestRemoteSource(t *testing.T) {
}
defer os.RemoveAll(dir)
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
tests := []struct {
name string
f func(ctx context.Context, t *testing.T, cs *Configstore)
@ -1417,7 +1432,7 @@ func TestRemoteSource(t *testing.T) {
}
ctx := context.Background()
cs, tetcd := setupConfigstore(ctx, t, dir)
cs, tetcd := setupConfigstore(ctx, t, logger, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")

8
internal/services/executor/driver/docker.go
View File

@ -15,13 +15,13 @@
package driver
import (
"bufio"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
@ -76,7 +76,11 @@ func (d *DockerDriver) createToolboxVolume(ctx context.Context, podID string) (*
if err != nil {
return nil, err
}
if _, err := io.Copy(os.Stdout, reader); err != nil {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
d.log.Infof("create toolbox volume image pull output: %s", scanner.Text())
}
if err := scanner.Err(); err != nil {
return nil, err
}

12
internal/services/executor/driver/docker_test.go
View File

@ -22,19 +22,15 @@ import (
"testing"
"time"
slog "agola.io/agola/internal/log"
"agola.io/agola/internal/testutil"
"github.com/docker/docker/api/types"
uuid "github.com/satori/go.uuid"
"github.com/google/go-cmp/cmp"
uuid "github.com/satori/go.uuid"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest"
)
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
func TestDockerPod(t *testing.T) {
if os.Getenv("SKIP_DOCKER_TESTS") == "1" {
t.Skip("skipping since env var SKIP_DOCKER_TESTS is 1")
@ -44,6 +40,8 @@ func TestDockerPod(t *testing.T) {
t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined")
}
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
d, err := NewDockerDriver(logger, "executorid01", toolboxPath)
if err != nil {
t.Fatalf("unexpected err: %v", err)

4
internal/services/executor/driver/k8s_test.go
View File

@ -26,6 +26,8 @@ import (
"agola.io/agola/internal/testutil"
uuid "github.com/satori/go.uuid"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
func TestK8sPod(t *testing.T) {
@ -37,6 +39,8 @@ func TestK8sPod(t *testing.T) {
t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined")
}
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
d, err := NewK8sDriver(logger, "executorid01", toolboxPath)
if err != nil {
t.Fatalf("unexpected err: %v", err)

6
internal/services/executor/executor.go
View File

@ -1339,10 +1339,14 @@ type Executor struct {
dynamic bool
}
func NewExecutor(c *config.Executor) (*Executor, error) {
func NewExecutor(ctx context.Context, l *zap.Logger, c *config.Executor) (*Executor, error) {
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
var err error
c.ToolboxPath, err = filepath.Abs(c.ToolboxPath)

7
internal/services/gateway/gateway.go
View File

@ -58,11 +58,16 @@ type Gateway struct {
sd *common.TokenSigningData
}
func NewGateway(gc *config.Config) (*Gateway, error) {
func NewGateway(ctx context.Context, l *zap.Logger, gc *config.Config) (*Gateway, error) {
c := &gc.Gateway
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
if c.Web.ListenAddress == "" {
return nil, errors.Errorf("listen address undefined")

6
internal/services/gitserver/main.go
View File

@ -130,10 +130,14 @@ type Gitserver struct {
c *config.Gitserver
}
func NewGitserver(c *config.Gitserver) (*Gitserver, error) {
func NewGitserver(ctx context.Context, l *zap.Logger, c *config.Gitserver) (*Gitserver, error) {
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
return &Gitserver{
c: c,

7
internal/services/notification/notification.go
View File

@ -42,11 +42,16 @@ type NotificationService struct {
configstoreClient *csclient.Client
}
func NewNotificationService(gc *config.Config) (*NotificationService, error) {
func NewNotificationService(ctx context.Context, l *zap.Logger, gc *config.Config) (*NotificationService, error) {
c := &gc.Notification
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
e, err := common.NewEtcd(&c.Etcd, logger, "notification")
if err != nil {

12
internal/services/runservice/runservice.go
View File

@ -25,6 +25,7 @@ import (
scommon "agola.io/agola/internal/common"
"agola.io/agola/internal/datamanager"
"agola.io/agola/internal/etcd"
slog "agola.io/agola/internal/log"
"agola.io/agola/internal/objectstorage"
"agola.io/agola/internal/services/config"
"agola.io/agola/internal/services/runservice/action"
@ -37,9 +38,14 @@ import (
"github.com/gorilla/mux"
etcdclientv3 "go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/mvcc/mvccpb"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
var log = logger.Sugar()
// etcdPingerLoop periodically updates a key.
// This is used by watchers to inform the client of the current revision
// this is needed since if other users are updating other unwatched keys on
@ -141,10 +147,14 @@ type Runservice struct {
maintenanceMode bool
}
func NewRunservice(ctx context.Context, c *config.Runservice) (*Runservice, error) {
func NewRunservice(ctx context.Context, l *zap.Logger, c *config.Runservice) (*Runservice, error) {
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
ost, err := scommon.NewObjectStorage(&c.ObjectStorage)
if err != nil {

7
internal/services/runservice/scheduler.go
View File

@ -25,7 +25,6 @@ import (
"agola.io/agola/internal/datamanager"
"agola.io/agola/internal/etcd"
slog "agola.io/agola/internal/log"
"agola.io/agola/internal/objectstorage"
"agola.io/agola/internal/runconfig"
"agola.io/agola/internal/services/runservice/common"
@ -35,8 +34,6 @@ import (
etcdclientv3 "go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/concurrency"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
errors "golang.org/x/xerrors"
)
@ -47,10 +44,6 @@ const (
defaultExecutorNotAliveInterval = 60 * time.Second
)
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
var log = logger.Sugar()
func (s *Runservice) runActiveExecutorTasks(ctx context.Context, runID string) ([]*types.ExecutorTask, error) {
// the real source of active tasks is the number of executor tasks in etcd
// we can't rely on RunTask.Status since it's only updated when receiveing

6
internal/services/scheduler/scheduler.go
View File

@ -197,10 +197,14 @@ type Scheduler struct {
runserviceClient *rsclient.Client
}
func NewScheduler(c *config.Scheduler) (*Scheduler, error) {
func NewScheduler(ctx context.Context, l *zap.Logger, c *config.Scheduler) (*Scheduler, error) {
if l != nil {
logger = l
}
if c.Debug {
level.SetLevel(zapcore.DebugLevel)
}
log = logger.Sugar()
return &Scheduler{
c: c,

3
internal/testutil/utils.go
View File

@ -167,6 +167,7 @@ func NewTestEmbeddedEtcd(t *testing.T, logger *zap.Logger, dir string, a ...stri
cfg.Name = uid
cfg.Dir = dataDir
cfg.Logger = "zap"
cfg.LogLevel = "fatal"
cfg.LogOutputs = []string{"stdout"}
lcurl, _ := url.Parse(fmt.Sprintf("http://%s:%s", listenAddress, port))
lpurl, _ := url.Parse(fmt.Sprintf("http://%s:%s", listenAddress2, port2))
@ -451,7 +452,7 @@ type TestGitea struct {
SSHPort string
}
func NewTestGitea(t *testing.T, logger *zap.Logger, dir, dockerBridgeAddress string, a ...string) (*TestGitea, error) {
func NewTestGitea(t *testing.T, dir, dockerBridgeAddress string, a ...string) (*TestGitea, error) {
u := uuid.NewV4()
uid := fmt.Sprintf("%x", u[:4])

32
tests/setup_test.go
View File

@ -27,7 +27,6 @@ import (
"testing"
"time"
slog "agola.io/agola/internal/log"
"agola.io/agola/internal/services/config"
"agola.io/agola/internal/services/configstore"
"agola.io/agola/internal/services/executor"
@ -44,7 +43,7 @@ import (
"code.gitea.io/sdk/gitea"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest"
errors "golang.org/x/xerrors"
"gopkg.in/src-d/go-billy.v4/memfs"
"gopkg.in/src-d/go-billy.v4/osfs"
@ -57,15 +56,12 @@ import (
"gopkg.in/src-d/go-git.v4/storage/memory"
)
var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
var logger = slog.New(level)
const (
giteaUser01 = "user01"
agolaUser01 = "user01"
)
func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd {
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
@ -86,7 +82,7 @@ func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) {
}
func setupGitea(t *testing.T, dir, dockerBridgeAddress string) *testutil.TestGitea {
tgitea, err := testutil.NewTestGitea(t, logger, dir, dockerBridgeAddress)
tgitea, err := testutil.NewTestGitea(t, dir, dockerBridgeAddress)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
@ -128,38 +124,38 @@ func shutdownGitea(tgitea *testutil.TestGitea) {
tgitea.Kill()
}
func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config) (<-chan error, error) {
rs, err := rsscheduler.NewRunservice(ctx, &c.Runservice)
func startAgola(ctx context.Context, t *testing.T, logger *zap.Logger, dir string, c *config.Config) (<-chan error, error) {
rs, err := rsscheduler.NewRunservice(ctx, logger, &c.Runservice)
if err != nil {
return nil, errors.Errorf("failed to start run service scheduler: %w", err)
}
ex, err := executor.NewExecutor(&c.Executor)
ex, err := executor.NewExecutor(ctx, logger, &c.Executor)
if err != nil {
return nil, errors.Errorf("failed to start run service executor: %w", err)
}
cs, err := configstore.NewConfigstore(ctx, &c.Configstore)
cs, err := configstore.NewConfigstore(ctx, logger, &c.Configstore)
if err != nil {
return nil, errors.Errorf("failed to start config store: %w", err)
}
sched, err := scheduler.NewScheduler(&c.Scheduler)
sched, err := scheduler.NewScheduler(ctx, logger, &c.Scheduler)
if err != nil {
return nil, errors.Errorf("failed to start scheduler: %w", err)
}
ns, err := notification.NewNotificationService(c)
ns, err := notification.NewNotificationService(ctx, logger, c)
if err != nil {
return nil, errors.Errorf("failed to start notification service: %w", err)
}
gw, err := gateway.NewGateway(c)
gw, err := gateway.NewGateway(ctx, logger, c)
if err != nil {
return nil, errors.Errorf("failed to start gateway: %w", err)
}
gs, err := gitserver.NewGitserver(&c.Gitserver)
gs, err := gitserver.NewGitserver(ctx, logger, &c.Gitserver)
if err != nil {
return nil, errors.Errorf("failed to start git server: %w", err)
}
@ -181,6 +177,8 @@ func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config)
}
func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbeddedEtcd, *testutil.TestGitea, *config.Config) {
logger := zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
dockerBridgeAddress := os.Getenv("DOCKER_BRIDGE_ADDRESS")
if dockerBridgeAddress == "" {
dockerBridgeAddress = "172.17.0.1"
@ -285,7 +283,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
tgitea := setupGitea(t, dir, dockerBridgeAddress)
etcdDir := filepath.Join(dir, "etcd")
tetcd := setupEtcd(t, etcdDir)
tetcd := setupEtcd(t, logger, etcdDir)
c.Runservice.Etcd.Endpoints = tetcd.Endpoint
c.Configstore.Etcd.Endpoints = tetcd.Endpoint
@ -336,7 +334,7 @@ func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbedde
c.Executor.RunserviceURL = rsURL
errCh, err := startAgola(ctx, t, dir, c)
errCh, err := startAgola(ctx, t, logger, dir, c)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}