agola/internal/services/configstore/configstore_test.go

508 lines
15 KiB
Go

// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package configstore
import (
"context"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"reflect"
"sync"
"testing"
"time"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/config"
"github.com/sorintlab/agola/internal/services/configstore/command"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/testutil"
"github.com/sorintlab/agola/internal/util"
)
func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tetcd.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tetcd.WaitUp(30 * time.Second); err != nil {
t.Fatalf("error waiting on etcd up: %v", err)
}
return tetcd
}
func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) {
if tetcd.Etcd != nil {
tetcd.Kill()
}
}
func setupConfigstore(t *testing.T, ctx context.Context, dir string) (*ConfigStore, *testutil.TestEmbeddedEtcd) {
etcdDir, err := ioutil.TempDir(dir, "etcd")
tetcd := setupEtcd(t, etcdDir)
listenAddress, port, err := testutil.GetFreePort(true, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ltsDir, err := ioutil.TempDir(dir, "lts")
csDir, err := ioutil.TempDir(dir, "cs")
baseConfig := config.ConfigStore{
Etcd: config.Etcd{
Endpoints: tetcd.Endpoint,
},
LTS: config.LTS{
Type: config.LTSTypePosix,
Path: ltsDir,
},
Web: config.Web{},
}
csConfig := baseConfig
csConfig.DataDir = csDir
csConfig.Web.ListenAddress = net.JoinHostPort(listenAddress, port)
cs, err := NewConfigStore(ctx, &csConfig)
if err != nil {
t.Fatalf("err: %v", err)
}
return cs, tetcd
}
func getProjects(cs *ConfigStore) ([]*types.Project, error) {
var projects []*types.Project
err := cs.readDB.Do(func(tx *db.Tx) error {
var err error
projects, err = cs.readDB.GetAllProjects(tx)
return err
})
return projects, err
}
func getUsers(cs *ConfigStore) ([]*types.User, error) {
var users []*types.User
err := cs.readDB.Do(func(tx *db.Tx) error {
var err error
users, err = cs.readDB.GetUsers(tx, "", 0, true)
return err
})
return users, err
}
func TestResync(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
etcdDir, err := ioutil.TempDir(dir, "etcd")
tetcd := setupEtcd(t, etcdDir)
defer shutdownEtcd(tetcd)
listenAddress1, port1, err := testutil.GetFreePort(true, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
listenAddress2, port2, err := testutil.GetFreePort(true, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
listenAddress3, port3, err := testutil.GetFreePort(true, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ctx := context.Background()
ltsDir, err := ioutil.TempDir(dir, "lts")
csDir1, err := ioutil.TempDir(dir, "cs1")
csDir2, err := ioutil.TempDir(dir, "cs2")
csDir3, err := ioutil.TempDir(dir, "cs3")
baseConfig := config.ConfigStore{
Etcd: config.Etcd{
Endpoints: tetcd.Endpoint,
},
LTS: config.LTS{
Type: config.LTSTypePosix,
Path: ltsDir,
},
Web: config.Web{},
}
cs1Config := baseConfig
cs1Config.DataDir = csDir1
cs1Config.Web.ListenAddress = net.JoinHostPort(listenAddress1, port1)
cs2Config := baseConfig
cs2Config.DataDir = csDir2
cs2Config.Web.ListenAddress = net.JoinHostPort(listenAddress2, port2)
cs1, err := NewConfigStore(ctx, &cs1Config)
if err != nil {
t.Fatalf("err: %v", err)
}
cs2, err := NewConfigStore(ctx, &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
ctx1 := context.Background()
ctx2, cancel2 := context.WithCancel(context.Background())
t.Logf("starting cs1")
go func() {
if err := cs1.Run(ctx1); err != nil {
t.Fatalf("err: %v", err)
}
}()
t.Logf("starting cs2")
go func() {
if err := cs2.Run(ctx2); err != nil {
t.Fatalf("err: %v", err)
}
}()
time.Sleep(1 * time.Second)
for i := 0; i < 10; i++ {
if _, err := cs1.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: fmt.Sprintf("user%d", i)}); err != nil {
t.Fatalf("err: %v", err)
}
time.Sleep(200 * time.Millisecond)
}
time.Sleep(5 * time.Second)
// stop cs2
log.Infof("stopping cs2")
cancel2()
// Do some more changes
for i := 11; i < 20; i++ {
if _, err := cs1.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: fmt.Sprintf("user%d", i)}); err != nil {
t.Fatalf("err: %v", err)
}
time.Sleep(200 * time.Millisecond)
}
time.Sleep(5 * time.Second)
// compact etcd
if err := tetcd.Compact(); err != nil {
t.Fatalf("err: %v", err)
}
// start cs2
// it should resync from wals since the etcd revision as been compacted
cs2, err = NewConfigStore(ctx, &cs2Config)
if err != nil {
t.Fatalf("err: %v", err)
}
log.Infof("starting cs2")
ctx2 = context.Background()
go cs2.Run(ctx2)
time.Sleep(5 * time.Second)
users1, err := getUsers(cs1)
if err != nil {
t.Fatalf("err: %v", err)
}
users2, err := getUsers(cs2)
if err != nil {
t.Fatalf("err: %v", err)
}
if !compareUsers(users1, users2) {
t.Logf("len(users1): %d", len(users1))
t.Logf("len(users2): %d", len(users2))
t.Logf("users1: %s", util.Dump(users1))
t.Logf("users2: %s", util.Dump(users2))
t.Fatalf("users are different between the two readdbs")
}
// start cs3, since it's a new instance it should do a full resync
cs3Config := baseConfig
cs3Config.DataDir = csDir3
cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3)
log.Infof("starting cs3")
cs3, err := NewConfigStore(ctx, &cs3Config)
if err != nil {
t.Fatalf("err: %v", err)
}
ctx3 := context.Background()
go cs3.Run(ctx3)
time.Sleep(5 * time.Second)
users1, err = getUsers(cs1)
if err != nil {
t.Fatalf("err: %v", err)
}
users3, err := getUsers(cs3)
if err != nil {
t.Fatalf("err: %v", err)
}
if !compareUsers(users1, users3) {
t.Logf("len(users1): %d", len(users1))
t.Logf("len(users3): %d", len(users3))
t.Logf("users1: %s", util.Dump(users1))
t.Logf("users3: %s", util.Dump(users3))
t.Fatalf("users are different between the two readdbs")
}
}
func compareUsers(u1, u2 []*types.User) bool {
u1ids := map[string]struct{}{}
u2ids := map[string]struct{}{}
for _, u := range u1 {
u1ids[u.ID] = struct{}{}
}
for _, u := range u2 {
u2ids[u.ID] = struct{}{}
}
return reflect.DeepEqual(u1ids, u2ids)
}
func TestUser(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
ctx := context.Background()
cs, tetcd := setupConfigstore(t, ctx, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
go func() {
if err := cs.Run(ctx); err != nil {
t.Fatalf("err: %v", err)
}
}()
// TODO(sgotti) change the sleep with a real check that all is ready
time.Sleep(2 * time.Second)
t.Run("create user", func(t *testing.T) {
_, err := cs.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: "user01"})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
// TODO(sgotti) change the sleep with a real check that user is in readdb
time.Sleep(2 * time.Second)
t.Run("create duplicated user", func(t *testing.T) {
expectedErr := fmt.Sprintf("user with name %q already exists", "user01")
_, err := cs.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: "user01"})
if err == nil {
t.Fatalf("expected error %v, got nil err", expectedErr)
}
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("concurrent user with same name creation", func(t *testing.T) {
prevUsers, err := getUsers(cs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
wg := sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go cs.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: "user02"})
wg.Done()
}
wg.Wait()
time.Sleep(5 * time.Second)
users, err := getUsers(cs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if len(users) != len(prevUsers)+1 {
t.Fatalf("expected %d users, got %d", len(prevUsers)+1, len(users))
}
})
}
func TestProjectGroupsAndProjects(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
ctx := context.Background()
cs, tetcd := setupConfigstore(t, ctx, dir)
defer shutdownEtcd(tetcd)
t.Logf("starting cs")
go func() {
if err := cs.Run(ctx); err != nil {
t.Fatalf("err: %v", err)
}
}()
// TODO(sgotti) change the sleep with a real check that all is ready
time.Sleep(2 * time.Second)
user, err := cs.ch.CreateUser(ctx, &command.CreateUserRequest{UserName: "user01"})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
org, err := cs.ch.CreateOrg(ctx, &types.Organization{Name: "org01"})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// TODO(sgotti) change the sleep with a real check that user is in readdb
time.Sleep(2 * time.Second)
t.Run("create a project in user root project group", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a project in org root project group", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a projectgroup in user root project group", func(t *testing.T) {
_, err := cs.ch.CreateProjectGroup(ctx, &types.ProjectGroup{Name: "projectgroup01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a projectgroup in org root project group", func(t *testing.T) {
_, err := cs.ch.CreateProjectGroup(ctx, &types.ProjectGroup{Name: "projectgroup01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a project in user non root project group with same name as a root project", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName, "projectgroup01")}})
if err != nil {
t.Fatalf("unexpected err: %+#v", err)
}
})
t.Run("create a project in org non root project group with same name as a root project", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name, "projectgroup01")}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create duplicated project in user root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("project with name %q, path %q already exists", projectName, path.Join("user", user.UserName, projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create duplicated project in org root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("project with name %q, path %q already exists", projectName, path.Join("org", org.Name, projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create duplicated project in user non root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("project with name %q, path %q already exists", projectName, path.Join("user", user.UserName, "projectgroup01", projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName, "projectgroup01")}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create duplicated project in org non root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("project with name %q, path %q already exists", projectName, path.Join("org", org.Name, "projectgroup01", projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name, "projectgroup01")}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project in unexistent project group", func(t *testing.T) {
expectedErr := `project group with id "unexistentid" doesn't exist`
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: "unexistentid"}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project without parent id specified", func(t *testing.T) {
expectedErr := "project parent id required"
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01"})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("concurrent project with same name creation", func(t *testing.T) {
prevProjects, err := getProjects(cs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
wg := sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go cs.ch.CreateProject(ctx, &types.Project{Name: "project02", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
wg.Done()
}
wg.Wait()
time.Sleep(1 * time.Second)
projects, err := getProjects(cs)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if len(projects) != len(prevProjects)+1 {
t.Fatalf("expected %d projects, got %d", len(prevProjects)+1, len(projects))
}
})
}