From cf0b7f8ded14ce35407d864eb6c66312dfb47277 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Wed, 29 May 2019 13:35:20 +0200 Subject: [PATCH] add initial integration tests --- .agola/config.jsonnet | 32 ++- go.mod | 2 + go.sum | 14 + internal/testutil/utils.go | 183 ++++++++++++ tests/setup_test.go | 554 +++++++++++++++++++++++++++++++++++++ 5 files changed, 778 insertions(+), 7 deletions(-) create mode 100644 tests/setup_test.go diff --git a/.agola/config.jsonnet b/.agola/config.jsonnet index b4fd149..689e644 100644 --- a/.agola/config.jsonnet +++ b/.agola/config.jsonnet @@ -25,12 +25,8 @@ local task_build_go(version, arch) = { runtime: go_runtime(version, arch), environment: { GO111MODULE: 'on', - VAR01: { - from_variable: 'var01', - }, }, steps: [ - { type: 'run', command: 'env' }, { type: 'clone' }, { type: 'restore_cache', keys: ['cache-sum-{{ md5sum "go.sum" }}', 'cache-date-'], dest_dir: '/go/pkg/mod/cache' }, { type: 'run', command: 'make' }, @@ -39,7 +35,9 @@ local task_build_go(version, arch) = { { type: 'run', name: 'install golangci-lint', command: 'curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1' }, { type: 'run', command: 'golangci-lint run --deadline 5m' }, { type: 'run', name: 'build docker/k8s drivers tests binary', command: 'CGO_ENABLED=0 go test -c ./internal/services/executor/driver -o ./bin/docker-tests' }, - { type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -v -count 1 ./...' }, + { type: 'run', name: 'build integration tests binary', command: 'go test -tags "sqlite_unlock_notify" -c ./tests -o ./bin/integration-tests' }, + { type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -v -count 1 ./cmd/... ./internal/...' }, + { type: 'run', name: 'fetch gitea binary for integration tests', command: 'curl -L https://github.com/go-gitea/gitea/releases/download/v1.8.3/gitea-1.8.3-linux-amd64 -o ./bin/gitea && chmod +x ./bin/gitea' }, { type: 'save_to_workspace', contents: [{ source_dir: './bin', dest_dir: '/bin/', paths: ['*'] }] }, ], }; @@ -51,7 +49,6 @@ local task_build_docker_tests(version, arch) = { GO111MODULE: 'on', }, steps: [ - { type: 'run', command: 'env' }, { type: 'clone' }, { type: 'restore_cache', keys: ['cache-sum-{{ md5sum "go.sum" }}', 'cache-date-'], dest_dir: '/go/pkg/mod/cache' }, ], @@ -72,7 +69,6 @@ local task_build_docker_tests(version, arch) = { name: 'test docker driver', runtime: dind_runtime('amd64'), steps: [ - { type: 'run', command: 'env' }, { type: 'restore_workspace', dest_dir: '.' }, { type: 'run', command: 'SKIP_K8S_TESTS=1 AGOLA_TOOLBOX_PATH="./bin" ./bin/docker-tests -test.parallel 1 -test.v' }, ], @@ -80,6 +76,28 @@ local task_build_docker_tests(version, arch) = { 'build go 1.12 amd64', ], }, + { + name: 'integration tests', + runtime: dind_runtime('amd64'), + steps: [ + { type: 'run', command: 'apk update && apk add bash git openssh-keygen' }, + { + type: 'run', + name: 'install alpine glibc package', + command: ||| + apk --no-cache add ca-certificates wget + wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.29-r0/glibc-2.29-r0.apk + apk add glibc-2.29-r0.apk + |||, + }, + { type: 'restore_workspace', dest_dir: '.' }, + { type: 'run', name: 'integration tests', command: 'AGOLA_TOOLBOX_PATH="./bin" GITEA_PATH=${PWD}/bin/gitea ./bin/integration-tests -test.parallel 1 -test.v' }, + ], + depends: [ + 'build go 1.12 amd64', + ], + }, ], }, ], diff --git a/go.mod b/go.mod index 658ac2a..34c0a6a 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,8 @@ require ( golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.42.0 // indirect + gopkg.in/src-d/go-billy.v4 v4.3.0 + gopkg.in/src-d/go-git.v4 v4.10.0 gopkg.in/yaml.v2 v2.2.2 gotest.tools v2.2.0+incompatible // indirect k8s.io/api v0.0.0-20190313235455-40a48860b5ab diff --git a/go.sum b/go.sum index e959d65..760b54b 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,10 @@ github.com/Unknwon/cae v0.0.0-20160715032808-c6aac99ea2ca/go.mod h1:IRSre9/SEhVu github.com/Unknwon/com v0.0.0-20170819223952-7677a1d7c113/go.mod h1:KYCjqMOeHpNuTOiFQU6WEcTG7poCJrUs0YgyHNtn1no= github.com/Unknwon/i18n v0.0.0-20171114194641-b64d33658966/go.mod h1:SFtfq0zFPsENI7DpE87QM2hcYu5QQ0fRdCgP+P1Hrqo= github.com/Unknwon/paginater v0.0.0-20151104151617-7748a72e0141/go.mod h1:fw0McLecf/G5NFwddCRmDckU6yovtk1YsgWIoepMbYo= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/andybalholm/cascadia v0.0.0-20161224141413-349dd0209470/go.mod h1:3I+3V7B6gTBYfdpYgIG2ymALS9H+5VDKUl3lHH7ToM4= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -79,6 +81,7 @@ github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:/Zj4wYkg github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f h1:AUj1VoZUfhPhOPHULCQQDnGhRelpFWHMLhQVWDsS0v4= github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a/go.mod h1:MkKY/CB98aVE4VxO63X5vTQKUgcn+3XP15LMASe3lYs= @@ -91,11 +94,13 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= @@ -193,6 +198,7 @@ github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80s github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= @@ -206,6 +212,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -280,6 +287,7 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -330,6 +338,7 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -349,6 +358,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xanzy/go-gitlab v0.14.1 h1:+CipI8+oQxqWNmKCU/9GQvlQnJ5v366ChHNEI4O83is= github.com/xanzy/go-gitlab v0.14.1/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -436,13 +446,17 @@ gopkg.in/ldap.v3 v3.0.2/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw= gopkg.in/macaron.v1 v1.3.2/go.mod h1:PrsiawTWAGZs6wFbT5hlr7SQ2Ns9h7cUVtcUu4lQOVo= gopkg.in/redis.v2 v2.3.2/go.mod h1:4wl9PJ/CqzeHk3LVq1hNLHH8krm3+AXEgut4jVc++LU= gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs= gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.10.0 h1:NWjTJTQnk8UpIGlssuefyDZ6JruEjo5s88vm88uASbw= gopkg.in/src-d/go-git.v4 v4.10.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU= gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/internal/testutil/utils.go b/internal/testutil/utils.go index e4ac51d..a0323f9 100644 --- a/internal/testutil/utils.go +++ b/internal/testutil/utils.go @@ -16,16 +16,20 @@ package testutil import ( "bufio" + "bytes" "context" "fmt" + "io/ioutil" "net" "net/url" "os" "os/exec" + "os/user" "path/filepath" "strconv" "sync" "testing" + "text/template" "time" "agola.io/agola/internal/etcd" @@ -339,6 +343,185 @@ func (te *TestEtcd) WaitDown(timeout time.Duration) error { return fmt.Errorf("timeout") } +const ( + giteaAppIniTmpl = ` +APP_NAME = Gitea: Git with a cup of tea +RUN_MODE = prod +RUN_USER = {{ .User }} + +[repository] +ROOT = {{ .Data }}/git/repositories + +[repository.local] +LOCAL_COPY_PATH = {{ .Data }}/gitea/tmp/local-repo + +[repository.upload] +TEMP_PATH = {{ .Data }}/gitea/uploads + +[server] +APP_DATA_PATH = {{ .Data }}/gitea +SSH_DOMAIN = {{ .ListenAddress }} +HTTP_PORT = {{ .HTTPPort }} +ROOT_URL = http://{{ .ListenAddress }}:{{ .HTTPPort }}/ +DISABLE_SSH = false +# Use built-in ssh server +START_SSH_SERVER = true +SSH_PORT = {{ .SSHPort }} +LFS_CONTENT_PATH = {{ .Data }}/git/lfs +DOMAIN = localhost +LFS_START_SERVER = true +LFS_JWT_SECRET = PI0Tfn0OcYpzpNb_u11JdoUfDbsMa2x6paWH2ckMVrw +OFFLINE_MODE = false + +[database] +PATH = {{ .Data }}/gitea/gitea.db +DB_TYPE = sqlite3 +HOST = +NAME = +USER = +PASSWD = +SSL_MODE = disable + +[indexer] +ISSUE_INDEXER_PATH = {{ .Data }}/gitea/indexers/issues.bleve + +[session] +PROVIDER_CONFIG = {{ .Data }}/gitea/sessions +PROVIDER = file + +[picture] +AVATAR_UPLOAD_PATH = {{ .Data }}/gitea/avatars +DISABLE_GRAVATAR = false +ENABLE_FEDERATED_AVATAR = true + +[attachment] +PATH = {{ .Data }}/gitea/attachments + +[log] +ROOT_PATH = {{ .Data }}/gitea/log +MODE = file +LEVEL = info + +[security] +INSTALL_LOCK = true +SECRET_KEY = vRCH8usxWj6e8JGBPBaqycpfVyWm079xC3P3k76YsjKbrgBmyHhQD9UyzRFICKBT +INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE1NTc0MDI0MDZ9.27f4bakIxBIOoO48ORyLmbvpQprsJMEHLM6PyXIqB5g + +[service] +DISABLE_REGISTRATION = false +REQUIRE_SIGNIN_VIEW = false +REGISTER_EMAIL_CONFIRM = false +ENABLE_NOTIFY_MAIL = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = false +ENABLE_CAPTCHA = false +DEFAULT_KEEP_EMAIL_PRIVATE = false +DEFAULT_ALLOW_CREATE_ORGANIZATION = true +DEFAULT_ENABLE_TIMETRACKING = true +NO_REPLY_ADDRESS = noreply.example.org + +[oauth2] +JWT_SECRET = hQdtj6H6lsd8vG6V1vCPYcOn8uP2C3i_bbnDozfCcIY + +[mailer] +ENABLED = false + +[openid] +ENABLE_OPENID_SIGNIN = true +ENABLE_OPENID_SIGNUP = true + ` +) + +type GiteaConfig struct { + Data string + User string + ListenAddress string + HTTPPort string + SSHPort string +} + +type TestGitea struct { + Process + + GiteaPath string + ConfigPath string + ListenAddress string + HTTPPort string + SSHPort string +} + +func NewTestGitea(t *testing.T, logger *zap.Logger, dir string, a ...string) (*TestGitea, error) { + u := uuid.NewV4() + uid := fmt.Sprintf("%x", u[:4]) + + giteaPath := os.Getenv("GITEA_PATH") + if giteaPath == "" { + t.Fatalf("env var GITEA_PATH is undefined") + } + + curUser, err := user.Current() + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + giteaDir := filepath.Join(dir, "gitea") + + _, httpPort, err := GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + listenAddress, sshPort, err := GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + giteaConfig := &GiteaConfig{ + Data: giteaDir, + User: curUser.Username, + ListenAddress: listenAddress, + HTTPPort: httpPort, + SSHPort: sshPort, + } + tmpl, err := template.New("gitea").Parse(giteaAppIniTmpl) + if err != nil { + return nil, err + } + conf := &bytes.Buffer{} + if err := tmpl.Execute(conf, giteaConfig); err != nil { + return nil, err + } + + if err := os.MkdirAll(filepath.Join(dir, "gitea", "conf"), 0775); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(dir, "gitea", "log"), 0775); err != nil { + return nil, err + } + configPath := filepath.Join(dir, "gitea", "conf", "app.ini") + if err := ioutil.WriteFile(configPath, conf.Bytes(), 0664); err != nil { + return nil, err + } + + args := []string{} + args = append(args, "web", "--config", configPath) + + tgitea := &TestGitea{ + Process: Process{ + t: t, + uid: uid, + name: "gitea", + bin: giteaPath, + args: args, + }, + GiteaPath: giteaPath, + ConfigPath: configPath, + ListenAddress: listenAddress, + HTTPPort: httpPort, + SSHPort: sshPort, + } + + return tgitea, nil +} + func testFreeTCPPort(port int) error { ln, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) if err != nil { diff --git a/tests/setup_test.go b/tests/setup_test.go new file mode 100644 index 0000000..307e25d --- /dev/null +++ b/tests/setup_test.go @@ -0,0 +1,554 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "testing" + "time" + + slog "agola.io/agola/internal/log" + "agola.io/agola/internal/services/config" + "agola.io/agola/internal/services/configstore" + "agola.io/agola/internal/services/executor" + "agola.io/agola/internal/services/gateway" + gwapi "agola.io/agola/internal/services/gateway/api" + "agola.io/agola/internal/services/gitserver" + "agola.io/agola/internal/services/notification" + rsscheduler "agola.io/agola/internal/services/runservice" + rstypes "agola.io/agola/internal/services/runservice/types" + "agola.io/agola/internal/services/scheduler" + "agola.io/agola/internal/services/types" + "agola.io/agola/internal/testutil" + "agola.io/agola/internal/util" + + gtypes "code.gitea.io/gitea/modules/structs" + "code.gitea.io/sdk/gitea" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + errors "golang.org/x/xerrors" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-git.v4" + gitconfig "gopkg.in/src-d/go-git.v4/config" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + "gopkg.in/src-d/go-git.v4/storage/memory" +) + +var level = zap.NewAtomicLevelAt(zapcore.InfoLevel) +var logger = slog.New(level) + +const ( + giteaUser01 = "user01" + agolaUser01 = "user01" +) + +func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd { + tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if err := tetcd.Start(); err != nil { + t.Fatalf("unexpected err: %v", err) + } + if err := tetcd.WaitUp(30 * time.Second); err != nil { + t.Fatalf("error waiting on etcd up: %v", err) + } + return tetcd +} + +func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) { + if tetcd.Etcd != nil { + _ = tetcd.Kill() + } +} + +func setupGitea(t *testing.T, dir string) *testutil.TestGitea { + tgitea, err := testutil.NewTestGitea(t, logger, dir) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if err := tgitea.Start(); err != nil { + t.Fatalf("unexpected err: %v", err) + } + time.Sleep(5 * time.Second) + + cmd := exec.Command(tgitea.GiteaPath, "admin", "create-user", "--name", giteaUser01, "--email", giteaUser01+"@example.com", "--password", "password", "--admin", "--config", tgitea.ConfigPath) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("unexpected err: %v, out: %s", err, out) + } + + return tgitea +} + +func shutdownGitea(tgitea *testutil.TestGitea) { + tgitea.Kill() +} + +func startAgola(ctx context.Context, t *testing.T, dir string, c *config.Config) (<-chan error, error) { + rs, err := rsscheduler.NewRunservice(ctx, &c.Runservice) + if err != nil { + return nil, errors.Errorf("failed to start run service scheduler: %w", err) + } + + ex, err := executor.NewExecutor(&c.Executor) + if err != nil { + return nil, errors.Errorf("failed to start run service executor: %w", err) + } + + cs, err := configstore.NewConfigstore(ctx, &c.Configstore) + if err != nil { + return nil, errors.Errorf("failed to start config store: %w", err) + } + + sched, err := scheduler.NewScheduler(&c.Scheduler) + if err != nil { + return nil, errors.Errorf("failed to start scheduler: %w", err) + } + + ns, err := notification.NewNotificationService(c) + if err != nil { + return nil, errors.Errorf("failed to start notification service: %w", err) + } + + gw, err := gateway.NewGateway(c) + if err != nil { + return nil, errors.Errorf("failed to start gateway: %w", err) + } + + gs, err := gitserver.NewGitserver(&c.Gitserver) + if err != nil { + return nil, errors.Errorf("failed to start git server: %w", err) + } + + errCh := make(chan error) + + go func() { errCh <- rs.Run(ctx) }() + go func() { errCh <- ex.Run(ctx) }() + go func() { errCh <- cs.Run(ctx) }() + go func() { errCh <- sched.Run(ctx) }() + go func() { errCh <- ns.Run(ctx) }() + go func() { errCh <- gw.Run(ctx) }() + go func() { errCh <- gs.Run(ctx) }() + + // TODO(sgotti) find a better way to test that all is ready instead of sleeping + time.Sleep(5 * time.Second) + + return errCh, nil +} + +func setup(ctx context.Context, t *testing.T, dir string) (*testutil.TestEmbeddedEtcd, *testutil.TestGitea, *config.Config) { + toolboxPath := os.Getenv("AGOLA_TOOLBOX_PATH") + if toolboxPath == "" { + t.Fatalf("env var AGOLA_TOOLBOX_PATH is undefined") + } + + c := &config.Config{ + ID: "agola", + Gateway: config.Gateway{ + Debug: false, + APIExposedURL: "", + WebExposedURL: "", + RunserviceURL: "", + ConfigstoreURL: "", + GitserverURL: "", + Web: config.Web{ + ListenAddress: "", + TLS: false, + }, + TokenSigning: config.TokenSigning{ + Duration: 12 * time.Hour, + Method: "hmac", + Key: "supersecretsigningkey", + }, + AdminToken: "admintoken", + }, + Scheduler: config.Scheduler{ + Debug: false, + RunserviceURL: "", + }, + Notification: config.Notification{ + Debug: false, + WebExposedURL: "", + RunserviceURL: "", + ConfigstoreURL: "", + Etcd: config.Etcd{ + Endpoints: "", + }, + }, + Runservice: config.Runservice{ + Debug: false, + DataDir: filepath.Join(dir, "runservice"), + Web: config.Web{ + ListenAddress: ":4000", + TLS: false, + }, + Etcd: config.Etcd{ + Endpoints: "", + }, + ObjectStorage: config.ObjectStorage{ + Type: "posix", + Path: filepath.Join(dir, "runservice/ost"), + }, + RunCacheExpireInterval: 604800000000000, + }, + Executor: config.Executor{ + Debug: false, + DataDir: filepath.Join(dir, "executor"), + RunserviceURL: "", + ToolboxPath: toolboxPath, + Web: config.Web{ + ListenAddress: ":4001", + TLS: false, + }, + Driver: config.Driver{ + Type: "docker", + }, + Labels: map[string]string{}, + ActiveTasksLimit: 2, + }, + Configstore: config.Configstore{ + Debug: false, + DataDir: filepath.Join(dir, "configstore"), + Web: config.Web{ + ListenAddress: ":4002", + TLS: false, + }, + Etcd: config.Etcd{ + Endpoints: "", + }, + ObjectStorage: config.ObjectStorage{ + Type: "posix", + Path: filepath.Join(dir, "configstore/ost"), + }, + }, + Gitserver: config.Gitserver{ + Debug: false, + DataDir: filepath.Join(dir, "gitserver"), + Web: config.Web{ + ListenAddress: ":4003", + TLS: false, + }, + Etcd: config.Etcd{ + Endpoints: "", + }, + }, + } + + tgitea := setupGitea(t, dir) + + etcdDir := filepath.Join(dir, "etcd") + tetcd := setupEtcd(t, etcdDir) + + c.Runservice.Etcd.Endpoints = tetcd.Endpoint + c.Configstore.Etcd.Endpoints = tetcd.Endpoint + + _, gwPort, err := testutil.GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, csPort, err := testutil.GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, rsPort, err := testutil.GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, exPort, err := testutil.GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + listenAddress, gitServerPort, err := testutil.GetFreePort(true, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + gwURL := fmt.Sprintf("http://%s:%s", listenAddress, gwPort) + csURL := fmt.Sprintf("http://%s:%s", listenAddress, csPort) + rsURL := fmt.Sprintf("http://%s:%s", listenAddress, rsPort) + gitServerURL := fmt.Sprintf("http://%s:%s", listenAddress, gitServerPort) + + c.Gateway.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, gwPort) + c.Configstore.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, csPort) + c.Runservice.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, rsPort) + c.Executor.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, exPort) + c.Gitserver.Web.ListenAddress = fmt.Sprintf("%s:%s", listenAddress, gitServerPort) + + c.Gateway.APIExposedURL = gwURL + c.Gateway.WebExposedURL = gwURL + c.Gateway.RunserviceURL = rsURL + c.Gateway.ConfigstoreURL = csURL + c.Gateway.GitserverURL = gitServerURL + + c.Scheduler.RunserviceURL = rsURL + + c.Notification.WebExposedURL = gwURL + c.Notification.RunserviceURL = rsURL + c.Notification.ConfigstoreURL = csURL + + c.Executor.RunserviceURL = rsURL + + errCh, err := startAgola(ctx, t, dir, c) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + go func() { + err := <-errCh + if err != nil { + panic(fmt.Errorf("agola component returned error: %+v", err)) + } + }() + + return tetcd, tgitea, c +} + +func TestCreateLinkedAccount(t *testing.T) { + dir, err := ioutil.TempDir("", "agola") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tetcd, tgitea, c := setup(ctx, t, dir) + defer shutdownGitea(tgitea) + defer shutdownEtcd(tetcd) + + createLinkedAccount(ctx, t, tgitea, c) +} + +func createLinkedAccount(ctx context.Context, t *testing.T, tgitea *testutil.TestGitea, c *config.Config) (string, string) { + giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort) + giteaClient := gitea.NewClient(giteaAPIURL, "") + + giteaToken, err := giteaClient.CreateAccessToken(giteaUser01, "password", gtypes.CreateAccessTokenOption{Name: "token01"}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created gitea user token: %s", giteaToken.Token) + + gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, "admintoken") + user, _, err := gwClient.CreateUser(ctx, &gwapi.CreateUserRequest{UserName: agolaUser01}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created agola user: %s", user.UserName) + + token, _, err := gwClient.CreateUserToken(ctx, agolaUser01, &gwapi.CreateUserTokenRequest{TokenName: "token01"}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created agola user token: %s", token.Token) + + rs, _, err := gwClient.CreateRemoteSource(ctx, &gwapi.CreateRemoteSourceRequest{ + Name: "gitea", + APIURL: giteaAPIURL, + Type: "gitea", + AuthType: "password", + SkipSSHHostKeyCheck: true, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created agola remote source: %s", rs.Name) + + // From now use the user token + gwClient = gwapi.NewClient(c.Gateway.APIExposedURL, token.Token) + + la, _, err := gwClient.CreateUserLA(ctx, agolaUser01, &gwapi.CreateUserLARequest{ + RemoteSourceName: "gitea", + RemoteSourceLoginName: giteaUser01, + RemoteSourceLoginPassword: "password", + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created user linked account: %s", util.Dump(la)) + + return giteaToken.Token, token.Token +} + +func TestCreateProject(t *testing.T) { + dir, err := ioutil.TempDir("", "agola") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tetcd, tgitea, c := setup(ctx, t, dir) + defer shutdownGitea(tgitea) + defer shutdownEtcd(tetcd) + + giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort) + + giteaToken, token := createLinkedAccount(ctx, t, tgitea, c) + + giteaClient := gitea.NewClient(giteaAPIURL, giteaToken) + gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, token) + + createProject(ctx, t, giteaClient, gwClient) +} + +func createProject(ctx context.Context, t *testing.T, giteaClient *gitea.Client, gwClient *gwapi.Client) (*gtypes.Repository, *gwapi.ProjectResponse) { + giteaRepo, err := giteaClient.CreateRepo(gtypes.CreateRepoOption{ + Name: "repo01", + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created gitea repo: %s", giteaRepo.Name) + + project, _, err := gwClient.CreateProject(ctx, &gwapi.CreateProjectRequest{ + Name: "project01", + ParentRef: path.Join("user", agolaUser01), + RemoteSourceName: "gitea", + RepoPath: path.Join(giteaUser01, "repo01"), + Visibility: types.VisibilityPublic, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + return giteaRepo, project +} + +func TestRun(t *testing.T) { + dir, err := ioutil.TempDir("", "agola") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tetcd, tgitea, c := setup(ctx, t, dir) + defer shutdownGitea(tgitea) + defer shutdownEtcd(tetcd) + + giteaAPIURL := fmt.Sprintf("http://%s:%s", tgitea.ListenAddress, tgitea.HTTPPort) + + giteaToken, token := createLinkedAccount(ctx, t, tgitea, c) + + giteaClient := gitea.NewClient(giteaAPIURL, giteaToken) + gwClient := gwapi.NewClient(c.Gateway.APIExposedURL, token) + + giteaRepo, project := createProject(ctx, t, giteaClient, gwClient) + + gitfs := memfs.New() + f, err := gitfs.Create(".agola/config.jsonnet") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, err = f.Write([]byte( + `{ + runs: [ + { + name: 'run01', + tasks: [ + { + name: 'task01', + runtime: { + containers: [ + { + image: 'busybox', + }, + ], + }, + steps: [ + { type: 'run', command: 'env' }, + ], + }, + ], + }, + ], +} +`)) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + r, err := git.Init(memory.NewStorage(), gitfs) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + if _, err := r.CreateRemote(&gitconfig.RemoteConfig{ + Name: "origin", + URLs: []string{giteaRepo.CloneURL}, + }); err != nil { + t.Fatalf("unexpected err: %v", err) + } + + wt, err := r.Worktree() + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if _, err := wt.Add(".agola/config.jsonnet"); err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, err = wt.Commit("commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "user01", + Email: "user01@example.com", + When: time.Now(), + }, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + t.Logf("sshurl: %s", giteaRepo.CloneURL) + if err := r.Push(&git.PushOptions{ + RemoteName: "origin", + Auth: &http.BasicAuth{ + Username: giteaUser01, + Password: giteaToken, + }, + }); err != nil { + t.Fatalf("unexpected err: %v", err) + } + + // TODO(sgotti) add an util to wait for a run phase + time.Sleep(10 * time.Second) + + runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/project", project.ID)}, nil, "", 0, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + t.Logf("runs: %s", util.Dump(runs)) + + if len(runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs)) + } + + run := runs[0] + if run.Phase != rstypes.RunPhaseFinished { + t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) + } +}