config: add starlark config support
Handle `.agola/config.star` files in starlark config format. To provide a context like done for jsonnet we require that the starlark agola config file contains a main function that will receive a config context as a dict. We also had to implement our own json conversion from a starlark dict since go starlark removed its own function.
This commit is contained in:
Simone Gotti
parent
714e561c75
commit
d91ec09d7d
1
go.mod
1
go.mod
|
|
@ -32,6 +32,7 @@ require (
|
|||
github.com/spf13/cobra v0.0.5
|
||||
github.com/xanzy/go-gitlab v0.26.0
|
||||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
|
||||
go.starlark.net v0.0.0-20200203144150-6677ee5c7211
|
||||
go.uber.org/zap v1.13.0
|
||||
golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -59,6 +59,9 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
|
|||
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/bmatcuk/doublestar v1.2.2 h1:oC24CykoSAB8zd7XgruHo33E0cHJf/WhQA/7BeXj+x0=
|
||||
github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
|
|
@ -429,6 +432,8 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0
|
|||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.starlark.net v0.0.0-20200203144150-6677ee5c7211 h1:Qoe+9POtDT51UBQ8XEnS9QKeHDQzEl2QRh3eok9R4aw=
|
||||
go.starlark.net v0.0.0-20200203144150-6677ee5c7211/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
|
|
@ -528,6 +533,7 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import (
|
|||
"agola.io/agola/services/types"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/google/go-jsonnet"
|
||||
errors "golang.org/x/xerrors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
)
|
||||
|
|
@ -44,6 +43,7 @@ const (
|
|||
// ConfigFormatJSON handles both json or yaml format (since json is a subset of yaml)
|
||||
ConfigFormatJSON ConfigFormat = iota
|
||||
ConfigFormatJsonnet
|
||||
ConfigFormatStarlark
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -660,19 +660,19 @@ type ConfigContext struct {
|
|||
|
||||
func ParseConfig(configData []byte, format ConfigFormat, configContext *ConfigContext) (*Config, error) {
|
||||
// Generate json from jsonnet
|
||||
if format == ConfigFormatJsonnet {
|
||||
// TODO(sgotti) support custom import files inside the configdir ???
|
||||
vm := jsonnet.MakeVM()
|
||||
cj, err := json.Marshal(configContext)
|
||||
switch format {
|
||||
case ConfigFormatJsonnet:
|
||||
var err error
|
||||
configData, err = execJsonnet(configData, configContext)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("failed to marshal config context: %w", err)
|
||||
return nil, errors.Errorf("failed to execute jsonnet: %w", err)
|
||||
}
|
||||
vm.TLACode("ctx", string(cj))
|
||||
out, err := vm.EvaluateSnippet("", string(configData))
|
||||
case ConfigFormatStarlark:
|
||||
var err error
|
||||
configData, err = execStarlark(configData, configContext)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("failed to evaluate jsonnet config: %w", err)
|
||||
return nil, errors.Errorf("failed to execute starlark: %w", err)
|
||||
}
|
||||
configData = []byte(out)
|
||||
}
|
||||
|
||||
config := DefaultConfig
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright 2020 Sorint.lab
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/google/go-jsonnet"
|
||||
errors "golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func execJsonnet(configData []byte, configContext *ConfigContext) ([]byte, error) {
|
||||
vm := jsonnet.MakeVM()
|
||||
cj, err := json.Marshal(configContext)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("failed to marshal config context: %w", err)
|
||||
}
|
||||
|
||||
vm.TLACode("ctx", string(cj))
|
||||
out, err := vm.EvaluateSnippet("", string(configData))
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("failed to evaluate jsonnet config: %w", err)
|
||||
}
|
||||
|
||||
return []byte(out), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
// Copyright 2020 Sorint.lab
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
errors "golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func starlarkArgs(cc *ConfigContext) (starlark.Tuple, error) {
|
||||
d := &starlark.Dict{}
|
||||
if err := d.SetKey(starlark.String("ref_type"), starlark.String(cc.RefType)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.SetKey(starlark.String("ref"), starlark.String(cc.Ref)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.SetKey(starlark.String("branch"), starlark.String(cc.Branch)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.SetKey(starlark.String("tag"), starlark.String(cc.Tag)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.SetKey(starlark.String("pull_request_id"), starlark.String(cc.PullRequestID)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.SetKey(starlark.String("commit_sha"), starlark.String(cc.CommitSHA)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []starlark.Value{d}, nil
|
||||
}
|
||||
|
||||
// based on (not existing anymore) function provided in
|
||||
// https://github.com/google/starlark-go/blob/6fffce7528ee0fce17d72a4abe2919f464225968/starlarkstruct/struct.go#L325
|
||||
// with changes to use go json marshalling functions
|
||||
func starlarkJSON(out *bytes.Buffer, v starlark.Value) error {
|
||||
switch v := v.(type) {
|
||||
case starlark.NoneType:
|
||||
out.WriteString("null")
|
||||
case starlark.Bool:
|
||||
fmt.Fprintf(out, "%t", v)
|
||||
case starlark.Int:
|
||||
data, err := json.Marshal(v.BigInt())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Write(data)
|
||||
case starlark.Float:
|
||||
data, err := json.Marshal(float64(v))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.Write(data)
|
||||
case starlark.String:
|
||||
// we have to use a json Encoder to disable noisy html
|
||||
// escaping. But the encoder appends a final \n so we
|
||||
// also should remove it.
|
||||
data := &bytes.Buffer{}
|
||||
e := json.NewEncoder(data)
|
||||
e.SetEscapeHTML(false)
|
||||
if err := e.Encode(string(v)); err != nil {
|
||||
return err
|
||||
}
|
||||
// remove final \n introduced by the encoder
|
||||
out.Write(bytes.TrimSuffix(data.Bytes(), []byte("\n")))
|
||||
case starlark.Indexable: // Tuple, List
|
||||
out.WriteByte('[')
|
||||
for i, n := 0, starlark.Len(v); i < n; i++ {
|
||||
if i > 0 {
|
||||
out.WriteString(", ")
|
||||
}
|
||||
if err := starlarkJSON(out, v.Index(i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out.WriteByte(']')
|
||||
case *starlark.Dict:
|
||||
out.WriteByte('{')
|
||||
for i, item := range v.Items() {
|
||||
if i > 0 {
|
||||
out.WriteString(", ")
|
||||
}
|
||||
if _, ok := item[0].(starlark.String); !ok {
|
||||
return fmt.Errorf("cannot convert non-string dict key to JSON")
|
||||
}
|
||||
if err := starlarkJSON(out, item[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
out.WriteString(": ")
|
||||
if err := starlarkJSON(out, item[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out.WriteByte('}')
|
||||
|
||||
default:
|
||||
return fmt.Errorf("cannot convert starlark type %q to JSON", v.Type())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func execStarlark(configData []byte, configContext *ConfigContext) ([]byte, error) {
|
||||
thread := &starlark.Thread{
|
||||
Name: "agola-starlark",
|
||||
// TODO(sgotti) redirect print to a logger?
|
||||
Print: func(_ *starlark.Thread, msg string) {},
|
||||
}
|
||||
globals, err := starlark.ExecFile(thread, "config.star", configData, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// we require a main function that will be called wiht one
|
||||
// arguments containing the config context
|
||||
mainVal, ok := globals["main"]
|
||||
if !ok {
|
||||
return nil, errors.Errorf("no main function in starlark config")
|
||||
}
|
||||
main, ok := mainVal.(starlark.Callable)
|
||||
if !ok {
|
||||
return nil, errors.Errorf("main in starlark config is not a function")
|
||||
}
|
||||
args, err := starlarkArgs(configContext)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("cannot create startlark arguments: %w", err)
|
||||
}
|
||||
mainVal, err = starlark.Call(thread, main, args, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
switch v := mainVal.(type) {
|
||||
case *starlark.Dict:
|
||||
if err := starlarkJSON(buf, v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, errors.Errorf("wrong starlark output, must be a dict")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
// Copyright 2020 Sorint.lab
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"go.starlark.net/starlark"
|
||||
)
|
||||
|
||||
func TestStarlarkJSON(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
in starlark.Value
|
||||
out string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "test key as a string",
|
||||
in: func() starlark.Value {
|
||||
s := &starlark.Dict{}
|
||||
_ = s.SetKey(starlark.String("key"), starlark.String("string01"))
|
||||
return starlark.Value(s)
|
||||
}(),
|
||||
out: `{"key": "string01"}`,
|
||||
},
|
||||
{
|
||||
name: "test key not a string",
|
||||
in: func() starlark.Value {
|
||||
s := &starlark.Dict{}
|
||||
_ = s.SetKey(starlark.MakeInt(10), starlark.String("string01"))
|
||||
return starlark.Value(s)
|
||||
}(),
|
||||
err: fmt.Errorf("cannot convert non-string dict key to JSON"),
|
||||
},
|
||||
{
|
||||
name: "test list",
|
||||
in: func() starlark.Value {
|
||||
l := []starlark.Value{
|
||||
starlark.String("\ns\ttring01"),
|
||||
starlark.MakeInt(10),
|
||||
starlark.Bool(true),
|
||||
starlark.Float(math.MaxFloat64),
|
||||
func() starlark.Value {
|
||||
s := &starlark.Dict{}
|
||||
_ = s.SetKey(starlark.String("key"), starlark.String("string01"))
|
||||
return starlark.Value(s)
|
||||
}(),
|
||||
}
|
||||
return starlark.NewList(l)
|
||||
}(),
|
||||
out: `["\ns\ttring01", 10, true, 1.7976931348623157e+308, {"key": "string01"}]`,
|
||||
},
|
||||
{
|
||||
name: "test string special chars",
|
||||
in: starlark.String("\ns\ttring01"),
|
||||
out: `"\ns\ttring01"`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
out := new(bytes.Buffer)
|
||||
if err := starlarkJSON(out, tt.in); err != nil {
|
||||
if tt.err == nil {
|
||||
t.Fatalf("got error: %v, expected no error", err)
|
||||
}
|
||||
if err.Error() != tt.err.Error() {
|
||||
t.Fatalf("got error: %v, want error: %v", err, tt.err)
|
||||
}
|
||||
} else {
|
||||
if tt.err != nil {
|
||||
t.Fatalf("got nil error, want error: %v", tt.err)
|
||||
}
|
||||
}
|
||||
|
||||
if tt.err == nil {
|
||||
if diff := cmp.Diff(tt.out, out.String()); diff != "" {
|
||||
t.Fatalf(diff)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -38,10 +38,11 @@ import (
|
|||
const (
|
||||
defaultSSHPort = "22"
|
||||
|
||||
agolaDefaultConfigDir = ".agola"
|
||||
agolaDefaultJsonnetConfigFile = "config.jsonnet"
|
||||
agolaDefaultJsonConfigFile = "config.json"
|
||||
agolaDefaultYamlConfigFile = "config.yml"
|
||||
agolaDefaultConfigDir = ".agola"
|
||||
agolaDefaultStarlarkConfigFile = "config.star"
|
||||
agolaDefaultJsonnetConfigFile = "config.jsonnet"
|
||||
agolaDefaultJsonConfigFile = "config.json"
|
||||
agolaDefaultYamlConfigFile = "config.yml"
|
||||
|
||||
// List of runs annotations
|
||||
AnnotationRunType = "run_type"
|
||||
|
|
@ -489,6 +490,8 @@ func (h *ActionHandler) CreateRuns(ctx context.Context, req *CreateRunRequest) e
|
|||
|
||||
var configFormat config.ConfigFormat
|
||||
switch path.Ext(filename) {
|
||||
case ".star":
|
||||
configFormat = config.ConfigFormatStarlark
|
||||
case ".jsonnet":
|
||||
configFormat = config.ConfigFormatJsonnet
|
||||
case ".json":
|
||||
|
|
@ -566,7 +569,7 @@ func (h *ActionHandler) fetchConfigFiles(ctx context.Context, gitSource gitsourc
|
|||
var data []byte
|
||||
var filename string
|
||||
err := util.ExponentialBackoff(ctx, util.FetchFileBackoff, func() (bool, error) {
|
||||
for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
|
||||
for _, filename = range []string{agolaDefaultStarlarkConfigFile, agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
|
||||
var err error
|
||||
data, err = gitSource.GetFile(repopath, commitSHA, path.Join(agolaDefaultConfigDir, filename))
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -64,6 +64,15 @@ const (
|
|||
agolaUser01 = "user01"
|
||||
)
|
||||
|
||||
type ConfigFormat string
|
||||
|
||||
const (
|
||||
// ConfigFormatJSON handles both json or yaml format (since json is a subset of yaml)
|
||||
ConfigFormatJSON ConfigFormat = "json"
|
||||
ConfigFormatJsonnet ConfigFormat = "jsonnet"
|
||||
ConfigFormatStarlark ConfigFormat = "starlark"
|
||||
)
|
||||
|
||||
func setupEtcd(t *testing.T, logger *zap.Logger, dir string) *testutil.TestEmbeddedEtcd {
|
||||
tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
|
||||
if err != nil {
|
||||
|
|
@ -850,7 +859,7 @@ func TestPush(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func directRun(t *testing.T, dir, config, gatewayURL, token string, args ...string) {
|
||||
func directRun(t *testing.T, dir, config string, configFormat ConfigFormat, gatewayURL, token string, args ...string) {
|
||||
agolaBinDir := os.Getenv("AGOLA_BIN_DIR")
|
||||
if agolaBinDir == "" {
|
||||
t.Fatalf("env var AGOLA_BIN_DIR is undefined")
|
||||
|
|
@ -868,7 +877,15 @@ func directRun(t *testing.T, dir, config, gatewayURL, token string, args ...stri
|
|||
gitfs := osfs.New(repoDir)
|
||||
dot, _ := gitfs.Chroot(".git")
|
||||
|
||||
f, err := gitfs.Create(".agola/config.jsonnet")
|
||||
var configPath string
|
||||
switch configFormat {
|
||||
case ConfigFormatJsonnet:
|
||||
configPath = ".agola/config.jsonnet"
|
||||
case ConfigFormatStarlark:
|
||||
configPath = ".agola/config.star"
|
||||
}
|
||||
|
||||
f, err := gitfs.Create(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
|
|
@ -987,7 +1004,7 @@ func TestDirectRun(t *testing.T) {
|
|||
// From now use the user token
|
||||
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
|
||||
|
||||
directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
directRun(t, dir, config, ConfigFormatJsonnet, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
|
||||
_ = testutil.Wait(30*time.Second, func() (bool, error) {
|
||||
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
|
||||
|
|
@ -1140,7 +1157,7 @@ func TestDirectRunVariables(t *testing.T) {
|
|||
// From now use the user token
|
||||
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
|
||||
|
||||
directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
directRun(t, dir, config, ConfigFormatJsonnet, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
|
||||
// TODO(sgotti) add an util to wait for a run phase
|
||||
_ = testutil.Wait(30*time.Second, func() (bool, error) {
|
||||
|
|
@ -1311,7 +1328,7 @@ func TestDirectRunLogs(t *testing.T) {
|
|||
// From now use the user token
|
||||
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
|
||||
|
||||
directRun(t, dir, config, c.Gateway.APIExposedURL, token)
|
||||
directRun(t, dir, config, ConfigFormatJsonnet, c.Gateway.APIExposedURL, token)
|
||||
|
||||
_ = testutil.Wait(30*time.Second, func() (bool, error) {
|
||||
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
|
||||
|
|
@ -1682,7 +1699,7 @@ func TestPullRequest(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestConfigContext(t *testing.T) {
|
||||
config := `
|
||||
jsonnetConfig := `
|
||||
function(ctx) {
|
||||
runs: [
|
||||
{
|
||||
|
|
@ -1714,6 +1731,41 @@ function(ctx) {
|
|||
},
|
||||
],
|
||||
}
|
||||
`
|
||||
|
||||
starlarkConfig := `
|
||||
def main(ctx):
|
||||
return {
|
||||
"runs": [
|
||||
{
|
||||
"name": 'run01',
|
||||
"tasks": [
|
||||
{
|
||||
"name": 'task01',
|
||||
"runtime": {
|
||||
"containers": [
|
||||
{
|
||||
"image": 'alpine/git',
|
||||
}
|
||||
]
|
||||
},
|
||||
"environment": {
|
||||
"REF_TYPE": ctx["ref_type"],
|
||||
"REF": ctx["ref"],
|
||||
"BRANCH": ctx["branch"],
|
||||
"TAG": ctx["tag"],
|
||||
"PULL_REQUEST_ID": ctx["pull_request_id"],
|
||||
"COMMIT_SHA": ctx["commit_sha"]
|
||||
},
|
||||
"steps": [
|
||||
{ "type": 'clone' },
|
||||
{ "type": 'run', "command": 'env' }
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
`
|
||||
|
||||
tests := []struct {
|
||||
|
|
@ -1758,111 +1810,121 @@ function(ctx) {
|
|||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "agola")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
for _, configFormat := range []ConfigFormat{ConfigFormatJsonnet, ConfigFormatStarlark} {
|
||||
for _, tt := range tests {
|
||||
t.Run(fmt.Sprintf("%s with %s config", tt.name, configFormat), func(t *testing.T) {
|
||||
var config string
|
||||
switch configFormat {
|
||||
case ConfigFormatJsonnet:
|
||||
config = jsonnetConfig
|
||||
case ConfigFormatStarlark:
|
||||
config = starlarkConfig
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir, err := ioutil.TempDir("", "agola")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
tetcd, tgitea, c := setup(ctx, t, dir)
|
||||
defer shutdownGitea(tgitea)
|
||||
defer shutdownEtcd(tetcd)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
|
||||
user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
t.Logf("created agola user: %s", user.UserName)
|
||||
tetcd, tgitea, c := setup(ctx, t, dir)
|
||||
defer shutdownGitea(tgitea)
|
||||
defer shutdownEtcd(tetcd)
|
||||
|
||||
token := createAgolaUserToken(ctx, t, c)
|
||||
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
|
||||
user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
t.Logf("created agola user: %s", user.UserName)
|
||||
|
||||
// From now use the user token
|
||||
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
|
||||
token := createAgolaUserToken(ctx, t, c)
|
||||
|
||||
directRun(t, dir, config, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
// From now use the user token
|
||||
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
|
||||
|
||||
directRun(t, dir, config, configFormat, c.Gateway.APIExposedURL, token, tt.args...)
|
||||
|
||||
// TODO(sgotti) add an util to wait for a run phase
|
||||
_ = testutil.Wait(30*time.Second, func() (bool, error) {
|
||||
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if len(runs) != 1 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
run := runs[0]
|
||||
if run.Phase != rstypes.RunPhaseFinished {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
// TODO(sgotti) add an util to wait for a run phase
|
||||
_ = testutil.Wait(30*time.Second, func() (bool, error) {
|
||||
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("runs: %s", util.Dump(runs))
|
||||
|
||||
if len(runs) != 1 {
|
||||
return false, nil
|
||||
t.Fatalf("expected 1 run got: %d", len(runs))
|
||||
}
|
||||
|
||||
run := runs[0]
|
||||
run, _, err := gwClient.GetRun(ctx, runs[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if run.Phase != rstypes.RunPhaseFinished {
|
||||
return false, nil
|
||||
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
|
||||
}
|
||||
if run.Result != rstypes.RunResultSuccess {
|
||||
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("runs: %s", util.Dump(runs))
|
||||
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("expected 1 run got: %d", len(runs))
|
||||
}
|
||||
|
||||
run, _, err := gwClient.GetRun(ctx, runs[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if run.Phase != rstypes.RunPhaseFinished {
|
||||
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
|
||||
}
|
||||
if run.Result != rstypes.RunResultSuccess {
|
||||
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
|
||||
}
|
||||
|
||||
var task *gwapitypes.RunResponseTask
|
||||
for _, t := range run.Tasks {
|
||||
if t.Name == "task01" {
|
||||
task = t
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := gwClient.GetLogs(ctx, run.ID, task.ID, false, 1, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
logs, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
curEnv, err := testutil.ParseEnvs(bytes.NewReader(logs))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
// update commit sha from annotations since it will change at every test
|
||||
tt.env["COMMIT_SHA"] = run.Annotations["commit_sha"]
|
||||
|
||||
for n, e := range tt.env {
|
||||
if ce, ok := curEnv[n]; !ok {
|
||||
t.Fatalf("missing env var %s", n)
|
||||
} else {
|
||||
if ce != e {
|
||||
t.Fatalf("different env var %s value, want: %q, got %q", n, e, ce)
|
||||
var task *gwapitypes.RunResponseTask
|
||||
for _, t := range run.Tasks {
|
||||
if t.Name == "task01" {
|
||||
task = t
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
resp, err := gwClient.GetLogs(ctx, run.ID, task.ID, false, 1, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
logs, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
curEnv, err := testutil.ParseEnvs(bytes.NewReader(logs))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
// update commit sha from annotations since it will change at every test
|
||||
tt.env["COMMIT_SHA"] = run.Annotations["commit_sha"]
|
||||
|
||||
for n, e := range tt.env {
|
||||
if ce, ok := curEnv[n]; !ok {
|
||||
t.Fatalf("missing env var %s", n)
|
||||
} else {
|
||||
if ce != e {
|
||||
t.Fatalf("different env var %s value, want: %q, got %q", n, e, ce)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue