a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

runservice: handle jsonnet and json config files 

Handle config files with name `config.jsonnet`, `config.json` and
`config.yml` and take the first from the repository in this order

For a jsonnet file execute it and use the generated output as the config
This commit is contained in:
Simone Gotti 2019-04-16 15:00:37 +02:00
parent 03451535c8
commit 4c30a5af1c
5 changed files with 71 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -19,6 +19,7 @@ require (
github.com/go-ini/ini v1.42.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/google/go-cmp v0.3.0
github.com/google/go-jsonnet v0.12.1
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.7.0
@ -38,6 +39,7 @@ require (
github.com/pkg/errors v0.8.0
github.com/sanity-io/litter v1.1.0
github.com/satori/go.uuid v1.2.0
github.com/sergi/go-diff v1.0.0 // indirect
github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect

7
go.sum
View File

@ -62,8 +62,12 @@ github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU=
github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
@ -124,7 +128,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -154,6 +157,8 @@ github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcT
github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a h1:u7WP9TGHJIkJoi/dRDhvYPSthMIdUQPDETiZET/Utl8=
github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a/go.mod h1:HvB0+YQff1QGS1nct9E3/J8wo8s/EVjq+VXrJSDlQEY=
github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=

22
internal/config/config.go
View File

@ -25,6 +25,7 @@ import (
"github.com/sorintlab/agola/internal/util"
"github.com/ghodss/yaml"
"github.com/google/go-jsonnet"
"github.com/pkg/errors"
)
@ -34,6 +35,14 @@ const (
maxStepNameLength = 100
)
type ConfigFormat int
const (
// ConfigFormatJSON handles both json or yaml format (since json is a subset of yaml)
ConfigFormatJSON ConfigFormat = iota
ConfigFormatJsonnet
)
var (
regExpDelimiters = []string{"/", "#"}
)
@ -595,7 +604,18 @@ func (r *Run) Task(taskName string) *Task {
var DefaultConfig = Config{}
func ParseConfig(configData []byte) (*Config, error) {
func ParseConfig(configData []byte, format ConfigFormat) (*Config, error) {
// Generate json from jsonnet
if format == ConfigFormatJsonnet {
// TODO(sgotti) support custom import files inside the configdir ???
vm := jsonnet.MakeVM()
out, err := vm.EvaluateSnippet("", string(configData))
if err != nil {
return nil, errors.Wrapf(err, "failed to evaluate jsonnet config")
}
configData = []byte(out)
}
config := DefaultConfig
if err := yaml.Unmarshal(configData, &config); err != nil {
return nil, errors.Wrapf(err, "failed to unmarshal config")

4
internal/config/config_test.go
View File

@ -109,7 +109,7 @@ func TestParseConfig(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := ParseConfig([]byte(tt.in)); err != nil {
if _, err := ParseConfig([]byte(tt.in), ConfigFormatJSON); err != nil {
if tt.err == nil {
t.Fatalf("got error: %v, expected no error", err)
}
@ -417,7 +417,7 @@ func TestParseOutput(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out, err := ParseConfig([]byte(tt.in))
out, err := ParseConfig([]byte(tt.in), ConfigFormatJSON)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

54
internal/services/gateway/webhook.go
View File

@ -39,7 +39,10 @@ import (
const (
defaultSSHPort = "22"
agolaDefaultConfigPath = ".agola/config.yml"
agolaDefaultConfigDir = ".agola"
agolaDefaultJsonnetConfigFile = "config.jsonnet"
agolaDefaultJsonConfigFile = "config.json"
agolaDefaultYamlConfigFile = "config.yml"
// List of runs annotations
AnnotationEventType = "event_type"
@ -252,16 +255,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
h.log.Infof("webhookData: %s", util.Dump(webhookData))
var data []byte
err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
var err error
data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, agolaDefaultConfigPath)
if err == nil {
return true, nil
}
h.log.Errorf("get file err: %v", err)
return false, nil
})
data, filename, err := h.fetchConfigFiles(gitSource, webhookData)
if err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to fetch config file")
}
@ -330,7 +324,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
group = genGroup(GroupTypeUser, userID, webhookData)
}
if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil {
if err := h.createRuns(ctx, filename, data, group, annotations, env, variables, webhookData); err != nil {
return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")
}
//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
@ -340,10 +334,42 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
return 0, "", nil
}
func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
// fetchConfigFiles tries to fetch a config file in one of the supported formats. The precedence is for jsonnet, then json and then yml
// TODO(sgotti) For jsonnet, if we'll support custom import files inside the configdir, also fetch them.
func (h *webhooksHandler) fetchConfigFiles(gitSource gitsource.GitSource, webhookData *types.WebhookData) ([]byte, string, error) {
var data []byte
var filename string
err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
var err error
data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, path.Join(agolaDefaultConfigDir, filename))
if err == nil {
return true, nil
}
h.log.Errorf("get file err: %v", err)
}
return false, nil
})
if err != nil {
return nil, "", err
}
return data, filename, nil
}
func (h *webhooksHandler) createRuns(ctx context.Context, filename string, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
setupErrors := []string{}
config, err := config.ParseConfig([]byte(configData))
var configFormat config.ConfigFormat
switch path.Ext(filename) {
case ".jsonnet":
configFormat = config.ConfigFormatJsonnet
case ".json":
fallthrough
case ".yml":
configFormat = config.ConfigFormatJSON
}
config, err := config.ParseConfig([]byte(configData), configFormat)
if err != nil {
log.Errorf("failed to parse config: %+v", err)