runservice: handle jsonnet and json config files

Handle config files with name `config.jsonnet`, `config.json` and
`config.yml` and take the first from the repository in this order

For a jsonnet file execute it and use the generated output as the config

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/go-ini/ini v1.42.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/google/go-cmp v0.3.0
+	github.com/google/go-jsonnet v0.12.1
 	github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
 	github.com/gorilla/handlers v1.4.0
 	github.com/gorilla/mux v1.7.0
@@ -38,6 +39,7 @@ require (
 	github.com/pkg/errors v0.8.0
 	github.com/sanity-io/litter v1.1.0
 	github.com/satori/go.uuid v1.2.0
+	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a
 	github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect

go.sum

@@ -62,8 +62,12 @@ github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU=
+github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
@@ -124,7 +128,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -154,6 +157,8 @@ github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcT
 github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a h1:u7WP9TGHJIkJoi/dRDhvYPSthMIdUQPDETiZET/Utl8=
 github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a/go.mod h1:HvB0+YQff1QGS1nct9E3/J8wo8s/EVjq+VXrJSDlQEY=
 github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=

internal/config/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/sorintlab/agola/internal/util"
 
 	"github.com/ghodss/yaml"
+	"github.com/google/go-jsonnet"
 	"github.com/pkg/errors"
 )
 
@@ -34,6 +35,14 @@ const (
 	maxStepNameLength = 100
 )
 
+type ConfigFormat int
+
+const (
+	// ConfigFormatJSON handles both json or yaml format (since json is a subset of yaml)
+	ConfigFormatJSON ConfigFormat = iota
+	ConfigFormatJsonnet
+)
+
 var (
 	regExpDelimiters = []string{"/", "#"}
 )
@@ -595,7 +604,18 @@ func (r *Run) Task(taskName string) *Task {
 
 var DefaultConfig = Config{}
 
-func ParseConfig(configData []byte) (*Config, error) {
+func ParseConfig(configData []byte, format ConfigFormat) (*Config, error) {
+	// Generate json from jsonnet
+	if format == ConfigFormatJsonnet {
+		// TODO(sgotti) support custom import files inside the configdir ???
+		vm := jsonnet.MakeVM()
+		out, err := vm.EvaluateSnippet("", string(configData))
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to evaluate jsonnet config")
+		}
+		configData = []byte(out)
+	}
+
 	config := DefaultConfig
 	if err := yaml.Unmarshal(configData, &config); err != nil {
 		return nil, errors.Wrapf(err, "failed to unmarshal config")

internal/config/config_test.go

@@ -109,7 +109,7 @@ func TestParseConfig(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if _, err := ParseConfig([]byte(tt.in)); err != nil {
+			if _, err := ParseConfig([]byte(tt.in), ConfigFormatJSON); err != nil {
 				if tt.err == nil {
 					t.Fatalf("got error: %v, expected no error", err)
 				}
@@ -417,7 +417,7 @@ func TestParseOutput(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			out, err := ParseConfig([]byte(tt.in))
+			out, err := ParseConfig([]byte(tt.in), ConfigFormatJSON)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}

internal/services/gateway/webhook.go

@@ -39,7 +39,10 @@ import (
 const (
 	defaultSSHPort = "22"
 
-	agolaDefaultConfigPath = ".agola/config.yml"
+	agolaDefaultConfigDir         = ".agola"
+	agolaDefaultJsonnetConfigFile = "config.jsonnet"
+	agolaDefaultJsonConfigFile    = "config.json"
+	agolaDefaultYamlConfigFile    = "config.yml"
 
 	// List of runs annotations
 	AnnotationEventType = "event_type"
@@ -252,16 +255,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 
 	h.log.Infof("webhookData: %s", util.Dump(webhookData))
 
-	var data []byte
+	data, filename, err := h.fetchConfigFiles(gitSource, webhookData)
-	err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
-		var err error
-		data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, agolaDefaultConfigPath)
-		if err == nil {
-			return true, nil
-		}
-		h.log.Errorf("get file err: %v", err)
-		return false, nil
-	})
 	if err != nil {
 		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to fetch config file")
 	}
@@ -330,7 +324,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 		group = genGroup(GroupTypeUser, userID, webhookData)
 	}
 
-	if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil {
+	if err := h.createRuns(ctx, filename, data, group, annotations, env, variables, webhookData); err != nil {
 		return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run")
 	}
 	//if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil {
@@ -340,10 +334,42 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) {
 	return 0, "", nil
 }
 
-func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
+// fetchConfigFiles tries to fetch a config file in one of the supported formats. The precedence is for jsonnet, then json and then yml
+// TODO(sgotti) For jsonnet, if we'll support custom import files inside the configdir, also fetch them.
+func (h *webhooksHandler) fetchConfigFiles(gitSource gitsource.GitSource, webhookData *types.WebhookData) ([]byte, string, error) {
+	var data []byte
+	var filename string
+	err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) {
+		for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} {
+			var err error
+			data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, path.Join(agolaDefaultConfigDir, filename))
+			if err == nil {
+				return true, nil
+			}
+			h.log.Errorf("get file err: %v", err)
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, "", err
+	}
+	return data, filename, nil
+}
+
+func (h *webhooksHandler) createRuns(ctx context.Context, filename string, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error {
 	setupErrors := []string{}
 
-	config, err := config.ParseConfig([]byte(configData))
+	var configFormat config.ConfigFormat
+	switch path.Ext(filename) {
+	case ".jsonnet":
+		configFormat = config.ConfigFormatJsonnet
+	case ".json":
+		fallthrough
+	case ".yml":
+		configFormat = config.ConfigFormatJSON
+
+	}
+	config, err := config.ParseConfig([]byte(configData), configFormat)
 	if err != nil {
 		log.Errorf("failed to parse config: %+v", err)