initial secret and variables impl

cmd/agola/cmd/projectsecret.go

@@ -0,0 +1,28 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var cmdProjectSecret = &cobra.Command{
+	Use:   "secret",
+	Short: "secret",
+}
+
+func init() {
+	cmdProject.AddCommand(cmdProjectSecret)
+}

cmd/agola/cmd/projectsecretcreate.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/services/gateway/api"
+	"github.com/spf13/cobra"
+)
+
+var cmdProjectSecretCreate = &cobra.Command{
+	Use:   "create",
+	Short: "create a project secret",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := projectSecretCreate(cmd, args); err != nil {
+			log.Fatalf("err: %v", err)
+		}
+	},
+}
+
+type projectSecretCreateOptions struct {
+	projectID string
+	name      string
+}
+
+var projectSecretCreateOpts projectSecretCreateOptions
+
+func init() {
+	flags := cmdProjectSecretCreate.Flags()
+
+	flags.StringVar(&projectSecretCreateOpts.projectID, "project", "", "project id or full path)")
+	flags.StringVarP(&projectSecretCreateOpts.name, "name", "n", "", "secret name")
+
+	cmdProjectSecretCreate.MarkFlagRequired("project")
+	cmdProjectSecretCreate.MarkFlagRequired("name")
+
+	cmdProjectSecret.AddCommand(cmdProjectSecretCreate)
+}
+
+func projectSecretCreate(cmd *cobra.Command, args []string) error {
+	gwclient := api.NewClient(gatewayURL, token)
+
+	req := &api.CreateSecretRequest{
+		Name: projectSecretCreateOpts.name,
+	}
+
+	log.Infof("creating project secret")
+	secret, _, err := gwclient.CreateProjectSecret(context.TODO(), url.PathEscape(projectSecretCreateOpts.projectID), req)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create project secret")
+	}
+	log.Infof("project secret %q created, ID: %q", secret.Name, secret.ID)
+
+	return nil
+}

cmd/agola/cmd/projectvariable.go

@@ -0,0 +1,28 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var cmdProjectVariable = &cobra.Command{
+	Use:   "variable",
+	Short: "variable",
+}
+
+func init() {
+	cmdUser.AddCommand(cmdProjectVariable)
+}

internal/services/configstore/api/api.go

@@ -16,7 +16,11 @@ package api
 
 import (
 	"net/http"
+	"net/url"
 
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/services/types"
 	"github.com/sorintlab/agola/internal/util"
 )
 
@@ -32,3 +36,24 @@ func httpError(w http.ResponseWriter, err error) bool {
 
 	return false
 }
+
+func GetConfigTypeRef(r *http.Request) (types.ConfigType, string, error) {
+	vars := mux.Vars(r)
+	projectRef, err := url.PathUnescape(vars["projectref"])
+	if err != nil {
+		return "", "", util.NewErrBadRequest(errors.Wrapf(err, "wrong projectref %q", vars["projectref"]))
+	}
+	if projectRef != "" {
+		return types.ConfigTypeProject, projectRef, nil
+	}
+
+	projectGroupRef, err := url.PathUnescape(vars["projectgroupref"])
+	if err != nil {
+		return "", "", util.NewErrBadRequest(errors.Wrapf(err, "wrong projectgroupref %q", vars["projectgroupref"]))
+	}
+	if projectGroupRef != "" {
+		return types.ConfigTypeProjectGroup, projectGroupRef, nil
+	}
+
+	return "", "", util.NewErrBadRequest(errors.Errorf("cannot get project or projectgroup ref"))
+}

internal/services/configstore/api/client.go

@@ -158,6 +158,110 @@ func (c *Client) DeleteProject(ctx context.Context, projectID string) (*http.Res
 	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s", url.PathEscape(projectID)), nil, jsonContent, nil)
 }
 
+func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectGroupRef string, tree bool) ([]*types.Secret, *http.Response, error) {
+	q := url.Values{}
+	if tree {
+		q.Add("tree", "")
+	}
+
+	secrets := []*types.Secret{}
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectGroupRef)), q, jsonContent, nil, &secrets)
+	return secrets, resp, err
+}
+
+func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree bool) ([]*types.Secret, *http.Response, error) {
+	q := url.Values{}
+	if tree {
+		q.Add("tree", "")
+	}
+
+	secrets := []*types.Secret{}
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/secrets", url.PathEscape(projectRef)), q, jsonContent, nil, &secrets)
+	return secrets, resp, err
+}
+
+func (c *Client) CreateProjectGroupSecret(ctx context.Context, projectGroupRef string, secret *types.Secret) (*types.Secret, *http.Response, error) {
+	pj, err := json.Marshal(secret)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secret = new(types.Secret)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectGroupRef)), nil, jsonContent, bytes.NewReader(pj), secret)
+	return secret, resp, err
+}
+
+func (c *Client) CreateProjectSecret(ctx context.Context, projectRef string, secret *types.Secret) (*types.Secret, *http.Response, error) {
+	pj, err := json.Marshal(secret)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secret = new(types.Secret)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/projects/%s/secrets", url.PathEscape(projectRef)), nil, jsonContent, bytes.NewReader(pj), secret)
+	return secret, resp, err
+}
+
+func (c *Client) DeleteProjectGroupSecret(ctx context.Context, projectGroupRef, secretName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projectgroups/%s/secrets/%s", url.PathEscape(projectGroupRef), secretName), nil, jsonContent, nil)
+}
+
+func (c *Client) DeleteProjectSecret(ctx context.Context, projectRef, secretName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s/secrets/%s", url.PathEscape(projectRef), secretName), nil, jsonContent, nil)
+}
+
+func (c *Client) GetProjectGroupVariables(ctx context.Context, projectGroupRef string, tree bool) ([]*types.Variable, *http.Response, error) {
+	q := url.Values{}
+	if tree {
+		q.Add("tree", "")
+	}
+
+	variables := []*types.Variable{}
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectGroupRef)), q, jsonContent, nil, &variables)
+	return variables, resp, err
+}
+
+func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tree bool) ([]*types.Variable, *http.Response, error) {
+	q := url.Values{}
+	if tree {
+		q.Add("tree", "")
+	}
+
+	variables := []*types.Variable{}
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), q, jsonContent, nil, &variables)
+	return variables, resp, err
+}
+
+func (c *Client) CreateProjectGroupVariable(ctx context.Context, projectGroupRef string, variable *types.Variable) (*types.Variable, *http.Response, error) {
+	pj, err := json.Marshal(variable)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	variable = new(types.Variable)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectGroupRef)), nil, jsonContent, bytes.NewReader(pj), variable)
+	return variable, resp, err
+}
+
+func (c *Client) CreateProjectVariable(ctx context.Context, projectRef string, variable *types.Variable) (*types.Variable, *http.Response, error) {
+	pj, err := json.Marshal(variable)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	variable = new(types.Variable)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), nil, jsonContent, bytes.NewReader(pj), variable)
+	return variable, resp, err
+}
+
+func (c *Client) DeleteProjectGroupVariable(ctx context.Context, projectGroupRef, variableName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projectgroups/%s/variables/%s", url.PathEscape(projectGroupRef), variableName), nil, jsonContent, nil)
+}
+
+func (c *Client) DeleteProjectVariable(ctx context.Context, projectRef, variableName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s/variables/%s", url.PathEscape(projectRef), variableName), nil, jsonContent, nil)
+}
+
 func (c *Client) GetUser(ctx context.Context, userID string) (*types.User, *http.Response, error) {
 	user := new(types.User)
 	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/user/%s", userID), nil, jsonContent, nil, user)

internal/services/configstore/api/secret.go

@@ -0,0 +1,181 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type SecretHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewSecretHandler(logger *zap.Logger, readDB *readdb.ReadDB) *SecretHandler {
+	return &SecretHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *SecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	secretID := vars["secretid"]
+
+	var secret *types.Secret
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		secret, err = h.readDB.GetSecretByID(tx, secretID)
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if secret == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(secret); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type SecretsHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewSecretsHandler(logger *zap.Logger, readDB *readdb.ReadDB) *SecretsHandler {
+	return &SecretsHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *SecretsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+
+	_, tree := query["tree"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var secrets []*types.Secret
+	err = h.readDB.Do(func(tx *db.Tx) error {
+		parentID, err := h.readDB.ResolveConfigID(tx, parentType, parentRef)
+		if err != nil {
+			return err
+		}
+		if tree {
+			secrets, err = h.readDB.GetSecretsTree(tx, parentType, parentID)
+		} else {
+			secrets, err = h.readDB.GetSecrets(tx, parentID)
+		}
+		// populate parent path
+		for _, s := range secrets {
+			pp, err := h.readDB.GetParentPath(tx, s.Parent.Type, s.Parent.ID)
+			if err != nil {
+				return err
+			}
+			s.Parent.Path = pp
+		}
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(secrets); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateSecretHandler struct {
+	log    *zap.SugaredLogger
+	ch     *command.CommandHandler
+	readDB *readdb.ReadDB
+}
+
+func NewCreateSecretHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateSecretHandler {
+	return &CreateSecretHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateSecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var secret *types.Secret
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&secret); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	secret.Parent.Type = parentType
+	secret.Parent.ID = parentRef
+
+	secret, err = h.ch.CreateSecret(ctx, secret)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(secret); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteSecretHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteSecretHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteSecretHandler {
+	return &DeleteSecretHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteSecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	secretName := vars["secretname"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	err = h.ch.DeleteSecret(ctx, parentType, parentRef, secretName)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+	}
+}

internal/services/configstore/api/variable.go

@@ -0,0 +1,146 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type VariablesHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewVariablesHandler(logger *zap.Logger, readDB *readdb.ReadDB) *VariablesHandler {
+	return &VariablesHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *VariablesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+
+	_, tree := query["tree"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var variables []*types.Variable
+	err = h.readDB.Do(func(tx *db.Tx) error {
+		parentID, err := h.readDB.ResolveConfigID(tx, parentType, parentRef)
+		if err != nil {
+			return err
+		}
+		if tree {
+			variables, err = h.readDB.GetVariablesTree(tx, parentType, parentID)
+		} else {
+			variables, err = h.readDB.GetVariables(tx, parentID)
+		}
+		// populate parent path
+		for _, v := range variables {
+			pp, err := h.readDB.GetParentPath(tx, v.Parent.Type, v.Parent.ID)
+			if err != nil {
+				return err
+			}
+			v.Parent.Path = pp
+		}
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(variables); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateVariableHandler struct {
+	log    *zap.SugaredLogger
+	ch     *command.CommandHandler
+	readDB *readdb.ReadDB
+}
+
+func NewCreateVariableHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateVariableHandler {
+	return &CreateVariableHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateVariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var variable *types.Variable
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&variable); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	variable.Parent.Type = parentType
+	variable.Parent.ID = parentRef
+
+	variable, err = h.ch.CreateVariable(ctx, variable)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(variable); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteVariableHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteVariableHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteVariableHandler {
+	return &DeleteVariableHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteVariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	variableName := vars["variablename"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	err = h.ch.DeleteVariable(ctx, parentType, parentRef, variableName)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+	}
+}

internal/services/configstore/command/command.go

@@ -822,3 +822,220 @@ func (s *CommandHandler) DeleteOrg(ctx context.Context, orgName string) error {
 	_, err = s.wal.WriteWal(ctx, actions, cgt)
 	return err
 }
+
+func (s *CommandHandler) CreateSecret(ctx context.Context, secret *types.Secret) (*types.Secret, error) {
+	if secret.Name == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("secret name required"))
+	}
+	if secret.Parent.Type == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("secret parent type required"))
+	}
+	if secret.Parent.ID == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("secret parentid required"))
+	}
+	if secret.Parent.Type != types.ConfigTypeProject && secret.Parent.Type != types.ConfigTypeProjectGroup {
+		return nil, util.NewErrBadRequest(errors.Errorf("invalid secret parent type %q", secret.Parent.Type))
+	}
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{secret.Name}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		parentID, err := s.readDB.ResolveConfigID(tx, secret.Parent.Type, secret.Parent.ID)
+		if err != nil {
+			return err
+		}
+		secret.Parent.ID = parentID
+
+		// check duplicate secret name
+		s, err := s.readDB.GetSecretByName(tx, secret.Parent.ID, secret.Name)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			return util.NewErrBadRequest(errors.Errorf("secret with name %q for %s with id %q already exists", secret.Name, secret.Parent.Type, secret.Parent.ID))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	secret.ID = uuid.NewV4().String()
+
+	secretj, err := json.Marshal(secret)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal secret")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageSecretFile(secret.ID),
+			Data:       secretj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return secret, err
+}
+
+func (s *CommandHandler) DeleteSecret(ctx context.Context, parentType types.ConfigType, parentRef, secretName string) error {
+	var secret *types.Secret
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		parentID, err := s.readDB.ResolveConfigID(tx, parentType, parentRef)
+		if err != nil {
+			return err
+		}
+
+		// check secret existance
+		secret, err = s.readDB.GetSecretByName(tx, parentID, secretName)
+		if err != nil {
+			return err
+		}
+		if secret == nil {
+			return util.NewErrBadRequest(errors.Errorf("secret with name %q doesn't exist", secretName))
+		}
+
+		cgNames := []string{secretName}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypeDelete,
+			Path:       common.StorageSecretFile(secret.ID),
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}
+
+func (s *CommandHandler) CreateVariable(ctx context.Context, variable *types.Variable) (*types.Variable, error) {
+	if variable.Name == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("variable name required"))
+	}
+	if variable.Parent.Type == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("variable parent type required"))
+	}
+	if variable.Parent.ID == "" {
+		return nil, util.NewErrBadRequest(errors.Errorf("variable parent id required"))
+	}
+	if variable.Parent.Type != types.ConfigTypeProject && variable.Parent.Type != types.ConfigTypeProjectGroup {
+		return nil, util.NewErrBadRequest(errors.Errorf("invalid variable parent type %q", variable.Parent.Type))
+	}
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{variable.Name}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		parentID, err := s.readDB.ResolveConfigID(tx, variable.Parent.Type, variable.Parent.ID)
+		if err != nil {
+			return err
+		}
+		variable.Parent.ID = parentID
+
+		// check duplicate variable name
+		s, err := s.readDB.GetVariableByName(tx, variable.Parent.ID, variable.Name)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			return util.NewErrBadRequest(errors.Errorf("variable with name %q for %s with id %q already exists", variable.Name, variable.Parent.Type, variable.Parent.ID))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	variable.ID = uuid.NewV4().String()
+
+	variablej, err := json.Marshal(variable)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal variable")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageVariableFile(variable.ID),
+			Data:       variablej,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return variable, err
+}
+
+func (s *CommandHandler) DeleteVariable(ctx context.Context, parentType types.ConfigType, parentRef, variableName string) error {
+	var variable *types.Variable
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		parentID, err := s.readDB.ResolveConfigID(tx, parentType, parentRef)
+		if err != nil {
+			return err
+		}
+
+		// check variable existance
+		variable, err = s.readDB.GetVariableByName(tx, parentID, variableName)
+		if err != nil {
+			return err
+		}
+		if variable == nil {
+			return util.NewErrBadRequest(errors.Errorf("variable with name %q doesn't exist", variableName))
+		}
+
+		cgNames := []string{variableName}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypeDelete,
+			Path:       common.StorageVariableFile(variable.ID),
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}

internal/services/configstore/common/common.go

@@ -31,6 +31,8 @@ var (
 	StorageProjectsDir      = path.Join(StorageDataDir, "projects")
 	StorageProjectGroupsDir = path.Join(StorageDataDir, "projectgroups")
 	StorageRemoteSourcesDir = path.Join(StorageDataDir, "remotesources")
+	StorageSecretsDir       = path.Join(StorageDataDir, "secrets")
+	StorageVariablesDir     = path.Join(StorageDataDir, "variables")
 )
 
 const (
@@ -57,6 +59,14 @@ func StorageRemoteSourceFile(userID string) string {
 	return path.Join(StorageRemoteSourcesDir, userID)
 }
 
+func StorageSecretFile(secretID string) string {
+	return path.Join(StorageSecretsDir, secretID)
+}
+
+func StorageVariableFile(variableID string) string {
+	return path.Join(StorageVariablesDir, variableID)
+}
+
 func PathToTypeID(p string) (types.ConfigType, string) {
 	var configType types.ConfigType
 	switch path.Dir(p) {
@@ -70,6 +80,10 @@ func PathToTypeID(p string) (types.ConfigType, string) {
 		configType = types.ConfigTypeProject
 	case StorageRemoteSourcesDir:
 		configType = types.ConfigTypeRemoteSource
+	case StorageSecretsDir:
+		configType = types.ConfigTypeSecret
+	case StorageVariablesDir:
+		configType = types.ConfigTypeVariable
 	default:
 		panic(fmt.Errorf("cannot determine configtype for path: %q", p))
 	}

internal/services/configstore/configstore.go

@@ -118,6 +118,14 @@ func (s *ConfigStore) Run(ctx context.Context) error {
 	createProjectHandler := api.NewCreateProjectHandler(logger, s.ch)
 	deleteProjectHandler := api.NewDeleteProjectHandler(logger, s.ch)
 
+	secretsHandler := api.NewSecretsHandler(logger, s.readDB)
+	createSecretHandler := api.NewCreateSecretHandler(logger, s.ch)
+	deleteSecretHandler := api.NewDeleteSecretHandler(logger, s.ch)
+
+	variablesHandler := api.NewVariablesHandler(logger, s.readDB)
+	createVariableHandler := api.NewCreateVariableHandler(logger, s.ch)
+	deleteVariableHandler := api.NewDeleteVariableHandler(logger, s.ch)
+
 	userHandler := api.NewUserHandler(logger, s.readDB)
 	usersHandler := api.NewUsersHandler(logger, s.readDB)
 	userByNameHandler := api.NewUserByNameHandler(logger, s.readDB)
@@ -152,7 +160,21 @@ func (s *ConfigStore) Run(ctx context.Context) error {
 
 	apirouter.Handle("/projects/{projectref}", projectHandler).Methods("GET")
 	apirouter.Handle("/projects", createProjectHandler).Methods("PUT")
-	apirouter.Handle("/projects/{projectid}", deleteProjectHandler).Methods("DELETE")
+	apirouter.Handle("/projects/{projectref}", deleteProjectHandler).Methods("DELETE")
+
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets", secretsHandler).Methods("GET")
+	apirouter.Handle("/projects/{projectref}/secrets", secretsHandler).Methods("GET")
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets", createSecretHandler).Methods("PUT")
+	apirouter.Handle("/projects/{projectref}/secrets", createSecretHandler).Methods("PUT")
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets/{secretname}", deleteSecretHandler).Methods("DELETE")
+	apirouter.Handle("/projects/{projectref}/secrets/{secretname}", deleteSecretHandler).Methods("DELETE")
+
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables", variablesHandler).Methods("GET")
+	apirouter.Handle("/projects/{projectref}/variables", variablesHandler).Methods("GET")
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables", createVariableHandler).Methods("PUT")
+	apirouter.Handle("/projects/{projectref}/variables", createVariableHandler).Methods("PUT")
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables/{variablename}", deleteVariableHandler).Methods("DELETE")
+	apirouter.Handle("/projects/{projectref}/variables/{variablename}", deleteVariableHandler).Methods("DELETE")
 
 	apirouter.Handle("/user/{userid}", userHandler).Methods("GET")
 	apirouter.Handle("/users", usersHandler).Methods("GET")

internal/services/configstore/readdb/create.go

@@ -44,9 +44,9 @@ var Stmts = []string{
 
 	"create table linkedaccount_project (id uuid, projectid uuid, PRIMARY KEY (id), FOREIGN KEY(projectid) REFERENCES user(id))",
 
-	"create table secret (id uuid, name varchar, containerid varchar, data bytea, PRIMARY KEY (id))",
+	"create table secret (id uuid, name varchar, parentid varchar, data bytea, PRIMARY KEY (id))",
 	"create index secret_name on secret(name)",
 
-	"create table variable (id uuid, name varchar, containerid varchar, data bytea, PRIMARY KEY (id))",
+	"create table variable (id uuid, name varchar, parentid varchar, data bytea, PRIMARY KEY (id))",
 	"create index variable_name on variable(name)",
 }

internal/services/configstore/readdb/parent.go

@@ -47,3 +47,35 @@ func (r *ReadDB) ResolveConfigID(tx *db.Tx, configType types.ConfigType, ref str
 		return "", util.NewErrBadRequest(errors.Errorf("unknown config type %q", configType))
 	}
 }
+
+func (r *ReadDB) GetParentPath(tx *db.Tx, parentType types.ConfigType, parentID string) (string, error) {
+	var p string
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		projectGroup, err := r.GetProjectGroup(tx, parentID)
+		if err != nil {
+			return "", err
+		}
+		if projectGroup == nil {
+			return "", errors.Errorf("projectgroup with id %q doesn't exist", parentID)
+		}
+		p, err = r.GetProjectGroupPath(tx, projectGroup)
+		if err != nil {
+			return "", err
+		}
+	case types.ConfigTypeProject:
+		project, err := r.GetProject(tx, parentID)
+		if err != nil {
+			return "", err
+		}
+		if project == nil {
+			return "", errors.Errorf("project with id %q doesn't exist", parentID)
+		}
+		p, err = r.GetProjectPath(tx, project)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return p, nil
+}

internal/services/configstore/readdb/readdb.go

@@ -641,6 +641,14 @@ func (r *ReadDB) applyAction(tx *db.Tx, action *wal.Action) error {
 			if err := r.insertRemoteSource(tx, action.Data); err != nil {
 				return err
 			}
+		case types.ConfigTypeSecret:
+			if err := r.insertSecret(tx, action.Data); err != nil {
+				return err
+			}
+		case types.ConfigTypeVariable:
+			if err := r.insertVariable(tx, action.Data); err != nil {
+				return err
+			}
 		}
 
 	case wal.ActionTypeDelete:
@@ -670,6 +678,16 @@ func (r *ReadDB) applyAction(tx *db.Tx, action *wal.Action) error {
 			if err := r.deleteRemoteSource(tx, ID); err != nil {
 				return err
 			}
+		case types.ConfigTypeSecret:
+			r.log.Debugf("deleting secret with id: %s", ID)
+			if err := r.deleteSecret(tx, ID); err != nil {
+				return err
+			}
+		case types.ConfigTypeVariable:
+			r.log.Debugf("deleting variable with id: %s", ID)
+			if err := r.deleteVariable(tx, ID); err != nil {
+				return err
+			}
 		}
 	}
 

internal/services/configstore/readdb/secret.go

@@ -0,0 +1,225 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+var (
+	secretSelect = sb.Select("id", "data").From("secret")
+	secretInsert = sb.Insert("secret").Columns("id", "name", "parentid", "data")
+)
+
+func (r *ReadDB) insertSecret(tx *db.Tx, data []byte) error {
+	secret := types.Secret{}
+	if err := json.Unmarshal(data, &secret); err != nil {
+		return errors.Wrap(err, "failed to unmarshal secret")
+	}
+	// poor man insert or update...
+	if err := r.deleteSecret(tx, secret.ID); err != nil {
+		return err
+	}
+	q, args, err := secretInsert.Values(secret.ID, secret.Name, secret.Parent.ID, data).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	_, err = tx.Exec(q, args...)
+	return errors.Wrap(err, "failed to insert secret")
+}
+
+func (r *ReadDB) deleteSecret(tx *db.Tx, id string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from secret where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete secret")
+	}
+	return nil
+}
+
+func (r *ReadDB) GetSecretByID(tx *db.Tx, secretID string) (*types.Secret, error) {
+	q, args, err := secretSelect.Where(sq.Eq{"id": secretID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	secrets, _, err := fetchSecrets(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(secrets) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(secrets) == 0 {
+		return nil, nil
+	}
+	return secrets[0], nil
+}
+
+func (r *ReadDB) GetSecretByName(tx *db.Tx, parentID, name string) (*types.Secret, error) {
+	q, args, err := secretSelect.Where(sq.Eq{"parentid": parentID, "name": name}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	secrets, _, err := fetchSecrets(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(secrets) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(secrets) == 0 {
+		return nil, nil
+	}
+	return secrets[0], nil
+}
+
+func (r *ReadDB) GetSecrets(tx *db.Tx, parentID string) ([]*types.Secret, error) {
+	q, args, err := secretSelect.Where(sq.Eq{"parentid": parentID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	secrets, _, err := fetchSecrets(tx, q, args...)
+	return secrets, err
+}
+
+func (r *ReadDB) GetSecretTree(tx *db.Tx, parentType types.ConfigType, parentID, name string) (*types.Secret, error) {
+	for parentType == types.ConfigTypeProjectGroup || parentType == types.ConfigTypeProject {
+		secret, err := r.GetSecretByName(tx, parentID, name)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get secret with name %q", name)
+		}
+		if secret != nil {
+			return secret, nil
+		}
+
+		switch parentType {
+		case types.ConfigTypeProjectGroup:
+			projectGroup, err := r.GetProjectGroup(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if projectGroup == nil {
+				return nil, errors.Errorf("projectgroup with id %q doesn't exist", parentID)
+			}
+			parentType = projectGroup.Parent.Type
+			parentID = projectGroup.Parent.ID
+		case types.ConfigTypeProject:
+			project, err := r.GetProject(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if project == nil {
+				return nil, errors.Errorf("project with id %q doesn't exist", parentID)
+			}
+			parentType = project.Parent.Type
+			parentID = project.Parent.ID
+		}
+	}
+
+	return nil, nil
+}
+
+func (r *ReadDB) GetSecretsTree(tx *db.Tx, parentType types.ConfigType, parentID string) ([]*types.Secret, error) {
+	allSecrets := []*types.Secret{}
+
+	for parentType == types.ConfigTypeProjectGroup || parentType == types.ConfigTypeProject {
+		secrets, err := r.GetSecrets(tx, parentID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get secrets for %s %q", parentType, parentID)
+		}
+		allSecrets = append(allSecrets, secrets...)
+
+		switch parentType {
+		case types.ConfigTypeProjectGroup:
+			projectGroup, err := r.GetProjectGroup(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if projectGroup == nil {
+				return nil, errors.Errorf("projectgroup with id %q doesn't exist", parentID)
+			}
+			parentType = projectGroup.Parent.Type
+			parentID = projectGroup.Parent.ID
+		case types.ConfigTypeProject:
+			project, err := r.GetProject(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if project == nil {
+				return nil, errors.Errorf("project with id %q doesn't exist", parentID)
+			}
+			parentType = project.Parent.Type
+			parentID = project.Parent.ID
+		}
+	}
+
+	return allSecrets, nil
+}
+
+func fetchSecrets(tx *db.Tx, q string, args ...interface{}) ([]*types.Secret, []string, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	return scanSecrets(rows)
+}
+
+func scanSecret(rows *sql.Rows, additionalFields ...interface{}) (*types.Secret, string, error) {
+	var id string
+	var data []byte
+	if err := rows.Scan(&id, &data); err != nil {
+		return nil, "", errors.Wrap(err, "failed to scan rows")
+	}
+	secret := types.Secret{}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &secret); err != nil {
+			return nil, "", errors.Wrap(err, "failed to unmarshal secret")
+		}
+	}
+
+	return &secret, id, nil
+}
+
+func scanSecrets(rows *sql.Rows) ([]*types.Secret, []string, error) {
+	secrets := []*types.Secret{}
+	ids := []string{}
+	for rows.Next() {
+		p, id, err := scanSecret(rows)
+		if err != nil {
+			rows.Close()
+			return nil, nil, err
+		}
+		secrets = append(secrets, p)
+		ids = append(ids, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return secrets, ids, nil
+}

internal/services/configstore/readdb/variable.go

@@ -0,0 +1,188 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+var (
+	variableSelect = sb.Select("id", "data").From("variable")
+	variableInsert = sb.Insert("variable").Columns("id", "name", "parentid", "data")
+)
+
+func (r *ReadDB) insertVariable(tx *db.Tx, data []byte) error {
+	variable := types.Variable{}
+	if err := json.Unmarshal(data, &variable); err != nil {
+		return errors.Wrap(err, "failed to unmarshal variable")
+	}
+	// poor man insert or update...
+	if err := r.deleteVariable(tx, variable.ID); err != nil {
+		return err
+	}
+	q, args, err := variableInsert.Values(variable.ID, variable.Name, variable.Parent.ID, data).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	_, err = tx.Exec(q, args...)
+	return errors.Wrap(err, "failed to insert variable")
+}
+
+func (r *ReadDB) deleteVariable(tx *db.Tx, id string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from variable where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete variable")
+	}
+	return nil
+}
+
+func (r *ReadDB) GetVariableByID(tx *db.Tx, variableID string) (*types.Variable, error) {
+	q, args, err := variableSelect.Where(sq.Eq{"id": variableID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	variables, _, err := fetchVariables(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(variables) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(variables) == 0 {
+		return nil, nil
+	}
+	return variables[0], nil
+}
+
+func (r *ReadDB) GetVariableByName(tx *db.Tx, parentID, name string) (*types.Variable, error) {
+	q, args, err := variableSelect.Where(sq.Eq{"parentid": parentID, "name": name}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	variables, _, err := fetchVariables(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(variables) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(variables) == 0 {
+		return nil, nil
+	}
+	return variables[0], nil
+}
+
+func (r *ReadDB) GetVariables(tx *db.Tx, parentID string) ([]*types.Variable, error) {
+	q, args, err := variableSelect.Where(sq.Eq{"parentid": parentID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	variables, _, err := fetchVariables(tx, q, args...)
+	return variables, err
+}
+
+func (r *ReadDB) GetVariablesTree(tx *db.Tx, parentType types.ConfigType, parentID string) ([]*types.Variable, error) {
+	allVariables := []*types.Variable{}
+
+	for parentType == types.ConfigTypeProjectGroup || parentType == types.ConfigTypeProject {
+		vars, err := r.GetVariables(tx, parentID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get variables for %s %q", parentType, parentID)
+		}
+		allVariables = append(allVariables, vars...)
+
+		switch parentType {
+		case types.ConfigTypeProjectGroup:
+			projectGroup, err := r.GetProjectGroup(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if projectGroup == nil {
+				return nil, errors.Errorf("projectgroup with id %q doesn't exist", parentID)
+			}
+			parentType = projectGroup.Parent.Type
+			parentID = projectGroup.Parent.ID
+		case types.ConfigTypeProject:
+			project, err := r.GetProject(tx, parentID)
+			if err != nil {
+				return nil, err
+			}
+			if project == nil {
+				return nil, errors.Errorf("project with id %q doesn't exist", parentID)
+			}
+			parentType = project.Parent.Type
+			parentID = project.Parent.ID
+		}
+	}
+
+	return allVariables, nil
+}
+
+func fetchVariables(tx *db.Tx, q string, args ...interface{}) ([]*types.Variable, []string, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	return scanVariables(rows)
+}
+
+func scanVariable(rows *sql.Rows, additionalFields ...interface{}) (*types.Variable, string, error) {
+	var id string
+	var data []byte
+	if err := rows.Scan(&id, &data); err != nil {
+		return nil, "", errors.Wrap(err, "failed to scan rows")
+	}
+	variable := types.Variable{}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &variable); err != nil {
+			return nil, "", errors.Wrap(err, "failed to unmarshal variable")
+		}
+	}
+
+	return &variable, id, nil
+}
+
+func scanVariables(rows *sql.Rows) ([]*types.Variable, []string, error) {
+	variables := []*types.Variable{}
+	ids := []string{}
+	for rows.Next() {
+		p, id, err := scanVariable(rows)
+		if err != nil {
+			rows.Close()
+			return nil, nil, err
+		}
+		variables = append(variables, p)
+		ids = append(ids, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return variables, ids, nil
+}

internal/services/gateway/api/api.go

@@ -16,7 +16,11 @@ package api
 
 import (
 	"net/http"
+	"net/url"
 
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/services/types"
 	"github.com/sorintlab/agola/internal/util"
 )
 
@@ -32,3 +36,24 @@ func httpError(w http.ResponseWriter, err error) bool {
 
 	return false
 }
+
+func GetConfigTypeRef(r *http.Request) (types.ConfigType, string, error) {
+	vars := mux.Vars(r)
+	projectRef, err := url.PathUnescape(vars["projectref"])
+	if err != nil {
+		return "", "", util.NewErrBadRequest(errors.Wrapf(err, "wrong projectref %q", vars["projectref"]))
+	}
+	if projectRef != "" {
+		return types.ConfigTypeProject, projectRef, nil
+	}
+
+	projectGroupRef, err := url.PathUnescape(vars["projectgroupref"])
+	if err != nil {
+		return "", "", util.NewErrBadRequest(errors.Wrapf(err, "wrong projectgroupref %q", vars["projectgroupref"]))
+	}
+	if projectGroupRef != "" {
+		return types.ConfigTypeProjectGroup, projectGroupRef, nil
+	}
+
+	return "", "", util.NewErrBadRequest(errors.Errorf("cannot get project or projectgroup ref"))
+}

internal/services/gateway/api/client.go

@@ -159,6 +159,47 @@ func (c *Client) CreateProject(ctx context.Context, req *CreateProjectRequest) (
 	return project, resp, err
 }
 
+func (c *Client) CreateProjectGroupSecret(ctx context.Context, projectGroupID string, req *CreateSecretRequest) (*SecretResponse, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secret := new(SecretResponse)
+	resp, err := c.getParsedResponse(ctx, "PUT", path.Join("/projectgroups", projectGroupID, "secrets"), nil, jsonContent, bytes.NewReader(reqj), secret)
+	return secret, resp, err
+}
+
+func (c *Client) CreateProjectSecret(ctx context.Context, projectID string, req *CreateSecretRequest) (*SecretResponse, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secret := new(SecretResponse)
+	resp, err := c.getParsedResponse(ctx, "PUT", path.Join("/projects", projectID, "secrets"), nil, jsonContent, bytes.NewReader(reqj), secret)
+	return secret, resp, err
+}
+
+func (c *Client) createSecret(ctx context.Context, containertype, containerid string, req *CreateSecretRequest) (*SecretResponse, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var basepath string
+	switch containertype {
+	case "project":
+		basepath = "projects"
+	default:
+		return nil, nil, fmt.Errorf("invalid container type")
+	}
+
+	secret := new(SecretResponse)
+	resp, err := c.getParsedResponse(ctx, "PUT", path.Join("/", basepath, containerid, "secrets"), nil, jsonContent, bytes.NewReader(reqj), secret)
+	return secret, resp, err
+}
+
 func (c *Client) DeleteCurrentUserProject(ctx context.Context, projectName string) (*http.Response, error) {
 	return c.deleteProject(ctx, "user", "", projectName)
 }

internal/services/gateway/api/secret.go

@@ -0,0 +1,191 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	csapi "github.com/sorintlab/agola/internal/services/configstore/api"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+	"go.uber.org/zap"
+
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+)
+
+type SecretResponse struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	ParentPath string `json:"parent_path"`
+}
+
+func createSecretResponse(s *types.Secret) *SecretResponse {
+	return &SecretResponse{
+		ID:         s.ID,
+		Name:       s.Name,
+		ParentPath: s.Parent.Path,
+	}
+}
+
+type SecretHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewSecretHandler(logger *zap.Logger, configstoreClient *csapi.Client) *SecretHandler {
+	return &SecretHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *SecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	query := r.URL.Query()
+	_, tree := query["tree"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var cssecrets []*types.Secret
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, parentRef, tree)
+	case types.ConfigTypeProject:
+		cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, parentRef, tree)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	secrets := make([]*SecretResponse, len(cssecrets))
+	for i, s := range cssecrets {
+		secrets[i] = createSecretResponse(s)
+	}
+
+	if err := json.NewEncoder(w).Encode(secrets); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateSecretRequest struct {
+	Name string `json:"name,omitempty"`
+
+	Type types.SecretType `json:"type,omitempty"`
+
+	// internal secret
+	Data map[string]string `json:"data,omitempty"`
+
+	// external secret
+	SecretProviderID string `json:"secret_provider_id,omitempty"`
+	Path             string `json:"path,omitempty"`
+}
+
+type CreateSecretHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewCreateSecretHandler(logger *zap.Logger, configstoreClient *csapi.Client) *CreateSecretHandler {
+	return &CreateSecretHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *CreateSecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		return
+	}
+
+	var req CreateSecretRequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if !util.ValidateName(req.Name) {
+		err := errors.Errorf("invalid secret name %q", req.Name)
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	s := &types.Secret{
+		Name: req.Name,
+	}
+
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		h.log.Infof("creating project group secret")
+		s, _, err = h.configstoreClient.CreateProjectGroupSecret(ctx, parentRef, s)
+	case types.ConfigTypeProject:
+		h.log.Infof("creating project secret")
+		s, _, err = h.configstoreClient.CreateProjectSecret(ctx, parentRef, s)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	h.log.Infof("secret %s created, ID: %s", s.Name, s.ID)
+
+	res := createSecretResponse(s)
+
+	if err := json.NewEncoder(w).Encode(res); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteSecretHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewDeleteSecretHandler(logger *zap.Logger, configstoreClient *csapi.Client) *DeleteSecretHandler {
+	return &DeleteSecretHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *DeleteSecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	secretName := vars["secretname"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		return
+	}
+
+	var resp *http.Response
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		h.log.Infof("deleting project group secret")
+		resp, err = h.configstoreClient.DeleteProjectGroupSecret(ctx, parentRef, secretName)
+	case types.ConfigTypeProject:
+		h.log.Infof("deleting project secret")
+		resp, err = h.configstoreClient.DeleteProjectSecret(ctx, parentRef, secretName)
+	}
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

internal/services/gateway/api/variable.go

@@ -0,0 +1,259 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	csapi "github.com/sorintlab/agola/internal/services/configstore/api"
+	"github.com/sorintlab/agola/internal/services/gateway/common"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+	"go.uber.org/zap"
+
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+)
+
+type VariableValue struct {
+	SecretName               string `json:"secret_name"`
+	SecretVar                string `json:"secret_var"`
+	MatchingSecretParentPath string `json:"matching_secret_parent_path"`
+
+	When *types.When `json:"when"`
+}
+
+type VariableResponse struct {
+	ID         string          `json:"id"`
+	Name       string          `json:"name"`
+	Values     []VariableValue `json:"values"`
+	ParentPath string          `json:"parent_path"`
+}
+
+func createVariableResponse(v *types.Variable, secrets []*types.Secret) *VariableResponse {
+	nv := &VariableResponse{
+		ID:         v.ID,
+		Name:       v.Name,
+		Values:     make([]VariableValue, len(v.Values)),
+		ParentPath: v.Parent.Path,
+	}
+
+	for i, varvalue := range v.Values {
+		nv.Values[i] = VariableValue{
+			SecretName: varvalue.SecretName,
+			SecretVar:  varvalue.SecretVar,
+			When:       varvalue.When,
+		}
+		// get matching secret for var value
+		secret := common.GetVarValueMatchingSecret(varvalue, v.Parent.Path, secrets)
+		if secret != nil {
+			nv.Values[i].MatchingSecretParentPath = secret.Parent.Path
+		}
+	}
+
+	return nv
+}
+
+type VariableHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewVariableHandler(logger *zap.Logger, configstoreClient *csapi.Client) *VariableHandler {
+	return &VariableHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *VariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	query := r.URL.Query()
+	_, tree := query["tree"]
+	_, removeoverriden := query["removeoverriden"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var csvars []*types.Variable
+	var cssecrets []*types.Secret
+
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		var err error
+		csvars, _, err = h.configstoreClient.GetProjectGroupVariables(ctx, parentRef, tree)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, parentRef, tree)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+	case types.ConfigTypeProject:
+		var err error
+		csvars, _, err = h.configstoreClient.GetProjectVariables(ctx, parentRef, tree)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, parentRef, tree)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if removeoverriden {
+		// remove overriden variables
+		csvars = common.FilterOverridenVariables(csvars)
+	}
+
+	variables := make([]*VariableResponse, len(csvars))
+	for i, v := range csvars {
+		variables[i] = createVariableResponse(v, cssecrets)
+	}
+
+	if err := json.NewEncoder(w).Encode(variables); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateVariableRequest struct {
+	Name string `json:"name,omitempty"`
+
+	SecretName string `json:"secret_name,omitempty"`
+	SecretVar  string `json:"secret_var,omitempty"`
+
+	When *types.When `json:"when,omitempty"`
+}
+
+type CreateVariableHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewCreateVariableHandler(logger *zap.Logger, configstoreClient *csapi.Client) *CreateVariableHandler {
+	return &CreateVariableHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *CreateVariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	var req CreateVariableRequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if !util.ValidateName(req.Name) {
+		err := errors.Errorf("invalid variable name %q", req.Name)
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	v := &types.Variable{
+		Name: req.Name,
+		Parent: types.Parent{
+			Type: parentType,
+			ID:   parentRef,
+		},
+	}
+
+	var cssecrets []*types.Secret
+
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		var err error
+		cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, parentRef, true)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		h.log.Infof("creating project group variable")
+		v, _, err = h.configstoreClient.CreateProjectGroupVariable(ctx, parentRef, v)
+	case types.ConfigTypeProject:
+		cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, parentRef, true)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		h.log.Infof("creating project variable")
+		v, _, err = h.configstoreClient.CreateProjectVariable(ctx, parentRef, v)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	h.log.Infof("variable %s created, ID: %s", v.Name, v.ID)
+
+	res := createVariableResponse(v, cssecrets)
+
+	if err := json.NewEncoder(w).Encode(res); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteVariableHandler struct {
+	log               *zap.SugaredLogger
+	configstoreClient *csapi.Client
+}
+
+func NewDeleteVariableHandler(logger *zap.Logger, configstoreClient *csapi.Client) *DeleteVariableHandler {
+	return &DeleteVariableHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
+}
+
+func (h *DeleteVariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	variableName := vars["variablename"]
+
+	parentType, parentRef, err := GetConfigTypeRef(r)
+	if httpError(w, err) {
+		return
+	}
+
+	var resp *http.Response
+	switch parentType {
+	case types.ConfigTypeProjectGroup:
+		h.log.Infof("deleting project group variable")
+		resp, err = h.configstoreClient.DeleteProjectGroupVariable(ctx, parentRef, variableName)
+	case types.ConfigTypeProject:
+		h.log.Infof("deleting project variable")
+		resp, err = h.configstoreClient.DeleteProjectVariable(ctx, parentRef, variableName)
+	}
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

internal/services/gateway/common/variables.go

@@ -0,0 +1,61 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"strings"
+
+	"github.com/sorintlab/agola/internal/services/types"
+)
+
+func FilterOverridenVariables(variables []*types.Variable) []*types.Variable {
+	variablesMap := map[string]*types.Variable{}
+	for _, v := range variables {
+		if _, ok := variablesMap[v.Name]; !ok {
+			variablesMap[v.Name] = v
+		}
+	}
+
+	filteredVariables := make([]*types.Variable, len(variablesMap))
+	i := 0
+	// keep the original order
+	for _, v := range variables {
+		if _, ok := variablesMap[v.Name]; !ok {
+			continue
+		}
+		filteredVariables[i] = v
+		delete(variablesMap, v.Name)
+		i++
+	}
+
+	return filteredVariables
+}
+
+func GetVarValueMatchingSecret(varval types.VariableValue, varParentPath string, secrets []*types.Secret) *types.Secret {
+	// get the secret value referenced by the variable, it must be a secret at the same level or a lower level
+	var secret *types.Secret
+	for _, s := range secrets {
+		// we assume the root path will be the same
+		if s.Name != varval.SecretName {
+			continue
+		}
+		if strings.Contains(varParentPath, s.Parent.Path) {
+			secret = s
+			break
+		}
+	}
+
+	return secret
+}

internal/services/gateway/common/variables_test.go

@@ -0,0 +1,283 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/sorintlab/agola/internal/services/types"
+)
+
+func TestFilterOverridenVariables(t *testing.T) {
+	tests := []struct {
+		name      string
+		variables []*types.Variable
+		out       []*types.Variable
+	}{
+		{
+			name:      "test empty variables",
+			variables: []*types.Variable{},
+			out:       []*types.Variable{},
+		},
+		{
+			name: "test variable overrides",
+			variables: []*types.Variable{
+				// variables must be in depth (from leaves to root) order as returned by the
+				// configstore apis
+				&types.Variable{
+					Name: "var04",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup02/projectgroup03/project02",
+					},
+				},
+				&types.Variable{
+					Name: "var03",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/project01",
+					},
+				},
+				&types.Variable{
+					Name: "var02",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/project01",
+					},
+				},
+				&types.Variable{
+					Name: "var02",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01",
+					},
+				},
+				&types.Variable{
+					Name: "var01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01",
+					},
+				},
+				&types.Variable{
+					Name: "var01",
+					Parent: types.Parent{
+						Path: "org/org01",
+					},
+				},
+			},
+			out: []*types.Variable{
+				&types.Variable{
+					Name: "var04",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup02/projectgroup03/project02",
+					},
+				},
+				&types.Variable{
+					Name: "var03",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/project01",
+					},
+				},
+				&types.Variable{
+					Name: "var02",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/project01",
+					},
+				},
+				&types.Variable{
+					Name: "var01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out := FilterOverridenVariables(tt.variables)
+
+			if diff := cmp.Diff(tt.out, out); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}
+
+func TestGetVarValueMatchingSecret(t *testing.T) {
+	tests := []struct {
+		name          string
+		varValue      types.VariableValue
+		varParentPath string
+		secrets       []*types.Secret
+		out           *types.Secret
+	}{
+		{
+			name: "test empty secrets",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/project01",
+			secrets:       []*types.Secret{},
+			out:           nil,
+		},
+		{
+			name: "test secret with different name",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				&types.Secret{
+					Name: "secret02",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02",
+					},
+				},
+			},
+			out: nil,
+		},
+		{
+			name: "test secret with tree",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				&types.Secret{
+					Name: "secret02",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup03",
+					},
+				},
+			},
+			out: nil,
+		},
+		{
+			name: "test secret in child of variable parent",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02/project01",
+					},
+				},
+			},
+			out: nil,
+		},
+		{
+			name: "test secret in same parent and also child of variable parent",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02/project01",
+					},
+				},
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02",
+					},
+				},
+			},
+			out: &types.Secret{
+				Name: "secret01",
+				Parent: types.Parent{
+					Path: "org/org01/projectgroup01/projectgroup02",
+				},
+			},
+		},
+		{
+			name: "test secret in parent",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01",
+					},
+				},
+			},
+			out: &types.Secret{
+				Name: "secret01",
+				Parent: types.Parent{
+					Path: "org/org01/projectgroup01",
+				},
+			},
+		},
+		{
+			name: "test multiple secrets in same branch and also child of variable parent",
+			varValue: types.VariableValue{
+				SecretName: "secret01",
+				SecretVar:  "secretvar01",
+			},
+			varParentPath: "org/org01/projectgroup01/projectgroup02",
+			secrets: []*types.Secret{
+				// secrets must be in depth (from leaves to root) order as returned by the
+				// configstore apis
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02/project01",
+					},
+				},
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01/projectgroup02",
+					},
+				},
+				&types.Secret{
+					Name: "secret01",
+					Parent: types.Parent{
+						Path: "org/org01/projectgroup01",
+					},
+				},
+			},
+			out: &types.Secret{
+				Name: "secret01",
+				Parent: types.Parent{
+					Path: "org/org01/projectgroup01/projectgroup02",
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out := GetVarValueMatchingSecret(tt.varValue, tt.varParentPath, tt.secrets)
+
+			if diff := cmp.Diff(tt.out, out); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}

internal/services/gateway/gateway.go

@@ -153,6 +153,14 @@ func (g *Gateway) Run(ctx context.Context) error {
 	deleteProjectHandler := api.NewDeleteProjectHandler(logger, g.configstoreClient)
 	projectReconfigHandler := api.NewProjectReconfigHandler(logger, g.ch, g.configstoreClient, g.c.APIExposedURL)
 
+	secretHandler := api.NewSecretHandler(logger, g.configstoreClient)
+	createSecretHandler := api.NewCreateSecretHandler(logger, g.configstoreClient)
+	deleteSecretHandler := api.NewDeleteSecretHandler(logger, g.configstoreClient)
+
+	variableHandler := api.NewVariableHandler(logger, g.configstoreClient)
+	createVariableHandler := api.NewCreateVariableHandler(logger, g.configstoreClient)
+	deleteVariableHandler := api.NewDeleteVariableHandler(logger, g.configstoreClient)
+
 	currentUserHandler := api.NewCurrentUserHandler(logger, g.configstoreClient)
 	userHandler := api.NewUserHandler(logger, g.configstoreClient)
 	userByNameHandler := api.NewUserByNameHandler(logger, g.configstoreClient)
@@ -209,6 +217,20 @@ func (g *Gateway) Run(ctx context.Context) error {
 	apirouter.Handle("/projects/{projectid}", authForcedHandler(deleteProjectHandler)).Methods("DELETE")
 	apirouter.Handle("/projects/{projectid}/reconfig", authForcedHandler(projectReconfigHandler)).Methods("POST")
 
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets", authForcedHandler(secretHandler)).Methods("GET")
+	apirouter.Handle("/projects/{projectref}/secrets", authForcedHandler(secretHandler)).Methods("GET")
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets", authForcedHandler(createSecretHandler)).Methods("PUT")
+	apirouter.Handle("/projects/{projectref}/secrets", authForcedHandler(createSecretHandler)).Methods("PUT")
+	apirouter.Handle("/projectgroups/{projectgroupref}/secrets/{secretname}", authForcedHandler(deleteSecretHandler)).Methods("DELETE")
+	apirouter.Handle("/projects/{projectref}/secrets/{secretname}", authForcedHandler(deleteSecretHandler)).Methods("DELETE")
+
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables", authForcedHandler(variableHandler)).Methods("GET")
+	apirouter.Handle("/projects/{projectref}/variables", authForcedHandler(variableHandler)).Methods("GET")
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables", authForcedHandler(createVariableHandler)).Methods("PUT")
+	apirouter.Handle("/projects/{projectref}/variables", authForcedHandler(createVariableHandler)).Methods("PUT")
+	apirouter.Handle("/projectgroups/{projectgroupref}/variables/{variablename}", authForcedHandler(deleteVariableHandler)).Methods("DELETE")
+	apirouter.Handle("/projects/{projectref}/variables/{variablename}", authForcedHandler(deleteVariableHandler)).Methods("DELETE")
+
 	apirouter.Handle("/user", authForcedHandler(currentUserHandler)).Methods("GET")
 	apirouter.Handle("/user/{userid}", authForcedHandler(userHandler)).Methods("GET")
 	apirouter.Handle("/users", authForcedHandler(usersHandler)).Methods("GET")

internal/services/types/types.go

@@ -29,11 +29,16 @@ const (
 	ConfigTypeProjectGroup ConfigType = "projectgroup"
 	ConfigTypeProject      ConfigType = "project"
 	ConfigTypeRemoteSource ConfigType = "remotesource"
+	ConfigTypeSecret       ConfigType = "secret"
+	ConfigTypeVariable     ConfigType = "variable"
 )
 
 type Parent struct {
 	Type ConfigType `json:"type,omitempty"`
 	ID   string     `json:"id,omitempty"`
+
+	// fields the is only used in api response and shoukd be empty when saved in the store
+	Path string `json:"path,omitempty"`
 }
 
 type User struct {
@@ -151,6 +156,53 @@ type Project struct {
 	SkipSSHHostKeyCheck bool `json:"skip_ssh_host_key_check,omitempty"`
 }
 
+type SecretType string
+
+const (
+	SecretTypeInternal SecretType = "internal"
+	SecretTypeExternal SecretType = "external"
+)
+
+type SecretProviderType string
+
+const (
+	// TODO(sgotti) unimplemented
+	SecretProviderK8s   SecretProviderType = "k8s"
+	SecretProviderVault SecretProviderType = "vault"
+)
+
+type Secret struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name,omitempty"`
+
+	Parent Parent `json:"parent,omitempty"`
+
+	Type SecretType `json:"type,omitempty"`
+
+	// internal secret
+	Data map[string]string `json:"data,omitempty"`
+
+	// external secret
+	SecretProviderID string `json:"secret_provider_id,omitempty"`
+	Path             string `json:"path,omitempty"`
+}
+
+type Variable struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name,omitempty"`
+
+	Parent Parent `json:"parent,omitempty"`
+
+	Values []VariableValue `json:"values,omitempty"`
+}
+
+type VariableValue struct {
+	SecretName string `json:"secret_name,omitempty"`
+	SecretVar  string `json:"secret_var,omitempty"`
+
+	When *When `json:"when,omitempty"`
+}
+
 type When struct {
 	Branch *WhenConditions `json:"branch,omitempty"`
 	Tag    *WhenConditions `json:"tag,omitempty"`
@@ -170,8 +222,8 @@ const (
 )
 
 type WhenCondition struct {
-	Type  WhenConditionType
+	Type  WhenConditionType `json:"type,omitempty"`
-	Match string
+	Match string            `json:"match,omitempty"`
 }
 
 func MatchWhen(when *When, branch, tag, ref string) bool {