configstore: initial implementation

cmd/agola/cmd/serve.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/sorintlab/agola/cmd"
 	"github.com/sorintlab/agola/internal/services/config"
+	"github.com/sorintlab/agola/internal/services/configstore"
 	"github.com/sorintlab/agola/internal/services/runservice/executor"
 	rsscheduler "github.com/sorintlab/agola/internal/services/runservice/scheduler"
 	"github.com/sorintlab/agola/internal/services/scheduler"
@@ -116,6 +117,11 @@ func serve(cmd *cobra.Command, args []string) error {
 		return errors.Wrapf(err, "failed to start run service executor")
 	}
 
+	cs, err := configstore.NewConfigStore(ctx, &c.ConfigStore)
+	if err != nil {
+		return errors.Wrapf(err, "failed to start config store")
+	}
+
 	sched1, err := scheduler.NewScheduler(&c.Scheduler)
 	if err != nil {
 		return errors.Wrapf(err, "failed to start scheduler")
@@ -125,6 +131,7 @@ func serve(cmd *cobra.Command, args []string) error {
 
 	go func() { errCh <- rsex1.Run(ctx) }()
 	go func() { errCh <- rssched1.Run(ctx) }()
+	go func() { errCh <- cs.Run(ctx) }()
 	go func() { errCh <- sched1.Run(ctx) }()
 
 	return <-errCh

go.mod

@@ -11,8 +11,7 @@ require (
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.3.3 // indirect
-	github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 // indirect
+	github.com/go-bindata/go-bindata v1.0.0
-	github.com/go-bindata/go-bindata v1.0.0 // indirect
 	github.com/go-ini/ini v1.42.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/google/go-cmp v0.3.0
@@ -41,7 +40,7 @@ require (
 	github.com/spf13/cobra v0.0.3
 	go.etcd.io/etcd v0.0.0-20181128220305-dedae6eb7c25
 	go.uber.org/zap v1.9.1
-	golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
+	golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 // indirect
 	golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e // indirect
 	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
 	google.golang.org/appengine v1.4.0 // indirect

go.sum

@@ -32,7 +32,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76/go.mod h1:KjxHHirfLaw19iGT70HvVjHQsL1vq1SRQB4yOsAfy2s=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -42,7 +41,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M=
 github.com/go-bindata/go-bindata v1.0.0/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
-github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
 github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=

internal/services/configstore/api/client.go

@@ -0,0 +1,318 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/pkg/errors"
+)
+
+var jsonContent = http.Header{"content-type": []string{"application/json"}}
+
+// Client represents a Gogs API client.
+type Client struct {
+	url    string
+	client *http.Client
+}
+
+// NewClient initializes and returns a API client.
+func NewClient(url string) *Client {
+	return &Client{
+		url:    strings.TrimSuffix(url, "/"),
+		client: &http.Client{},
+	}
+}
+
+// SetHTTPClient replaces default http.Client with user given one.
+func (c *Client) SetHTTPClient(client *http.Client) {
+	c.client = client
+}
+
+func (c *Client) doRequest(ctx context.Context, method, path string, query url.Values, header http.Header, ibody io.Reader) (*http.Response, error) {
+	u, err := url.Parse(c.url + "/api/v1alpha" + path)
+	if err != nil {
+		return nil, err
+	}
+	u.RawQuery = query.Encode()
+
+	req, err := http.NewRequest(method, u.String(), ibody)
+	req = req.WithContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range header {
+		req.Header[k] = v
+	}
+
+	return c.client.Do(req)
+}
+
+func (c *Client) getResponse(ctx context.Context, method, path string, query url.Values, header http.Header, ibody io.Reader) (*http.Response, error) {
+	resp, err := c.doRequest(ctx, method, path, query, header, ibody)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode/100 != 2 {
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(data) <= 1 {
+			return resp, errors.New(resp.Status)
+		}
+
+		// TODO(sgotti) use a json error response
+
+		return resp, errors.New(string(data))
+	}
+
+	return resp, nil
+}
+
+func (c *Client) getParsedResponse(ctx context.Context, method, path string, query url.Values, header http.Header, ibody io.Reader, obj interface{}) (*http.Response, error) {
+	resp, err := c.getResponse(ctx, method, path, query, header, ibody)
+	if err != nil {
+		return resp, err
+	}
+	defer resp.Body.Close()
+
+	d := json.NewDecoder(resp.Body)
+
+	return resp, d.Decode(obj)
+}
+
+func (c *Client) GetProject(ctx context.Context, projectID string) (*types.Project, *http.Response, error) {
+	project := new(types.Project)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/project/%s", projectID), nil, jsonContent, nil, project)
+	return project, resp, err
+}
+
+func (c *Client) GetProjectByName(ctx context.Context, projectName string) (*types.Project, *http.Response, error) {
+	project := new(types.Project)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s", projectName), nil, jsonContent, nil, project)
+	return project, resp, err
+}
+
+func (c *Client) CreateProject(ctx context.Context, project *types.Project) (*types.Project, *http.Response, error) {
+	pj, err := json.Marshal(project)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	project = new(types.Project)
+	resp, err := c.getParsedResponse(ctx, "PUT", "/projects", nil, jsonContent, bytes.NewReader(pj), project)
+	return project, resp, err
+}
+
+func (c *Client) DeleteProject(ctx context.Context, projectName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s", projectName), nil, jsonContent, nil)
+}
+
+func (c *Client) GetProjects(ctx context.Context, start string, limit int, asc bool) ([]*types.Project, *http.Response, error) {
+	q := url.Values{}
+	if start != "" {
+		q.Add("start", start)
+	}
+	if limit > 0 {
+		q.Add("limit", strconv.Itoa(limit))
+	}
+	if asc {
+		q.Add("asc", "")
+	}
+
+	projects := []*types.Project{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/projects", q, jsonContent, nil, &projects)
+	return projects, resp, err
+}
+
+func (c *Client) GetUser(ctx context.Context, userID string) (*types.User, *http.Response, error) {
+	user := new(types.User)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/user/%s", userID), nil, jsonContent, nil, user)
+	return user, resp, err
+}
+
+func (c *Client) GetUserByName(ctx context.Context, userName string) (*types.User, *http.Response, error) {
+	user := new(types.User)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/users/%s", userName), nil, jsonContent, nil, user)
+	return user, resp, err
+}
+
+func (c *Client) GetUserByToken(ctx context.Context, token string) (*types.User, *http.Response, error) {
+	q := url.Values{}
+	q.Add("query_type", "bytoken")
+	q.Add("token", token)
+
+	users := []*types.User{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users)
+	if err != nil {
+		return nil, resp, err
+	}
+	return users[0], resp, err
+}
+
+func (c *Client) GetUserByLinkedAccountRemoteUserAndSource(ctx context.Context, remoteUserID, remoteSourceID string) (*types.User, *http.Response, error) {
+	q := url.Values{}
+	q.Add("query_type", "byremoteuser")
+	q.Add("remoteuserid", remoteUserID)
+	q.Add("remotesourceid", remoteSourceID)
+
+	users := []*types.User{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users)
+	if err != nil {
+		return nil, resp, err
+	}
+	return users[0], resp, err
+}
+
+func (c *Client) GetUserByLinkedAccount(ctx context.Context, linkedAccountID string) (*types.User, *http.Response, error) {
+	q := url.Values{}
+	q.Add("query_type", "bylinkedaccount")
+	q.Add("linkedaccountid", linkedAccountID)
+
+	users := []*types.User{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users)
+	if err != nil {
+		return nil, resp, err
+	}
+	return users[0], resp, err
+}
+
+func (c *Client) CreateUser(ctx context.Context, user *types.User) (*types.User, *http.Response, error) {
+	uj, err := json.Marshal(user)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	user = new(types.User)
+	resp, err := c.getParsedResponse(ctx, "PUT", "/users", nil, jsonContent, bytes.NewReader(uj), user)
+	return user, resp, err
+}
+
+func (c *Client) DeleteUser(ctx context.Context, userName string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/users/%s", userName), nil, jsonContent, nil)
+}
+
+func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) ([]*types.User, *http.Response, error) {
+	q := url.Values{}
+	if start != "" {
+		q.Add("start", start)
+	}
+	if limit > 0 {
+		q.Add("limit", strconv.Itoa(limit))
+	}
+	if asc {
+		q.Add("asc", "")
+	}
+
+	users := []*types.User{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users)
+	return users, resp, err
+}
+
+func (c *Client) CreateUserLA(ctx context.Context, userName string, req *CreateUserLARequest) (*types.LinkedAccount, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	la := new(types.LinkedAccount)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/users/%s/linkedaccounts", userName), nil, jsonContent, bytes.NewReader(reqj), la)
+	return la, resp, err
+}
+
+func (c *Client) DeleteUserLA(ctx context.Context, userName, laID string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/users/%s/linkedaccounts/%s", userName, laID), nil, jsonContent, nil)
+}
+
+func (c *Client) UpdateUserLA(ctx context.Context, userName, laID string, req *UpdateUserLARequest) (*types.LinkedAccount, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	la := new(types.LinkedAccount)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/users/%s/linkedaccounts/%s", userName, laID), nil, jsonContent, bytes.NewReader(reqj), la)
+	return la, resp, err
+}
+
+func (c *Client) CreateUserToken(ctx context.Context, userName string, req *CreateUserTokenRequest) (*CreateUserTokenResponse, *http.Response, error) {
+	reqj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	tresp := new(CreateUserTokenResponse)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/users/%s/tokens", userName), nil, jsonContent, bytes.NewReader(reqj), tresp)
+	return tresp, resp, err
+}
+
+func (c *Client) GetRemoteSource(ctx context.Context, rsID string) (*types.RemoteSource, *http.Response, error) {
+	rs := new(types.RemoteSource)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesource/%s", rsID), nil, jsonContent, nil, rs)
+	return rs, resp, err
+}
+
+func (c *Client) GetRemoteSourceByName(ctx context.Context, rsName string) (*types.RemoteSource, *http.Response, error) {
+	rs := new(types.RemoteSource)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsName), nil, jsonContent, nil, rs)
+	return rs, resp, err
+}
+
+func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) ([]*types.RemoteSource, *http.Response, error) {
+	q := url.Values{}
+	if start != "" {
+		q.Add("start", start)
+	}
+	if limit > 0 {
+		q.Add("limit", strconv.Itoa(limit))
+	}
+	if asc {
+		q.Add("asc", "")
+	}
+
+	rss := []*types.RemoteSource{}
+	resp, err := c.getParsedResponse(ctx, "GET", "/remotesources", q, jsonContent, nil, &rss)
+	return rss, resp, err
+}
+
+func (c *Client) CreateRemoteSource(ctx context.Context, rs *types.RemoteSource) (*types.RemoteSource, *http.Response, error) {
+	uj, err := json.Marshal(rs)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	rs = new(types.RemoteSource)
+	resp, err := c.getParsedResponse(ctx, "PUT", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs)
+	return rs, resp, err
+}
+
+func (c *Client) DeleteRemoteSource(ctx context.Context, name string) (*http.Response, error) {
+	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/remotesources/%s", name), nil, jsonContent, nil)
+}

internal/services/configstore/api/project.go

@@ -0,0 +1,207 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type GetProjectHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetProjectHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetProjectHandler {
+	return &GetProjectHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	projectID := vars["projectid"]
+
+	var project *types.Project
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		project, err = h.readDB.GetProject(tx, projectID)
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if project == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(project); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type GetProjectByNameHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetProjectByNameHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetProjectByNameHandler {
+	return &GetProjectByNameHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetProjectByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	projectName := vars["projectname"]
+
+	var project *types.Project
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		project, err = h.readDB.GetProjectByName(tx, projectName)
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if project == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(project); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateProjectHandler struct {
+	log    *zap.SugaredLogger
+	ch     *command.CommandHandler
+	readDB *readdb.ReadDB
+}
+
+func NewCreateProjectHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateProjectHandler {
+	return &CreateProjectHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	var req types.Project
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	project, err := h.ch.CreateProject(ctx, &req)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(project); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteProjectHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteProjectHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteProjectHandler {
+	return &DeleteProjectHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	projectName := vars["projectname"]
+
+	if err := h.ch.DeleteProject(ctx, projectName); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+const (
+	DefaultProjectsLimit = 10
+	MaxProjectsLimit     = 20
+)
+
+type ProjectsHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewProjectsHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectsHandler {
+	return &ProjectsHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *ProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+
+	limitS := query.Get("limit")
+	limit := DefaultProjectsLimit
+	if limitS != "" {
+		var err error
+		limit, err = strconv.Atoi(limitS)
+		if err != nil {
+			http.Error(w, "", http.StatusBadRequest)
+			return
+		}
+	}
+	if limit < 0 {
+		http.Error(w, "limit must be greater or equal than 0", http.StatusBadRequest)
+		return
+	}
+	if limit > MaxProjectsLimit {
+		limit = MaxProjectsLimit
+	}
+	asc := false
+	if _, ok := query["asc"]; ok {
+		asc = true
+	}
+
+	start := query.Get("start")
+
+	projects, err := h.readDB.GetProjects(start, limit, asc)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(projects); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

internal/services/configstore/api/remotesource.go

@@ -0,0 +1,207 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type GetRemoteSourceHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetRemoteSourceHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetRemoteSourceHandler {
+	return &GetRemoteSourceHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	remoteSourceID := vars["id"]
+
+	var remoteSource *types.RemoteSource
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		remoteSource, err = h.readDB.GetRemoteSource(tx, remoteSourceID)
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if remoteSource == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(remoteSource); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type GetRemoteSourceByNameHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetRemoteSourceByNameHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetRemoteSourceByNameHandler {
+	return &GetRemoteSourceByNameHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetRemoteSourceByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	remoteSourceName := vars["name"]
+
+	var remoteSource *types.RemoteSource
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		remoteSource, err = h.readDB.GetRemoteSourceByName(tx, remoteSourceName)
+		return err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if remoteSource == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(remoteSource); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateRemoteSourceHandler struct {
+	log    *zap.SugaredLogger
+	ch     *command.CommandHandler
+	readDB *readdb.ReadDB
+}
+
+func NewCreateRemoteSourceHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateRemoteSourceHandler {
+	return &CreateRemoteSourceHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	var req types.RemoteSource
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	remoteSource, err := h.ch.CreateRemoteSource(ctx, &req)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(remoteSource); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteRemoteSourceHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteRemoteSourceHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteRemoteSourceHandler {
+	return &DeleteRemoteSourceHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	remoteSourceName := vars["name"]
+
+	if err := h.ch.DeleteRemoteSource(ctx, remoteSourceName); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+const (
+	DefaultRemoteSourcesLimit = 10
+	MaxRemoteSourcesLimit     = 20
+)
+
+type RemoteSourcesHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewRemoteSourcesHandler(logger *zap.Logger, readDB *readdb.ReadDB) *RemoteSourcesHandler {
+	return &RemoteSourcesHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+
+	limitS := query.Get("limit")
+	limit := DefaultRemoteSourcesLimit
+	if limitS != "" {
+		var err error
+		limit, err = strconv.Atoi(limitS)
+		if err != nil {
+			http.Error(w, "", http.StatusBadRequest)
+			return
+		}
+	}
+	if limit < 0 {
+		http.Error(w, "limit must be greater or equal than 0", http.StatusBadRequest)
+		return
+	}
+	if limit > MaxRemoteSourcesLimit {
+		limit = MaxRemoteSourcesLimit
+	}
+	asc := false
+	if _, ok := query["asc"]; ok {
+		asc = true
+	}
+
+	start := query.Get("start")
+
+	remoteSources, err := h.readDB.GetRemoteSources(start, limit, asc)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(remoteSources); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

internal/services/configstore/api/user.go

@@ -0,0 +1,460 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+type GetUserHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetUserHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetUserHandler {
+	return &GetUserHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	userID := vars["userid"]
+
+	var user *types.User
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = h.readDB.GetUser(tx, userID)
+		return err
+	})
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if user == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(user); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type GetUserByNameHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewGetUserByNameHandler(logger *zap.Logger, readDB *readdb.ReadDB) *GetUserByNameHandler {
+	return &GetUserByNameHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *GetUserByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	userName := vars["username"]
+
+	var user *types.User
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = h.readDB.GetUserByName(tx, userName)
+		return err
+	})
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if user == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(user); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateUserHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewCreateUserHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateUserHandler {
+	return &CreateUserHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	var req types.User
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	user, err := h.ch.CreateUser(ctx, &req)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(user); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteUserHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteUserHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteUserHandler {
+	return &DeleteUserHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.log.Infof("deleteuserhandler")
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	userName := vars["username"]
+
+	if err := h.ch.DeleteUser(ctx, userName); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+const (
+	DefaultUsersLimit = 10
+	MaxUsersLimit     = 20
+)
+
+type UsersHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+}
+
+func NewUsersHandler(logger *zap.Logger, readDB *readdb.ReadDB) *UsersHandler {
+	return &UsersHandler{log: logger.Sugar(), readDB: readDB}
+}
+
+func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+
+	limitS := query.Get("limit")
+	limit := DefaultUsersLimit
+	if limitS != "" {
+		var err error
+		limit, err = strconv.Atoi(limitS)
+		if err != nil {
+			http.Error(w, "", http.StatusBadRequest)
+			return
+		}
+	}
+	if limit < 0 {
+		http.Error(w, "limit must be greater or equal than 0", http.StatusBadRequest)
+		return
+	}
+	if limit > MaxUsersLimit {
+		limit = MaxUsersLimit
+	}
+	asc := false
+	if _, ok := query["asc"]; ok {
+		asc = true
+	}
+
+	start := query.Get("start")
+
+	// handle special queries, like get user by token
+	queryType := query.Get("query_type")
+	h.log.Infof("query_type: %s", queryType)
+
+	var users []*types.User
+	switch queryType {
+	case "bytoken":
+		token := query.Get("token")
+		var user *types.User
+		err := h.readDB.Do(func(tx *db.Tx) error {
+			var err error
+			user, err = h.readDB.GetUserByTokenValue(tx, token)
+			return err
+		})
+		h.log.Infof("user: %s", util.Dump(user))
+		if err != nil {
+			h.log.Errorf("err: %+v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if user == nil {
+			http.Error(w, "", http.StatusNotFound)
+			return
+		}
+		users = []*types.User{user}
+	case "bylinkedaccount":
+		linkedAccountID := query.Get("linkedaccountid")
+		var user *types.User
+		err := h.readDB.Do(func(tx *db.Tx) error {
+			var err error
+			user, err = h.readDB.GetUserByLinkedAccount(tx, linkedAccountID)
+			return err
+		})
+		h.log.Infof("user: %s", util.Dump(user))
+		if err != nil {
+			h.log.Errorf("err: %+v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if user == nil {
+			http.Error(w, "", http.StatusNotFound)
+			return
+		}
+		users = []*types.User{user}
+	case "byremoteuser":
+		remoteUserID := query.Get("remoteuserid")
+		remoteSourceID := query.Get("remotesourceid")
+		var user *types.User
+		err := h.readDB.Do(func(tx *db.Tx) error {
+			var err error
+			user, err = h.readDB.GetUserByLinkedAccountRemoteUserIDandSource(tx, remoteUserID, remoteSourceID)
+			return err
+		})
+		h.log.Infof("user: %s", util.Dump(user))
+		if err != nil {
+			h.log.Errorf("err: %+v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if user == nil {
+			http.Error(w, "", http.StatusNotFound)
+			return
+		}
+		users = []*types.User{user}
+	default:
+		// default query
+		err := h.readDB.Do(func(tx *db.Tx) error {
+			var err error
+			users, err = h.readDB.GetUsers(tx, start, limit, asc)
+			return err
+		})
+		if err != nil {
+			h.log.Errorf("err: %+v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+
+	if err := json.NewEncoder(w).Encode(users); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateUserLARequest struct {
+	RemoteSourceName   string `json:"remote_source_name"`
+	RemoteUserID       string `json:"remote_user_id"`
+	RemoteUserName     string `json:"remote_user_name"`
+	UserAccessToken    string `json:"user_access_token"`
+	Oauth2AccessToken  string `json:"oauth2_access_token"`
+	Oauth2RefreshToken string `json:"oauth2_refresh_token"`
+}
+
+type CreateUserLAHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewCreateUserLAHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateUserLAHandler {
+	return &CreateUserLAHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateUserLAHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	userName := vars["username"]
+
+	var req CreateUserLARequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	creq := &command.CreateUserLARequest{
+		UserName:           userName,
+		RemoteSourceName:   req.RemoteSourceName,
+		RemoteUserID:       req.RemoteUserID,
+		RemoteUserName:     req.RemoteUserName,
+		Oauth2AccessToken:  req.Oauth2AccessToken,
+		Oauth2RefreshToken: req.Oauth2RefreshToken,
+		UserAccessToken:    req.UserAccessToken,
+	}
+	user, err := h.ch.CreateUserLA(ctx, creq)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(user); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type DeleteUserLAHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewDeleteUserLAHandler(logger *zap.Logger, ch *command.CommandHandler) *DeleteUserLAHandler {
+	return &DeleteUserLAHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *DeleteUserLAHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	userName := vars["username"]
+	laID := vars["laid"]
+
+	if err := h.ch.DeleteUserLA(ctx, userName, laID); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+type UpdateUserLARequest struct {
+	RemoteUserID       string `json:"remote_user_id"`
+	RemoteUserName     string `json:"remote_user_name"`
+	UserAccessToken    string `json:"user_access_token"`
+	Oauth2AccessToken  string `json:"oauth2_access_token"`
+	Oauth2RefreshToken string `json:"oauth2_refresh_token"`
+}
+
+type UpdateUserLAHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewUpdateUserLAHandler(logger *zap.Logger, ch *command.CommandHandler) *UpdateUserLAHandler {
+	return &UpdateUserLAHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *UpdateUserLAHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	userName := vars["username"]
+	linkedAccountID := vars["laid"]
+
+	var req UpdateUserLARequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	creq := &command.UpdateUserLARequest{
+		UserName:           userName,
+		LinkedAccountID:    linkedAccountID,
+		RemoteUserID:       req.RemoteUserID,
+		RemoteUserName:     req.RemoteUserName,
+		Oauth2AccessToken:  req.Oauth2AccessToken,
+		Oauth2RefreshToken: req.Oauth2RefreshToken,
+		UserAccessToken:    req.UserAccessToken,
+	}
+	user, err := h.ch.UpdateUserLA(ctx, creq)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(user); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+type CreateUserTokenRequest struct {
+	TokenName string `json:"token_name"`
+}
+
+type CreateUserTokenResponse struct {
+	Token string `json:"token"`
+}
+
+type CreateUserTokenHandler struct {
+	log *zap.SugaredLogger
+	ch  *command.CommandHandler
+}
+
+func NewCreateUserTokenHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateUserTokenHandler {
+	return &CreateUserTokenHandler{log: logger.Sugar(), ch: ch}
+}
+
+func (h *CreateUserTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	userName := vars["username"]
+
+	var req CreateUserTokenRequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	token, err := h.ch.CreateUserToken(ctx, userName, req.TokenName)
+	if err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	resp := &CreateUserTokenResponse{
+		Token: token,
+	}
+	if err := json.NewEncoder(w).Encode(resp); err != nil {
+		h.log.Errorf("err: %+v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

internal/services/configstore/command/command.go

@@ -0,0 +1,595 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/configstore/common"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+	"github.com/sorintlab/agola/internal/wal"
+
+	"github.com/pkg/errors"
+	uuid "github.com/satori/go.uuid"
+	"go.uber.org/zap"
+)
+
+type CommandHandler struct {
+	log    *zap.SugaredLogger
+	readDB *readdb.ReadDB
+	wal    *wal.WalManager
+}
+
+func NewCommandHandler(logger *zap.Logger, readDB *readdb.ReadDB, wal *wal.WalManager) *CommandHandler {
+	return &CommandHandler{
+		log:    logger.Sugar(),
+		readDB: readDB,
+		wal:    wal,
+	}
+}
+
+func (s *CommandHandler) CreateProject(ctx context.Context, project *types.Project) (*types.Project, error) {
+	if project.Name == "" {
+		return nil, errors.Errorf("project name required")
+	}
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{project.Name}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check duplicate project name
+		p, err := s.readDB.GetProjectByName(tx, project.Name)
+		if err != nil {
+			return err
+		}
+		if p != nil {
+			return errors.Errorf("project %q already exists", p.Name)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	project.ID = uuid.NewV4().String()
+
+	pcj, err := json.Marshal(project)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal project")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageProjectFile(project.ID),
+			Data:       pcj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return project, err
+}
+
+func (s *CommandHandler) DeleteProject(ctx context.Context, projectName string) error {
+	var project *types.Project
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{project.Name}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check project existance
+		project, err = s.readDB.GetProjectByName(tx, projectName)
+		if err != nil {
+			return err
+		}
+		if project == nil {
+			return errors.Errorf("project %q doesn't exist", projectName)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypeDelete,
+			Path:       common.StorageProjectFile(project.ID),
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}
+
+func (s *CommandHandler) CreateUser(ctx context.Context, user *types.User) (*types.User, error) {
+	if user.UserName == "" {
+		return nil, errors.Errorf("user name required")
+	}
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{user.UserName}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check duplicate user name
+		u, err := s.readDB.GetUserByName(tx, user.UserName)
+		if err != nil {
+			return err
+		}
+		if u != nil {
+			return errors.Errorf("user %q already exists", u.UserName)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	user.ID = uuid.NewV4().String()
+
+	pcj, err := json.Marshal(user)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal user")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageUserFile(user.ID),
+			Data:       pcj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return user, err
+}
+
+func (s *CommandHandler) DeleteUser(ctx context.Context, userName string) error {
+	var user *types.User
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{user.UserName}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check user existance
+		user, err = s.readDB.GetUserByName(tx, userName)
+		if err != nil {
+			return err
+		}
+		if user == nil {
+			return errors.Errorf("user %q doesn't exist", userName)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypeDelete,
+			Path:       common.StorageUserFile(user.ID),
+		},
+	}
+
+	// changegroup is the username (and in future the email) to ensure no
+	// concurrent user creation/modification using the same name
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}
+
+type CreateUserLARequest struct {
+	UserName           string
+	RemoteSourceName   string
+	RemoteUserID       string
+	RemoteUserName     string
+	UserAccessToken    string
+	Oauth2AccessToken  string
+	Oauth2RefreshToken string
+}
+
+func (s *CommandHandler) CreateUserLA(ctx context.Context, req *CreateUserLARequest) (*types.LinkedAccount, error) {
+	if req.UserName == "" {
+		return nil, errors.Errorf("user name required")
+	}
+	if req.RemoteSourceName == "" {
+		return nil, errors.Errorf("remote source name required")
+	}
+
+	var user *types.User
+	var rs *types.RemoteSource
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = s.readDB.GetUserByName(tx, req.UserName)
+		if err != nil {
+			return err
+		}
+		if user == nil {
+			return errors.Errorf("user %q doesn't exist", req.UserName)
+		}
+
+		cgNames := []string{user.ID}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		rs, err = s.readDB.GetRemoteSourceByName(tx, req.RemoteSourceName)
+		if err != nil {
+			return err
+		}
+		if rs == nil {
+			return errors.Errorf("remote source %q doesn't exist", req.RemoteSourceName)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if user.LinkedAccounts == nil {
+		user.LinkedAccounts = make(map[string]*types.LinkedAccount)
+	}
+
+	la := &types.LinkedAccount{
+		ID:                 uuid.NewV4().String(),
+		RemoteSourceID:     rs.ID,
+		RemoteUserID:       req.RemoteUserID,
+		RemoteUserName:     req.RemoteUserName,
+		UserAccessToken:    req.UserAccessToken,
+		Oauth2AccessToken:  req.Oauth2AccessToken,
+		Oauth2RefreshToken: req.Oauth2RefreshToken,
+	}
+
+	user.LinkedAccounts[la.ID] = la
+
+	pcj, err := json.Marshal(user)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal user")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageUserFile(user.ID),
+			Data:       pcj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return la, err
+}
+
+func (s *CommandHandler) DeleteUserLA(ctx context.Context, userName, laID string) error {
+	if userName == "" {
+		return errors.Errorf("user name required")
+	}
+	if laID == "" {
+		return errors.Errorf("user linked account id required")
+	}
+
+	var user *types.User
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = s.readDB.GetUserByName(tx, userName)
+		if err != nil {
+			return err
+		}
+		if user == nil {
+			return errors.Errorf("user %q doesn't exist", userName)
+		}
+
+		cgNames := []string{user.ID}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	_, ok := user.LinkedAccounts[laID]
+	if !ok {
+		return errors.Errorf("linked account id %q for user %q doesn't exist", laID, userName)
+	}
+
+	delete(user.LinkedAccounts, laID)
+
+	pcj, err := json.Marshal(user)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal user")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageUserFile(user.ID),
+			Data:       pcj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}
+
+type UpdateUserLARequest struct {
+	UserName           string
+	LinkedAccountID    string
+	RemoteUserID       string
+	RemoteUserName     string
+	UserAccessToken    string
+	Oauth2AccessToken  string
+	Oauth2RefreshToken string
+}
+
+func (s *CommandHandler) UpdateUserLA(ctx context.Context, req *UpdateUserLARequest) (*types.LinkedAccount, error) {
+	if req.UserName == "" {
+		return nil, errors.Errorf("user name required")
+	}
+
+	var user *types.User
+	var rs *types.RemoteSource
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = s.readDB.GetUserByName(tx, req.UserName)
+		if err != nil {
+			return err
+		}
+		if user == nil {
+			return errors.Errorf("user %q doesn't exist", req.UserName)
+		}
+
+		cgNames := []string{user.ID}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		la, ok := user.LinkedAccounts[req.LinkedAccountID]
+		if !ok {
+			return errors.Errorf("linked account id %q for user %q doesn't exist", req.LinkedAccountID, user.UserName)
+		}
+
+		rs, err = s.readDB.GetRemoteSource(tx, la.RemoteSourceID)
+		if err != nil {
+			return err
+		}
+		if rs == nil {
+			return errors.Errorf("remote source with id %q doesn't exist", la.RemoteSourceID)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	la := user.LinkedAccounts[req.LinkedAccountID]
+
+	la.RemoteUserID = req.RemoteUserID
+	la.RemoteUserName = req.RemoteUserName
+	la.UserAccessToken = req.UserAccessToken
+	la.Oauth2AccessToken = req.Oauth2AccessToken
+	la.Oauth2RefreshToken = req.Oauth2RefreshToken
+
+	userj, err := json.Marshal(user)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal user")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageUserFile(user.ID),
+			Data:       userj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return la, err
+}
+
+func (s *CommandHandler) CreateUserToken(ctx context.Context, userName, tokenName string) (string, error) {
+	if userName == "" {
+		return "", errors.Errorf("user name required")
+	}
+
+	var user *types.User
+
+	var cgt *wal.ChangeGroupsUpdateToken
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		user, err = s.readDB.GetUserByName(tx, userName)
+		if err != nil {
+			return err
+		}
+		if user == nil {
+			return errors.Errorf("user %q doesn't exist", userName)
+		}
+
+		cgNames := []string{user.ID}
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	if user.Tokens == nil {
+		user.Tokens = make(map[string]string)
+	}
+
+	token := util.EncodeSha1Hex(uuid.NewV4().String())
+	user.Tokens[tokenName] = token
+
+	userj, err := json.Marshal(user)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to marshal user")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageUserFile(user.ID),
+			Data:       userj,
+		},
+	}
+
+	// changegroup is the userid
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return token, err
+}
+
+func (s *CommandHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) {
+	if remoteSource.Name == "" {
+		return nil, errors.Errorf("remotesource name required")
+	}
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{remoteSource.Name}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	// since the use token is related to the transaction time
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check duplicate remoteSource name
+		u, err := s.readDB.GetRemoteSourceByName(tx, remoteSource.Name)
+		if err != nil {
+			return err
+		}
+		if u != nil {
+			return errors.Errorf("remoteSource %q already exists", u.Name)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	remoteSource.ID = uuid.NewV4().String()
+
+	rsj, err := json.Marshal(remoteSource)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal remotesource")
+	}
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypePut,
+			Path:       common.StorageRemoteSourceFile(remoteSource.ID),
+			Data:       rsj,
+		},
+	}
+
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return remoteSource, err
+}
+
+func (s *CommandHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName string) error {
+	var remoteSource *types.RemoteSource
+
+	var cgt *wal.ChangeGroupsUpdateToken
+	cgNames := []string{remoteSource.ID}
+
+	// must do all the check in a single transaction to avoid concurrent changes
+	err := s.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+
+		// check remoteSource existance
+		remoteSource, err = s.readDB.GetRemoteSourceByName(tx, remoteSourceName)
+		if err != nil {
+			return err
+		}
+		if remoteSource == nil {
+			return errors.Errorf("remotesource %q doesn't exist", remoteSourceName)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	actions := []*wal.Action{
+		{
+			ActionType: wal.ActionTypeDelete,
+			Path:       common.StorageRemoteSourceFile(remoteSource.ID),
+		},
+	}
+
+	// changegroup is all the remote sources
+	_, err = s.wal.WriteWal(ctx, actions, cgt)
+	return err
+}

internal/services/configstore/common/common.go

@@ -0,0 +1,68 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"path"
+)
+
+var (
+	// Storage paths. Always use path (not filepath) to use the "/" separator
+	StorageDataDir          = "data"
+	StorageProjectsDir      = path.Join(StorageDataDir, "projects")
+	StorageUsersDir         = path.Join(StorageDataDir, "users")
+	StorageRemoteSourcesDir = path.Join(StorageDataDir, "remotesources")
+)
+
+const (
+	etcdWalsMinRevisionRange = 100
+)
+
+func StorageProjectFile(projectID string) string {
+	return path.Join(StorageProjectsDir, projectID)
+}
+
+func StorageUserFile(userID string) string {
+	return path.Join(StorageUsersDir, userID)
+}
+
+func StorageRemoteSourceFile(userID string) string {
+	return path.Join(StorageRemoteSourcesDir, userID)
+}
+
+type ConfigType string
+
+const (
+	ConfigTypeProject      ConfigType = "project"
+	ConfigTypeUser         ConfigType = "user"
+	ConfigTypeRemoteSource ConfigType = "remotesource"
+)
+
+func PathToTypeID(p string) (ConfigType, string) {
+	var configType ConfigType
+	switch path.Dir(p) {
+	case StorageProjectsDir:
+		configType = ConfigTypeProject
+	case StorageUsersDir:
+		configType = ConfigTypeUser
+	case StorageRemoteSourcesDir:
+		configType = ConfigTypeRemoteSource
+	default:
+		panic(fmt.Errorf("cannot determine configtype for path: %q", p))
+	}
+
+	return configType, path.Base(p)
+}

internal/services/configstore/configstore.go

@@ -0,0 +1,198 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configstore
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+	"path/filepath"
+
+	scommon "github.com/sorintlab/agola/internal/common"
+	"github.com/sorintlab/agola/internal/etcd"
+	slog "github.com/sorintlab/agola/internal/log"
+	"github.com/sorintlab/agola/internal/objectstorage"
+	"github.com/sorintlab/agola/internal/services/config"
+	"github.com/sorintlab/agola/internal/services/configstore/api"
+	"github.com/sorintlab/agola/internal/services/configstore/command"
+	"github.com/sorintlab/agola/internal/services/configstore/readdb"
+	"github.com/sorintlab/agola/internal/util"
+	"github.com/sorintlab/agola/internal/wal"
+
+	ghandlers "github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+var level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
+var logger = slog.New(level)
+var log = logger.Sugar()
+
+type ConfigStore struct {
+	c             *config.ConfigStore
+	e             *etcd.Store
+	wal           *wal.WalManager
+	readDB        *readdb.ReadDB
+	lts           *objectstorage.ObjStorage
+	ch            *command.CommandHandler
+	listenAddress string
+}
+
+func NewConfigStore(ctx context.Context, c *config.ConfigStore) (*ConfigStore, error) {
+	if c.Debug {
+		level.SetLevel(zapcore.DebugLevel)
+	}
+
+	lts, err := scommon.NewLTS(&c.LTS)
+	if err != nil {
+		return nil, err
+	}
+	e, err := scommon.NewEtcd(&c.Etcd, logger, "configstore")
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &ConfigStore{
+		c:   c,
+		e:   e,
+		lts: lts,
+	}
+
+	walConf := &wal.WalManagerConfig{
+		E:   e,
+		Lts: lts,
+	}
+	wal, err := wal.NewWalManager(ctx, logger, walConf)
+	if err != nil {
+		return nil, err
+	}
+	readDB, err := readdb.NewReadDB(ctx, logger, filepath.Join(c.DataDir, "readdb"), e, lts, wal)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.wal = wal
+	cs.readDB = readDB
+
+	ch := command.NewCommandHandler(logger, readDB, wal)
+	cs.ch = ch
+
+	return cs, nil
+}
+
+func (s *ConfigStore) Run(ctx context.Context) error {
+	errCh := make(chan error)
+
+	go func() { errCh <- s.wal.Run(ctx) }()
+	go func() { errCh <- s.readDB.Run(ctx) }()
+
+	// noop coors handler
+	corsHandler := func(h http.Handler) http.Handler {
+		return h
+	}
+
+	corsAllowedMethodsOptions := ghandlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE"})
+	corsAllowedHeadersOptions := ghandlers.AllowedHeaders([]string{"Accept", "Accept-Encoding", "Authorization", "Content-Length", "Content-Type", "X-CSRF-Token", "Authorization"})
+	corsAllowedOriginsOptions := ghandlers.AllowedOrigins([]string{"*"})
+	corsHandler = ghandlers.CORS(corsAllowedMethodsOptions, corsAllowedHeadersOptions, corsAllowedOriginsOptions)
+
+	getProjectHandler := api.NewGetProjectHandler(logger, s.readDB)
+	projectsHandler := api.NewProjectsHandler(logger, s.readDB)
+	getProjectByNameHandler := api.NewGetProjectByNameHandler(logger, s.readDB)
+	createProjectHandler := api.NewCreateProjectHandler(logger, s.ch)
+	deleteProjectHandler := api.NewDeleteProjectHandler(logger, s.ch)
+
+	getUserHandler := api.NewGetUserHandler(logger, s.readDB)
+	usersHandler := api.NewUsersHandler(logger, s.readDB)
+	getUserByNameHandler := api.NewGetUserByNameHandler(logger, s.readDB)
+	createUserHandler := api.NewCreateUserHandler(logger, s.ch)
+	deleteUserHandler := api.NewDeleteUserHandler(logger, s.ch)
+
+	createUserLAHandler := api.NewCreateUserLAHandler(logger, s.ch)
+	deleteUserLAHandler := api.NewDeleteUserLAHandler(logger, s.ch)
+	updateUserLAHandler := api.NewUpdateUserLAHandler(logger, s.ch)
+
+	createUserTokenHandler := api.NewCreateUserTokenHandler(logger, s.ch)
+
+	getRemoteSourceHandler := api.NewGetRemoteSourceHandler(logger, s.readDB)
+	remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, s.readDB)
+	getRemoteSourceByNameHandler := api.NewGetRemoteSourceByNameHandler(logger, s.readDB)
+	createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, s.ch)
+	deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, s.ch)
+
+	router := mux.NewRouter()
+	apirouter := router.PathPrefix("/api/v1alpha").Subrouter()
+
+	apirouter.Handle("/project/{projectid}", getProjectHandler).Methods("GET")
+	apirouter.Handle("/projects", projectsHandler).Methods("GET")
+	apirouter.Handle("/projects", createProjectHandler).Methods("PUT")
+	apirouter.Handle("/projects/{projectname}", getProjectByNameHandler).Methods("GET")
+	apirouter.Handle("/projects/{projectname}", deleteProjectHandler).Methods("DELETE")
+
+	apirouter.Handle("/user/{userid}", getUserHandler).Methods("GET")
+	apirouter.Handle("/users", usersHandler).Methods("GET")
+	apirouter.Handle("/users", createUserHandler).Methods("PUT")
+	apirouter.Handle("/users/{username}", getUserByNameHandler).Methods("GET")
+	apirouter.Handle("/users/{username}", deleteUserHandler).Methods("DELETE")
+
+	apirouter.Handle("/users/{username}/linkedaccounts", createUserLAHandler).Methods("PUT")
+	apirouter.Handle("/users/{username}/linkedaccounts/{laid}", deleteUserLAHandler).Methods("DELETE")
+	apirouter.Handle("/users/{username}/linkedaccounts/{laid}", updateUserLAHandler).Methods("PUT")
+	apirouter.Handle("/users/{username}/tokens", createUserTokenHandler).Methods("PUT")
+
+	apirouter.Handle("/remotesource/{id}", getRemoteSourceHandler).Methods("GET")
+	apirouter.Handle("/remotesources", remoteSourcesHandler).Methods("GET")
+	apirouter.Handle("/remotesources", createRemoteSourceHandler).Methods("PUT")
+	apirouter.Handle("/remotesources/{name}", getRemoteSourceByNameHandler).Methods("GET")
+	apirouter.Handle("/remotesources/{name}", deleteRemoteSourceHandler).Methods("DELETE")
+
+	mainrouter := mux.NewRouter()
+	mainrouter.PathPrefix("/").Handler(corsHandler(router))
+
+	var tlsConfig *tls.Config
+	if s.c.Web.TLS {
+		var err error
+		tlsConfig, err = util.NewTLSConfig(s.c.Web.TLSCertFile, s.c.Web.TLSKeyFile, "", false)
+		if err != nil {
+			log.Errorf("err: %+v")
+			return err
+		}
+	}
+
+	httpServer := http.Server{
+		Addr:      s.c.Web.ListenAddress,
+		Handler:   mainrouter,
+		TLSConfig: tlsConfig,
+	}
+
+	lerrCh := make(chan error)
+	go func() {
+		lerrCh <- httpServer.ListenAndServe()
+	}()
+
+	select {
+	case <-ctx.Done():
+		log.Infof("configstore exiting")
+		httpServer.Close()
+		return nil
+	case err := <-lerrCh:
+		log.Errorf("http server listen error: %+v", err)
+		return err
+	case err := <-errCh:
+		log.Errorf("error: %+v", err)
+		return err
+	}
+}

internal/services/configstore/configstore_test.go

@@ -0,0 +1,232 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configstore
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/sorintlab/agola/internal/services/config"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/testutil"
+	"github.com/sorintlab/agola/internal/util"
+)
+
+func setupEtcd(t *testing.T, dir string) *testutil.TestEmbeddedEtcd {
+	tetcd, err := testutil.NewTestEmbeddedEtcd(t, logger, dir)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	if err := tetcd.Start(); err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	if err := tetcd.WaitUp(30 * time.Second); err != nil {
+		t.Fatalf("error waiting on etcd up: %v", err)
+	}
+	return tetcd
+}
+
+func shutdownEtcd(tetcd *testutil.TestEmbeddedEtcd) {
+	if tetcd.Etcd != nil {
+		tetcd.Kill()
+	}
+}
+
+func TestResync(t *testing.T) {
+	dir, err := ioutil.TempDir("", "agola")
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	etcdDir, err := ioutil.TempDir(dir, "etcd")
+	tetcd := setupEtcd(t, etcdDir)
+	defer shutdownEtcd(tetcd)
+
+	listenAddress1, port1, err := testutil.GetFreePort(true, false)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	listenAddress2, port2, err := testutil.GetFreePort(true, false)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	listenAddress3, port3, err := testutil.GetFreePort(true, false)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	ctx := context.Background()
+
+	ltsDir, err := ioutil.TempDir(dir, "lts")
+	csDir1, err := ioutil.TempDir(dir, "cs1")
+	csDir2, err := ioutil.TempDir(dir, "cs2")
+	csDir3, err := ioutil.TempDir(dir, "cs3")
+
+	baseConfig := config.ConfigStore{
+		Etcd: config.Etcd{
+			Endpoints: tetcd.Endpoint,
+		},
+		LTS: config.LTS{
+			Type: config.LTSTypePosix,
+			Path: ltsDir,
+		},
+		Web: config.Web{},
+	}
+	cs1Config := baseConfig
+	cs1Config.DataDir = csDir1
+	cs1Config.Web.ListenAddress = net.JoinHostPort(listenAddress1, port1)
+
+	cs2Config := baseConfig
+	cs2Config.DataDir = csDir2
+	cs2Config.Web.ListenAddress = net.JoinHostPort(listenAddress2, port2)
+
+	cs1, err := NewConfigStore(ctx, &cs1Config)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	cs2, err := NewConfigStore(ctx, &cs2Config)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	ctx1 := context.Background()
+	ctx2, cancel2 := context.WithCancel(context.Background())
+
+	t.Logf("starting cs1")
+	go func() {
+		if err := cs1.Run(ctx1); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	}()
+	go func() {
+		if err := cs2.Run(ctx2); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	}()
+
+	time.Sleep(1 * time.Second)
+
+	for i := 0; i < 10; i++ {
+		if _, err := cs1.ch.CreateProject(ctx, &types.Project{Name: fmt.Sprintf("project%d", i)}); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	time.Sleep(5 * time.Second)
+
+	// stop cs2
+	log.Infof("stopping cs2")
+	cancel2()
+
+	// Do some more changes
+	for i := 11; i < 20; i++ {
+		if _, err := cs1.ch.CreateProject(ctx, &types.Project{Name: fmt.Sprintf("project%d", i)}); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	time.Sleep(5 * time.Second)
+
+	// compact etcd
+	if err := tetcd.Compact(); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// start cs2
+	// it should resync from wals since the etcd revision as been compacted
+	cs2, err = NewConfigStore(ctx, &cs2Config)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	log.Infof("starting cs2")
+	ctx2 = context.Background()
+	go cs2.Run(ctx2)
+
+	time.Sleep(5 * time.Second)
+
+	projects1, err := cs1.readDB.GetProjects("", 0, true)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	projects2, err := cs2.readDB.GetProjects("", 0, true)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !compareProjects(projects1, projects2) {
+		t.Logf("len(projects1): %d", len(projects1))
+		t.Logf("len(projects2): %d", len(projects2))
+		t.Logf("projects1: %s", util.Dump(projects1))
+		t.Logf("projects2: %s", util.Dump(projects2))
+		t.Fatalf("projects are different between the two readdbs")
+	}
+
+	// start cs3, since it's a new instance it should do a full resync
+	cs3Config := baseConfig
+	cs3Config.DataDir = csDir3
+	cs3Config.Web.ListenAddress = net.JoinHostPort(listenAddress3, port3)
+
+	log.Infof("starting cs3")
+	cs3, err := NewConfigStore(ctx, &cs3Config)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	ctx3 := context.Background()
+	go cs3.Run(ctx3)
+
+	time.Sleep(5 * time.Second)
+
+	projects1, err = cs1.readDB.GetProjects("", 0, true)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	projects3, err := cs3.readDB.GetProjects("", 0, true)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !compareProjects(projects1, projects3) {
+		t.Logf("len(projects1): %d", len(projects1))
+		t.Logf("len(projects3): %d", len(projects3))
+		t.Logf("projects1: %s", util.Dump(projects1))
+		t.Logf("projects3: %s", util.Dump(projects3))
+		t.Fatalf("projects are different between the two readdbs")
+	}
+}
+
+func compareProjects(p1, p2 []*types.Project) bool {
+	p1ids := map[string]struct{}{}
+	p2ids := map[string]struct{}{}
+
+	for _, p := range p1 {
+		p1ids[p.ID] = struct{}{}
+	}
+	for _, p := range p2 {
+		p2ids[p.ID] = struct{}{}
+	}
+
+	return reflect.DeepEqual(p1ids, p2ids)
+}

internal/services/configstore/readdb/migration.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+var Stmts = []string{
+	// last processed etcd event revision
+	"create table revision (revision bigint, PRIMARY KEY(revision))",
+
+	// committedwalsequence stores the last committed wal sequence
+	"create table committedwalsequence (seq varchar, PRIMARY KEY (seq))",
+
+	// changegrouprevision stores the current revision of the changegroup for optimistic locking
+	"create table changegrouprevision (id varchar, revision varchar, PRIMARY KEY (id, revision))",
+
+	"create table project (id uuid, name varchar, ownerid varchar, data bytea, PRIMARY KEY (id))",
+	"create index project_name on project(name)",
+
+	"create table user (id uuid, name varchar, data bytea, PRIMARY KEY (id))",
+	"create table user_token (tokenvalue varchar, userid uuid, PRIMARY KEY (tokenvalue, userid))",
+
+	"create table remotesource (id uuid, name varchar, data bytea, PRIMARY KEY (id))",
+
+	"create table projectsource (id uuid, name varchar, data bytea, PRIMARY KEY (id))",
+
+	"create table linkedaccount_user (id uuid, remotesourceid uuid, userid uuid, remoteuserid uuid, PRIMARY KEY (id), FOREIGN KEY(userid) REFERENCES user(id))",
+
+	"create table linkedaccount_project (id uuid, projectid uuid, PRIMARY KEY (id), FOREIGN KEY(projectid) REFERENCES user(id))",
+}

internal/services/configstore/readdb/project.go

@@ -0,0 +1,184 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+var (
+	projectSelect = sb.Select("id", "data").From("project")
+	projectInsert = sb.Insert("project").Columns("id", "name", "data")
+)
+
+func (r *ReadDB) insertProject(tx *db.Tx, data []byte) error {
+	project := types.Project{}
+	if err := json.Unmarshal(data, &project); err != nil {
+		return errors.Wrap(err, "failed to unmarshal project")
+	}
+	// poor man insert or update...
+	if err := r.deleteProject(tx, project.ID); err != nil {
+		return err
+	}
+	q, args, err := projectInsert.Values(project.ID, project.Name, data).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	_, err = tx.Exec(q, args...)
+	return errors.Wrap(err, "failed to insert project")
+}
+
+func (r *ReadDB) deleteProject(tx *db.Tx, id string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from project where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete project")
+	}
+	return nil
+}
+
+func (r *ReadDB) GetProject(tx *db.Tx, projectID string) (*types.Project, error) {
+	q, args, err := projectSelect.Where(sq.Eq{"id": projectID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	projects, _, err := fetchProjects(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(projects) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(projects) == 0 {
+		return nil, nil
+	}
+	return projects[0], nil
+}
+
+func (r *ReadDB) GetProjectByName(tx *db.Tx, name string) (*types.Project, error) {
+	q, args, err := projectSelect.Where(sq.Eq{"name": name}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	projects, _, err := fetchProjects(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(projects) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(projects) == 0 {
+		return nil, nil
+	}
+	return projects[0], nil
+}
+
+func getProjectsFilteredQuery(startProjectName string, limit int, asc bool) sq.SelectBuilder {
+	fields := []string{"id", "data"}
+
+	s := sb.Select(fields...).From("project as project")
+	if asc {
+		s = s.OrderBy("project.name asc")
+	} else {
+		s = s.OrderBy("project.name desc")
+	}
+	if startProjectName != "" {
+		if asc {
+			s = s.Where(sq.Gt{"project.name": startProjectName})
+		} else {
+			s = s.Where(sq.Lt{"project.name": startProjectName})
+		}
+	}
+	if limit > 0 {
+		s = s.Limit(uint64(limit))
+	}
+
+	return s
+}
+func (r *ReadDB) GetProjects(startProjectName string, limit int, asc bool) ([]*types.Project, error) {
+	var projects []*types.Project
+
+	s := getProjectsFilteredQuery(startProjectName, limit, asc)
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	err = r.rdb.Do(func(tx *db.Tx) error {
+		rows, err := tx.Query(q, args...)
+		if err != nil {
+			return err
+		}
+
+		projects, _, err = scanProjects(rows)
+		return err
+	})
+	return projects, errors.WithStack(err)
+}
+
+func fetchProjects(tx *db.Tx, q string, args ...interface{}) ([]*types.Project, []string, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	return scanProjects(rows)
+}
+
+func scanProject(rows *sql.Rows, additionalFields ...interface{}) (*types.Project, string, error) {
+	var id string
+	var data []byte
+	if err := rows.Scan(&id, &data); err != nil {
+		return nil, "", errors.Wrap(err, "failed to scan rows")
+	}
+	project := types.Project{}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &project); err != nil {
+			return nil, "", errors.Wrap(err, "failed to unmarshal project")
+		}
+	}
+
+	return &project, id, nil
+}
+
+func scanProjects(rows *sql.Rows) ([]*types.Project, []string, error) {
+	projects := []*types.Project{}
+	ids := []string{}
+	for rows.Next() {
+		p, id, err := scanProject(rows)
+		if err != nil {
+			rows.Close()
+			return nil, nil, err
+		}
+		projects = append(projects, p)
+		ids = append(ids, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return projects, ids, nil
+}

internal/services/configstore/readdb/readdb.go

@@ -0,0 +1,812 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/etcd"
+	"github.com/sorintlab/agola/internal/objectstorage"
+	"github.com/sorintlab/agola/internal/sequence"
+	"github.com/sorintlab/agola/internal/services/configstore/common"
+	"github.com/sorintlab/agola/internal/util"
+	"github.com/sorintlab/agola/internal/wal"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+)
+
+var (
+	// Use postgresql $ placeholder. It'll be converted to ? from the provided db functions
+	sb = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)
+
+	revisionSelect = sb.Select("revision").From("revision")
+	revisionInsert = sb.Insert("revision").Columns("revision")
+
+	committedwalsequenceSelect = sb.Select("seq").From("committedwalsequence")
+	committedwalsequenceInsert = sb.Insert("committedwalsequence").Columns("seq")
+
+	changegrouprevisionSelect = sb.Select("id, revision").From("changegrouprevision")
+	changegrouprevisionInsert = sb.Insert("changegrouprevision").Columns("id", "revision")
+)
+
+type ReadDB struct {
+	log     *zap.SugaredLogger
+	dataDir string
+	e       *etcd.Store
+	rdb     *db.DB
+	lts     *objectstorage.ObjStorage
+	wal     *wal.WalManager
+
+	Initialized bool
+	initMutex   sync.Mutex
+}
+
+func NewReadDB(ctx context.Context, logger *zap.Logger, dataDir string, e *etcd.Store, lts *objectstorage.ObjStorage, wal *wal.WalManager) (*ReadDB, error) {
+	if err := os.MkdirAll(dataDir, 0770); err != nil {
+		return nil, err
+	}
+	rdb, err := db.NewDB(db.Sqlite3, filepath.Join(dataDir, "db"))
+	if err != nil {
+		return nil, err
+	}
+
+	// populate readdb
+	if err := rdb.Create(Stmts); err != nil {
+		return nil, err
+	}
+
+	readDB := &ReadDB{
+		log:     logger.Sugar(),
+		dataDir: dataDir,
+		rdb:     rdb,
+		e:       e,
+		lts:     lts,
+		wal:     wal,
+	}
+
+	return readDB, nil
+}
+
+// Initialize populates the readdb with the current etcd data and save the
+// revision to then feed it with the etcd events
+func (r *ReadDB) Initialize(ctx context.Context) error {
+	//  sync the rdb
+	if err := r.SyncRDB(ctx); err != nil {
+		return errors.Wrapf(err, "error syncing db")
+	}
+	return nil
+}
+
+func (r *ReadDB) ResetDB() error {
+	// TODO(sgotti) this needs to be protected by a mutex
+	r.rdb.Close()
+
+	// drop rdb
+	if err := os.Remove(filepath.Join(r.dataDir, "db")); err != nil {
+		return err
+	}
+
+	rdb, err := db.NewDB(db.Sqlite3, filepath.Join(r.dataDir, "db"))
+	if err != nil {
+		return err
+	}
+
+	// populate readdb
+	if err := rdb.Create(Stmts); err != nil {
+		return err
+	}
+
+	r.rdb = rdb
+
+	return nil
+}
+
+func (r *ReadDB) SyncFromFiles() (string, error) {
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	var lastCheckpointedWal string
+	// Get last checkpointed wal from lts
+	for wal := range r.wal.ListLtsWals("") {
+		if wal.Err != nil {
+			return "", wal.Err
+		}
+		if wal.Checkpointed {
+			lastCheckpointedWal = wal.WalSequence
+		}
+	}
+
+	doneCh = make(chan struct{})
+	haveConfigFiles := false
+	for object := range r.wal.List(common.StorageDataDir, "", true, doneCh) {
+		if object.Err != nil {
+			close(doneCh)
+			return "", object.Err
+		}
+
+		haveConfigFiles = true
+		break
+	}
+	close(doneCh)
+
+	if lastCheckpointedWal == "" && haveConfigFiles {
+		return "", errors.Errorf("no last checkpointed wal in lts but the storage has config files. This should never happen!")
+	}
+
+	if !haveConfigFiles {
+		return lastCheckpointedWal, nil
+	}
+
+	insertfunc := func(objs []string) error {
+		err := r.rdb.Do(func(tx *db.Tx) error {
+			for _, obj := range objs {
+				f, _, err := r.wal.ReadObject(obj, nil)
+				if err != nil {
+					if err == objectstorage.ErrNotExist {
+						r.log.Warnf("object %s disappeared, ignoring", obj)
+					}
+					return err
+				}
+				data, err := ioutil.ReadAll(f)
+				if err != nil {
+					f.Close()
+					return err
+				}
+				f.Close()
+
+				action := &wal.Action{
+					ActionType: wal.ActionTypePut,
+					Path:       obj,
+					Data:       data,
+				}
+				if err := r.applyAction(tx, action); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		return err
+	}
+
+	objs := []string{}
+	count := 0
+	doneCh = make(chan struct{})
+	defer close(doneCh)
+
+	// file may have changed in the meantime (due to checkpointing) but we don't
+	// need to have a consistent snapshot since we'll apply all the wals and handle
+	// them
+	for object := range r.wal.List(common.StorageDataDir, "", true, doneCh) {
+		if object.Err != nil {
+			return "", object.Err
+		}
+
+		objs = append(objs, object.Path)
+
+		if count > 100 {
+			if err := insertfunc(objs); err != nil {
+				return "", err
+			}
+			count = 0
+			objs = []string{}
+		} else {
+			count++
+		}
+	}
+	if err := insertfunc(objs); err != nil {
+		return "", err
+	}
+
+	// save the wal sequence of the last checkpointed wal before syncing from files
+	err := r.rdb.Do(func(tx *db.Tx) error {
+		return r.insertCommittedWalSequence(tx, lastCheckpointedWal)
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return lastCheckpointedWal, nil
+}
+
+func (r *ReadDB) SyncFromWals(startWalSeq, endWalSeq string) (string, error) {
+	insertfunc := func(walFiles []*wal.WalFile) error {
+		err := r.rdb.Do(func(tx *db.Tx) error {
+			for _, walFile := range walFiles {
+				walFilef, err := r.wal.ReadWal(walFile.WalSequence)
+				if err != nil {
+					return err
+				}
+				dec := json.NewDecoder(walFilef)
+				var header *wal.WalHeader
+				if err = dec.Decode(&header); err != nil && err != io.EOF {
+					walFilef.Close()
+					return err
+				}
+				walFilef.Close()
+				if err := r.insertCommittedWalSequence(tx, walFile.WalSequence); err != nil {
+					return err
+				}
+				if err := r.applyWal(tx, header.WalDataFileID); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		return err
+	}
+
+	lastWalSeq := startWalSeq
+	walFiles := []*wal.WalFile{}
+	count := 0
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	for walFile := range r.wal.ListLtsWals(startWalSeq) {
+		if walFile.Err != nil {
+			return "", walFile.Err
+		}
+
+		walFiles = append(walFiles, walFile)
+		lastWalSeq = walFile.WalSequence
+
+		if count > 100 {
+			if err := insertfunc(walFiles); err != nil {
+				return "", err
+			}
+			count = 0
+			walFiles = []*wal.WalFile{}
+		} else {
+			count++
+		}
+	}
+	if err := insertfunc(walFiles); err != nil {
+		return "", err
+	}
+
+	return lastWalSeq, nil
+}
+
+func (r *ReadDB) SyncRDB(ctx context.Context) error {
+	// get the last committed storage wal sequence saved in the rdb
+	curWalSeq := ""
+	err := r.rdb.Do(func(tx *db.Tx) error {
+		var err error
+		curWalSeq, err = r.GetCommittedWalSequence(tx)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	lastCommittedStorageWal, _, err := r.wal.LastCommittedStorageWal(ctx)
+	if err != nil {
+		return err
+	}
+
+	doFullSync := false
+	if curWalSeq == "" {
+		doFullSync = true
+		r.log.Warn("no startWalSeq in db, doing a full sync")
+	} else {
+		ok, err := r.wal.HasLtsWal(curWalSeq)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			r.log.Warnf("no wal with seq %q in lts, doing a full sync", curWalSeq)
+			doFullSync = true
+		}
+
+		// if the epoch of the wals has changed this means etcd has been reset. If so we should do a full resync since we are saving in the rdb also data that was not yet committed to lts so we should have the rdb ahead of the current lts data
+		// TODO(sgotti) improve this to avoid doing a full resync
+		curWalSequence, err := sequence.Parse(curWalSeq)
+		if err != nil {
+			return err
+		}
+		curWalEpoch := curWalSequence.Epoch
+
+		lastCommittedStorageWalSequence, err := sequence.Parse(lastCommittedStorageWal)
+		if err != nil {
+			return err
+		}
+		if curWalEpoch != lastCommittedStorageWalSequence.Epoch {
+			r.log.Warnf("current rdb wal sequence epoch %d different than new wal sequence epoch %d, doing a full sync", curWalEpoch, lastCommittedStorageWalSequence.Epoch)
+			doFullSync = true
+		}
+	}
+
+	if doFullSync {
+		r.log.Infof("doing a full sync from lts files")
+		if err := r.ResetDB(); err != nil {
+			return err
+		}
+
+		var err error
+		curWalSeq, err = r.SyncFromFiles()
+		if err != nil {
+			return err
+		}
+	}
+
+	r.log.Infof("startWalSeq: %s", curWalSeq)
+
+	// Sync from wals
+	// sync from lts until the current known lastCommittedStorageWal in etcd
+	// since wals are first committed to lts and then in etcd we would like to
+	// avoid to store in rdb something that is not yet marked as committedstorage
+	// in etcd
+	curWalSeq, err = r.SyncFromWals(curWalSeq, lastCommittedStorageWal)
+	if err != nil {
+		return errors.Wrap(err, "failed to sync from wals")
+	}
+
+	// Get the first available wal from etcd and check that our current walseq
+	// from wals on lts is >=
+	// if not (this happens when syncFromWals takes some time and in the meantime
+	// many new wals are written, the next sync should be faster and able to continue
+	firstAvailableWalData, revision, err := r.wal.FirstAvailableWalData(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to get first available wal data")
+	}
+	r.log.Infof("firstAvailableWalData: %s", util.Dump(firstAvailableWalData))
+	r.log.Infof("revision: %d", revision)
+	if firstAvailableWalData == nil {
+		if curWalSeq != "" {
+			// this happens if etcd has been reset
+			return errors.Errorf("our curwalseq is %q but there's no wal data on etcd", curWalSeq)
+		}
+	}
+	if firstAvailableWalData != nil {
+		if curWalSeq < firstAvailableWalData.WalSequence {
+			return errors.Errorf("current applied wal seq %q is smaller than the first available wal on etcd %q", curWalSeq, firstAvailableWalData.WalSequence)
+		}
+	}
+
+	err = r.rdb.Do(func(tx *db.Tx) error {
+		if err := r.insertRevision(tx, revision); err != nil {
+			return err
+		}
+
+		// use the same revision as previous operation
+		for walElement := range r.wal.ListEtcdWals(ctx, revision) {
+			if walElement.Err != nil {
+				return err
+			}
+			if walElement.WalData.WalSequence <= curWalSeq {
+				continue
+			}
+			//if walElement.WalData.WalStatus == wal.WalStatusCommittedStorage {
+
+			if err := r.insertCommittedWalSequence(tx, walElement.WalData.WalSequence); err != nil {
+				return err
+			}
+			//}
+
+			//// update readdb only when the wal has been committed to lts
+			//if walElement.WalData.WalStatus != wal.WalStatusCommittedStorage {
+			//	return nil
+			//}
+
+			r.log.Debugf("applying wal to db")
+			if err := r.applyWal(tx, walElement.WalData.WalDataFileID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func (r *ReadDB) Run(ctx context.Context) error {
+	revision, err := r.GetRevision()
+	if err != nil {
+		return err
+	}
+
+	if revision == 0 {
+		for {
+			err := r.Initialize(ctx)
+			if err == nil {
+				r.Initialized = true
+				break
+			}
+			r.log.Errorf("initialize err: %+v", err)
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+	for {
+		if !r.Initialized {
+			r.Initialize(ctx)
+		}
+		if err := r.HandleEvents(ctx); err != nil {
+			r.log.Errorf("handleevents err: %+v", err)
+		}
+
+		select {
+		case <-ctx.Done():
+			r.log.Infof("readdb exiting")
+			r.rdb.Close()
+			return nil
+		default:
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+}
+
+// TODO(sgotti) improve to apply when the wal have been "committedstorage" and
+// not only "committed", in this way we don't have to full resync when etcd is
+// lost/reset
+func (r *ReadDB) HandleEvents(ctx context.Context) error {
+	var revision int64
+	err := r.rdb.Do(func(tx *db.Tx) error {
+		err := tx.QueryRow("select revision from revision order by revision desc limit 1").Scan(&revision)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				revision = 0
+			} else {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	wctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	r.log.Infof("revision: %d", revision)
+	wch := r.wal.Watch(wctx, revision+1)
+	for we := range wch {
+		r.log.Debugf("we: %s", util.Dump(we))
+		if we.Err != nil {
+			err := we.Err
+			if err == wal.ErrCompacted {
+				r.log.Warnf("required events already compacted, reinitializing readdb")
+				r.Initialized = false
+				return nil
+			}
+			return errors.Wrapf(err, "watch error")
+		}
+
+		// a single transaction for every response (every response contains all the
+		// events happened in an etcd revision).
+		err = r.rdb.Do(func(tx *db.Tx) error {
+
+			// if theres a wal seq epoch change something happened to etcd, usually (if
+			// the user hasn't messed up with etcd keys) this means etcd has been reset
+			// in such case we should resync from the lts state to ensure we apply all the
+			// wal marked as committedstorage (since they could have been lost from etcd)
+			curWalSeq, err := r.GetCommittedWalSequence(tx)
+			if err != nil {
+				return err
+			}
+			r.log.Debugf("curWalSeq: %q", curWalSeq)
+			if curWalSeq != "" && we.WalData != nil {
+				curWalSequence, err := sequence.Parse(curWalSeq)
+				if err != nil {
+					return err
+				}
+				curWalEpoch := curWalSequence.Epoch
+
+				weWalSequence, err := sequence.Parse(we.WalData.WalSequence)
+				if err != nil {
+					return err
+				}
+				r.log.Infof("we.WalData.WalSequence: %q", we.WalData.WalSequence)
+				weWalEpoch := weWalSequence.Epoch
+				if curWalEpoch != weWalEpoch {
+					r.Initialized = false
+					return errors.Errorf("current rdb wal sequence epoch %d different than new wal sequence epoch %d, resyncing from lts", curWalEpoch, weWalEpoch)
+				}
+			}
+
+			if err := r.handleEvent(tx, we); err != nil {
+				return err
+			}
+
+			if err := r.insertRevision(tx, we.Revision); err != nil {
+				return err
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+	r.log.Infof("wch closed")
+
+	return nil
+}
+
+func (r *ReadDB) handleEvent(tx *db.Tx, we *wal.WatchElement) error {
+	//r.log.Debugf("event: %s %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value)
+	//key := string(ev.Kv.Key)
+
+	if err := r.handleWalEvent(tx, we); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *ReadDB) handleWalEvent(tx *db.Tx, we *wal.WatchElement) error {
+	// update readdb only when the wal has been committed to lts
+	//if we.WalData.WalStatus != wal.WalStatusCommittedStorage {
+	//	return nil
+	//}
+
+	if we.WalData != nil {
+		if err := r.insertCommittedWalSequence(tx, we.WalData.WalSequence); err != nil {
+			return err
+		}
+	}
+
+	for cgName, cgRev := range we.ChangeGroupsRevisions {
+		if err := r.insertChangeGroupRevision(tx, cgName, cgRev); err != nil {
+			return err
+		}
+	}
+
+	if we.WalData != nil {
+		r.log.Debugf("applying wal to db")
+		return r.applyWal(tx, we.WalData.WalDataFileID)
+	}
+	return nil
+}
+
+func (r *ReadDB) applyWal(tx *db.Tx, walDataFileID string) error {
+	walFile, err := r.wal.ReadWalData(walDataFileID)
+	if err != nil {
+		return errors.Wrapf(err, "cannot read wal data file %q", walDataFileID)
+	}
+	defer walFile.Close()
+
+	dec := json.NewDecoder(walFile)
+	for {
+		var action *wal.Action
+
+		err := dec.Decode(&action)
+		if err == io.EOF {
+			// all done
+			break
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to decode wal file")
+		}
+
+		if err := r.applyAction(tx, action); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *ReadDB) applyAction(tx *db.Tx, action *wal.Action) error {
+	configType, ID := common.PathToTypeID(action.Path)
+
+	switch action.ActionType {
+	case wal.ActionTypePut:
+		switch configType {
+		case common.ConfigTypeProject:
+			if err := r.insertProject(tx, action.Data); err != nil {
+				return err
+			}
+		case common.ConfigTypeUser:
+			if err := r.insertUser(tx, action.Data); err != nil {
+				return err
+			}
+		case common.ConfigTypeRemoteSource:
+			if err := r.insertRemoteSource(tx, action.Data); err != nil {
+				return err
+			}
+		}
+
+	case wal.ActionTypeDelete:
+		switch configType {
+		case common.ConfigTypeProject:
+			r.log.Debugf("deleting project with id: %s", ID)
+			if err := r.deleteProject(tx, ID); err != nil {
+				return err
+			}
+		case common.ConfigTypeUser:
+			r.log.Debugf("deleting user with id: %s", ID)
+			if err := r.deleteUser(tx, ID); err != nil {
+				return err
+			}
+		case common.ConfigTypeRemoteSource:
+			r.log.Debugf("deleting remote source with id: %s", ID)
+			if err := r.deleteRemoteSource(tx, ID); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (r *ReadDB) Do(f func(tx *db.Tx) error) error {
+	return r.rdb.Do(f)
+}
+
+func (r *ReadDB) insertRevision(tx *db.Tx, revision int64) error {
+	//r.log.Infof("insert revision: %d", revision)
+	// poor man insert or update that works because transaction isolation level is serializable
+	if _, err := tx.Exec("delete from revision"); err != nil {
+		return errors.Wrap(err, "failed to delete revision")
+	}
+	// TODO(sgotti) go database/sql and mattn/sqlite3 don't support uint64 types...
+	//q, args, err = revisionInsert.Values(int64(wresp.Header.ClusterId), run.Revision).ToSql()
+	q, args, err := revisionInsert.Values(revision).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	if _, err = tx.Exec(q, args...); err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+func (r *ReadDB) GetRevision() (int64, error) {
+	var revision int64
+
+	err := r.rdb.Do(func(tx *db.Tx) error {
+		var err error
+		revision, err = r.getRevision(tx)
+		return err
+	})
+	return revision, err
+}
+
+func (r *ReadDB) getRevision(tx *db.Tx) (int64, error) {
+	var revision int64
+
+	q, args, err := revisionSelect.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to build query")
+	}
+
+	err = tx.QueryRow(q, args...).Scan(&revision)
+	if err == sql.ErrNoRows {
+		return 0, nil
+	}
+	return revision, err
+}
+
+func (r *ReadDB) insertCommittedWalSequence(tx *db.Tx, seq string) error {
+	r.log.Infof("insert seq: %s", seq)
+	// poor man insert or update that works because transaction isolation level is serializable
+	if _, err := tx.Exec("delete from committedwalsequence"); err != nil {
+		return errors.Wrap(err, "failed to delete committedwalsequence")
+	}
+	q, args, err := committedwalsequenceInsert.Values(seq).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	if _, err = tx.Exec(q, args...); err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+func (r *ReadDB) GetCommittedWalSequence(tx *db.Tx) (string, error) {
+	var seq string
+
+	q, args, err := committedwalsequenceSelect.OrderBy("seq").Limit(1).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to build query")
+	}
+
+	err = tx.QueryRow(q, args...).Scan(&seq)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	return seq, err
+}
+
+func (r *ReadDB) insertChangeGroupRevision(tx *db.Tx, changegroup string, revision int64) error {
+	r.log.Infof("insertChangeGroupRevision: %s %d", changegroup, revision)
+
+	// poor man insert or update that works because transaction isolation level is serializable
+	if _, err := tx.Exec("delete from changegrouprevision where id = $1", changegroup); err != nil {
+		return errors.Wrap(err, "failed to delete run")
+	}
+	// insert only if revision > 0
+	if revision > 0 {
+		q, args, err := changegrouprevisionInsert.Values(changegroup, revision).ToSql()
+		if err != nil {
+			return errors.Wrap(err, "failed to build query")
+		}
+		if _, err = tx.Exec(q, args...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *ReadDB) GetChangeGroupsUpdateTokens(tx *db.Tx, groups []string) (*wal.ChangeGroupsUpdateToken, error) {
+	s := changegrouprevisionSelect.Where(sq.Eq{"id": groups})
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+	cgr, err := fetchChangeGroupsRevision(tx, q, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	revision, err := r.getRevision(tx)
+	if err != nil {
+		return nil, err
+	}
+
+	// for non existing changegroups use a changegroup with revision = 0
+	for _, g := range groups {
+		if _, ok := cgr[g]; !ok {
+			cgr[g] = 0
+		}
+	}
+
+	return &wal.ChangeGroupsUpdateToken{CurRevision: revision, ChangeGroupsRevisions: cgr}, nil
+}
+
+func fetchChangeGroupsRevision(tx *db.Tx, q string, args ...interface{}) (map[string]int64, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	return scanChangeGroupsRevision(rows)
+}
+
+func scanChangeGroupsRevision(rows *sql.Rows) (map[string]int64, error) {
+	changegroups := map[string]int64{}
+	for rows.Next() {
+		var (
+			id       string
+			revision int64
+		)
+		if err := rows.Scan(&id, &revision); err != nil {
+			return nil, errors.Wrap(err, "failed to scan rows")
+		}
+		changegroups[id] = revision
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return changegroups, nil
+}

internal/services/configstore/readdb/remotesource.go

@@ -0,0 +1,184 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+var (
+	remotesourceSelect = sb.Select("id", "data").From("remotesource")
+	remotesourceInsert = sb.Insert("remotesource").Columns("id", "name", "data")
+)
+
+func (r *ReadDB) insertRemoteSource(tx *db.Tx, data []byte) error {
+	remoteSource := types.RemoteSource{}
+	if err := json.Unmarshal(data, &remoteSource); err != nil {
+		return errors.Wrap(err, "failed to unmarshal remotesource")
+	}
+	// poor man insert or update...
+	if err := r.deleteRemoteSource(tx, remoteSource.ID); err != nil {
+		return err
+	}
+	q, args, err := remotesourceInsert.Values(remoteSource.ID, remoteSource.Name, data).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	_, err = tx.Exec(q, args...)
+	return errors.Wrap(err, "failed to insert remotesource")
+}
+
+func (r *ReadDB) deleteRemoteSource(tx *db.Tx, id string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from remotesource where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete remotesource")
+	}
+	return nil
+}
+
+func (r *ReadDB) GetRemoteSource(tx *db.Tx, remoteSourceID string) (*types.RemoteSource, error) {
+	q, args, err := remotesourceSelect.Where(sq.Eq{"id": remoteSourceID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	remoteSources, _, err := fetchRemoteSources(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(remoteSources) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(remoteSources) == 0 {
+		return nil, nil
+	}
+	return remoteSources[0], nil
+}
+
+func (r *ReadDB) GetRemoteSourceByName(tx *db.Tx, name string) (*types.RemoteSource, error) {
+	q, args, err := remotesourceSelect.Where(sq.Eq{"name": name}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	remoteSources, _, err := fetchRemoteSources(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(remoteSources) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(remoteSources) == 0 {
+		return nil, nil
+	}
+	return remoteSources[0], nil
+}
+
+func getRemoteSourcesFilteredQuery(startRemoteSourceName string, limit int, asc bool) sq.SelectBuilder {
+	fields := []string{"id", "data"}
+
+	s := sb.Select(fields...).From("remotesource as remotesource")
+	if asc {
+		s = s.OrderBy("remotesource.name asc")
+	} else {
+		s = s.OrderBy("remotesource.name desc")
+	}
+	if startRemoteSourceName != "" {
+		if asc {
+			s = s.Where(sq.Gt{"remotesource.name": startRemoteSourceName})
+		} else {
+			s = s.Where(sq.Lt{"remotesource.name": startRemoteSourceName})
+		}
+	}
+	if limit > 0 {
+		s = s.Limit(uint64(limit))
+	}
+
+	return s
+}
+func (r *ReadDB) GetRemoteSources(startRemoteSourceName string, limit int, asc bool) ([]*types.RemoteSource, error) {
+	var remoteSources []*types.RemoteSource
+
+	s := getRemoteSourcesFilteredQuery(startRemoteSourceName, limit, asc)
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	err = r.rdb.Do(func(tx *db.Tx) error {
+		rows, err := tx.Query(q, args...)
+		if err != nil {
+			return err
+		}
+
+		remoteSources, _, err = scanRemoteSources(rows)
+		return err
+	})
+	return remoteSources, errors.WithStack(err)
+}
+
+func fetchRemoteSources(tx *db.Tx, q string, args ...interface{}) ([]*types.RemoteSource, []string, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	return scanRemoteSources(rows)
+}
+
+func scanRemoteSource(rows *sql.Rows, additionalFields ...interface{}) (*types.RemoteSource, string, error) {
+	var id string
+	var data []byte
+	if err := rows.Scan(&id, &data); err != nil {
+		return nil, "", errors.Wrap(err, "failed to scan rows")
+	}
+	remoteSource := types.RemoteSource{}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &remoteSource); err != nil {
+			return nil, "", errors.Wrap(err, "failed to unmarshal remotesource")
+		}
+	}
+
+	return &remoteSource, id, nil
+}
+
+func scanRemoteSources(rows *sql.Rows) ([]*types.RemoteSource, []string, error) {
+	remoteSources := []*types.RemoteSource{}
+	ids := []string{}
+	for rows.Next() {
+		p, id, err := scanRemoteSource(rows)
+		if err != nil {
+			rows.Close()
+			return nil, nil, err
+		}
+		remoteSources = append(remoteSources, p)
+		ids = append(ids, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return remoteSources, ids, nil
+}

internal/services/configstore/readdb/user.go

@@ -0,0 +1,381 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readdb
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/sorintlab/agola/internal/db"
+	"github.com/sorintlab/agola/internal/services/types"
+	"github.com/sorintlab/agola/internal/util"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+var (
+	userSelect = sb.Select("user.id", "user.data").From("user")
+	userInsert = sb.Insert("user").Columns("id", "name", "data")
+
+	linkedaccountSelect        = sb.Select("id", "data").From("linkedaccount")
+	linkedaccountInsert        = sb.Insert("linkedaccount").Columns("id", "name", "data")
+	linkedaccountuserInsert    = sb.Insert("linkedaccount_user").Columns("id", "remotesourceid", "userid", "remoteuserid")
+	linkedaccountuserSelect    = sb.Select("id", "userid").From("linkedaccount_user")
+	linkedaccountprojectInsert = sb.Insert("linkedaccount_project").Columns("id", "userid")
+
+	usertokenSelect = sb.Select("tokenvalue", "userid").From("user_token")
+	usertokenInsert = sb.Insert("user_token").Columns("tokenvalue", "userid")
+)
+
+func (r *ReadDB) insertUser(tx *db.Tx, data []byte) error {
+	user := types.User{}
+	if err := json.Unmarshal(data, &user); err != nil {
+		return errors.Wrap(err, "failed to unmarshal user")
+	}
+	r.log.Infof("inserting user: %s", util.Dump(user))
+	// poor man insert or update...
+	if err := r.deleteUser(tx, user.ID); err != nil {
+		return err
+	}
+	q, args, err := userInsert.Values(user.ID, user.UserName, data).ToSql()
+	if err != nil {
+		return errors.Wrap(err, "failed to build query")
+	}
+	if _, err := tx.Exec(q, args...); err != nil {
+		return errors.Wrap(err, "failed to insert user")
+	}
+
+	// insert linkedaccounts_user
+	for _, la := range user.LinkedAccounts {
+		if err := r.deleteUserLinkedAccount(tx, la.ID); err != nil {
+			return err
+		}
+		q, args, err = linkedaccountuserInsert.Values(la.ID, la.RemoteSourceID, user.ID, la.RemoteUserID).ToSql()
+		if err != nil {
+			return errors.Wrap(err, "failed to build query")
+		}
+		if _, err := tx.Exec(q, args...); err != nil {
+			return errors.Wrap(err, "failed to insert user")
+		}
+	}
+	// insert user_token
+	for _, tokenValue := range user.Tokens {
+		r.log.Infof("inserting user token: %s", tokenValue)
+		if err := r.deleteUserToken(tx, tokenValue); err != nil {
+			return err
+		}
+		q, args, err = usertokenInsert.Values(tokenValue, user.ID).ToSql()
+		if err != nil {
+			return errors.Wrap(err, "failed to build query")
+		}
+		if _, err := tx.Exec(q, args...); err != nil {
+			return errors.Wrap(err, "failed to insert user")
+		}
+	}
+
+	return nil
+}
+
+func (r *ReadDB) deleteUser(tx *db.Tx, userID string) error {
+	// delete user linked accounts
+	if err := r.deleteUserLinkedAccounts(tx, userID); err != nil {
+		return errors.Wrap(err, "failed to delete user linked accounts")
+	}
+
+	// delete user tokens
+	if _, err := tx.Exec("delete from user_token where userid = $1", userID); err != nil {
+		return errors.Wrap(err, "failed to delete usertokens")
+	}
+
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from user where id = $1", userID); err != nil {
+		return errors.Wrap(err, "failed to delete user")
+	}
+
+	return nil
+}
+
+func (r *ReadDB) deleteUserLinkedAccounts(tx *db.Tx, userID string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from linkedaccount_user where userid = $1", userID); err != nil {
+		return errors.Wrap(err, "failed to delete linked account")
+	}
+	if _, err := tx.Exec("delete from linkedaccount_project where id = $1", userID); err != nil {
+		return errors.Wrap(err, "failed to delete linked account")
+	}
+	return nil
+}
+
+func (r *ReadDB) deleteUserLinkedAccount(tx *db.Tx, id string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from linkedaccount_user where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete linked account")
+	}
+	if _, err := tx.Exec("delete from linkedaccount_project where id = $1", id); err != nil {
+		return errors.Wrap(err, "failed to delete linked account")
+	}
+	return nil
+}
+
+func (r *ReadDB) deleteAllUserTokens(tx *db.Tx, userID string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from user_token where userid = $1", userID); err != nil {
+		return errors.Wrap(err, "failed to delete user_token")
+	}
+	return nil
+}
+
+func (r *ReadDB) deleteUserToken(tx *db.Tx, tokenValue string) error {
+	// poor man insert or update...
+	if _, err := tx.Exec("delete from user_token where tokenvalue = $1", tokenValue); err != nil {
+		return errors.Wrap(err, "failed to delete user_token")
+	}
+	return nil
+}
+
+func (r *ReadDB) GetUser(tx *db.Tx, userID string) (*types.User, error) {
+	q, args, err := userSelect.Where(sq.Eq{"id": userID}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	users, _, err := fetchUsers(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(users) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(users) == 0 {
+		return nil, nil
+	}
+	return users[0], nil
+}
+
+func (r *ReadDB) GetUserByName(tx *db.Tx, name string) (*types.User, error) {
+	q, args, err := userSelect.Where(sq.Eq{"name": name}).ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	users, _, err := fetchUsers(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(users) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(users) == 0 {
+		return nil, nil
+	}
+	return users[0], nil
+}
+
+func (r *ReadDB) GetUserByTokenValue(tx *db.Tx, tokenValue string) (*types.User, error) {
+	s := userSelect
+	s = s.Join("user_token on user_token.userid = user.id")
+	s = s.Where(sq.Eq{"user_token.tokenvalue": tokenValue})
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	users, _, err := fetchUsers(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(users) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(users) == 0 {
+		return nil, nil
+	}
+	return users[0], nil
+}
+
+func (r *ReadDB) GetUserByLinkedAccount(tx *db.Tx, linkedAccountID string) (*types.User, error) {
+	s := userSelect
+	s = s.Join("linkedaccount_user as lau on lau.userid = user.id")
+	s = s.Where(sq.Eq{"lau.id": linkedAccountID})
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	users, _, err := fetchUsers(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(users) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(users) == 0 {
+		return nil, nil
+	}
+	return users[0], nil
+}
+
+func (r *ReadDB) GetUserByLinkedAccountRemoteUserIDandSource(tx *db.Tx, remoteUserID, remoteSourceID string) (*types.User, error) {
+	s := userSelect
+	s = s.Join("linkedaccount_user as lau on lau.userid = user.id")
+	s = s.Where(sq.Eq{"lau.remoteuserid": remoteUserID, "lau.remotesourceid": remoteSourceID})
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	users, _, err := fetchUsers(tx, q, args...)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if len(users) > 1 {
+		return nil, errors.Errorf("too many rows returned")
+	}
+	if len(users) == 0 {
+		return nil, nil
+	}
+	return users[0], nil
+}
+
+func getUsersFilteredQuery(startUserName string, limit int, asc bool) sq.SelectBuilder {
+	fields := []string{"id", "data"}
+
+	s := sb.Select(fields...).From("user as user")
+	if asc {
+		s = s.OrderBy("user.name asc")
+	} else {
+		s = s.OrderBy("user.name desc")
+	}
+	if startUserName != "" {
+		if asc {
+			s = s.Where(sq.Gt{"user.name": startUserName})
+		} else {
+			s = s.Where(sq.Lt{"user.name": startUserName})
+		}
+	}
+	if limit > 0 {
+		s = s.Limit(uint64(limit))
+	}
+
+	return s
+}
+func (r *ReadDB) GetUsers(tx *db.Tx, startUserName string, limit int, asc bool) ([]*types.User, error) {
+	var users []*types.User
+
+	s := getUsersFilteredQuery(startUserName, limit, asc)
+	q, args, err := s.ToSql()
+	r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build query")
+	}
+
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	users, _, err = scanUsers(rows)
+	return users, err
+}
+
+func fetchUsers(tx *db.Tx, q string, args ...interface{}) ([]*types.User, []string, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	return scanUsers(rows)
+}
+
+func scanUser(rows *sql.Rows, additionalFields ...interface{}) (*types.User, string, error) {
+	var id string
+	var data []byte
+	if err := rows.Scan(&id, &data); err != nil {
+		return nil, "", errors.Wrap(err, "failed to scan rows")
+	}
+	user := types.User{}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &user); err != nil {
+			return nil, "", errors.Wrap(err, "failed to unmarshal user")
+		}
+	}
+
+	return &user, id, nil
+}
+
+func scanUsers(rows *sql.Rows) ([]*types.User, []string, error) {
+	users := []*types.User{}
+	ids := []string{}
+	for rows.Next() {
+		p, id, err := scanUser(rows)
+		if err != nil {
+			rows.Close()
+			return nil, nil, err
+		}
+		users = append(users, p)
+		ids = append(ids, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return users, ids, nil
+}
+
+type LinkedAccountUser struct {
+	ID     string
+	UserID string
+}
+
+func fetchLinkedAccounts(tx *db.Tx, q string, args ...interface{}) ([]*LinkedAccountUser, error) {
+	rows, err := tx.Query(q, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	linkedAccounts, err := scanLinkedAccounts(rows)
+
+	return linkedAccounts, err
+}
+
+func scanLinkedAccount(rows *sql.Rows, additionalFields ...interface{}) (*LinkedAccountUser, error) {
+	var id, userid string
+	if err := rows.Scan(&id, &userid); err != nil {
+		return nil, errors.Wrap(err, "failed to scan rows")
+	}
+
+	return &LinkedAccountUser{ID: id, UserID: userid}, nil
+}
+
+func scanLinkedAccounts(rows *sql.Rows) ([]*LinkedAccountUser, error) {
+	linkedAccounts := []*LinkedAccountUser{}
+	for rows.Next() {
+		linkedAccount, err := scanLinkedAccount(rows)
+		if err != nil {
+			rows.Close()
+			return nil, err
+		}
+		linkedAccounts = append(linkedAccounts, linkedAccount)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return linkedAccounts, nil
+}