*: implement remote source update

cmd/agola/cmd/remotesourceupdate.go

@@ -0,0 +1,103 @@
+// Copyright 2019 Sorint.lab
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/sorintlab/agola/internal/services/gateway/api"
+
+	"github.com/spf13/cobra"
+)
+
+var cmdRemoteSourceUpdate = &cobra.Command{
+	Use:   "update",
+	Short: "update a remotesource",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := remoteSourceUpdate(cmd, args); err != nil {
+			log.Fatalf("err: %v", err)
+		}
+	},
+}
+
+type remoteSourceUpdateOptions struct {
+	ref string
+
+	newName             string
+	apiURL              string
+	skipVerify          bool
+	oauth2ClientID      string
+	oauth2ClientSecret  string
+	sshHostKey          string
+	skipSSHHostKeyCheck bool
+}
+
+var remoteSourceUpdateOpts remoteSourceUpdateOptions
+
+func init() {
+	flags := cmdRemoteSourceUpdate.Flags()
+
+	flags.StringVarP(&remoteSourceUpdateOpts.ref, "ref", "", "", "current remotesource name or id")
+	flags.StringVarP(&remoteSourceUpdateOpts.newName, "new-name", "", "", "remotesource new name")
+	flags.StringVar(&remoteSourceUpdateOpts.apiURL, "api-url", "", "remotesource api url")
+	flags.BoolVarP(&remoteSourceUpdateOpts.skipVerify, "skip-verify", "", false, "skip remote source api tls certificate verification")
+	flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientID, "clientid", "", "remotesource oauth2 client id")
+	flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientSecret, "secret", "", "remotesource oauth2 secret")
+	flags.StringVar(&remoteSourceUpdateOpts.sshHostKey, "ssh-host-key", "", "remotesource ssh public host key")
+	flags.BoolVarP(&remoteSourceUpdateOpts.skipSSHHostKeyCheck, "skip-ssh-host-key-check", "s", false, "skip ssh host key check")
+
+	cmdRemoteSourceUpdate.MarkFlagRequired("ref")
+
+	cmdRemoteSource.AddCommand(cmdRemoteSourceUpdate)
+}
+
+func remoteSourceUpdate(cmd *cobra.Command, args []string) error {
+	gwclient := api.NewClient(gatewayURL, token)
+
+	req := &api.UpdateRemoteSourceRequest{}
+
+	flags := cmd.Flags()
+	if flags.Changed("new-name") {
+		req.Name = &remoteSourceUpdateOpts.newName
+	}
+	if flags.Changed("api-url") {
+		req.APIURL = &remoteSourceUpdateOpts.apiURL
+	}
+	if flags.Changed("skip-verify") {
+		req.SkipVerify = &remoteSourceUpdateOpts.skipVerify
+	}
+	if flags.Changed("clientid") {
+		req.Oauth2ClientID = &remoteSourceUpdateOpts.oauth2ClientID
+	}
+	if flags.Changed("secret") {
+		req.Oauth2ClientSecret = &remoteSourceUpdateOpts.oauth2ClientSecret
+	}
+	if flags.Changed("ssh-host-key") {
+		req.SSHHostKey = &remoteSourceUpdateOpts.sshHostKey
+	}
+	if flags.Changed("skip-ssh-host-key-check") {
+		req.SkipSSHHostKeyCheck = &remoteSourceUpdateOpts.skipSSHHostKeyCheck
+	}
+
+	log.Infof("updating remotesource")
+	remoteSource, _, err := gwclient.UpdateRemoteSource(context.TODO(), remoteSourceUpdateOpts.ref, req)
+	if err != nil {
+		return errors.Wrapf(err, "failed to update remotesource")
+	}
+	log.Infof("remotesource %s updated, ID: %s", remoteSource.Name, remoteSource.ID)
+
+	return nil
+}

internal/services/configstore/action/remotesource.go

@@ -27,40 +27,48 @@ import (
 	uuid "github.com/satori/go.uuid"
 )
 
-func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) {
+func (h *ActionHandler) ValidateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) error {
 	if remoteSource.Name == "" {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required"))
+		return util.NewErrBadRequest(errors.Errorf("remotesource name required"))
 	}
 	if !util.ValidateName(remoteSource.Name) {
-		return nil, util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name))
+		return util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name))
 	}
 
 	if remoteSource.Name == "" {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required"))
+		return util.NewErrBadRequest(errors.Errorf("remotesource name required"))
 	}
 	if remoteSource.APIURL == "" {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource api url required"))
+		return util.NewErrBadRequest(errors.Errorf("remotesource api url required"))
 	}
 	if remoteSource.Type == "" {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource type required"))
+		return util.NewErrBadRequest(errors.Errorf("remotesource type required"))
 	}
 	if remoteSource.AuthType == "" {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource auth type required"))
+		return util.NewErrBadRequest(errors.Errorf("remotesource auth type required"))
 	}
 
-	// validate if the remote source type supports the required auth type
+	// validate if the remotesource type supports the required auth type
 	if !types.SourceSupportsAuthType(types.RemoteSourceType(remoteSource.Type), types.RemoteSourceAuthType(remoteSource.AuthType)) {
-		return nil, util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType))
+		return util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType))
 	}
 	if remoteSource.AuthType == types.RemoteSourceAuthTypeOauth2 {
 		if remoteSource.Oauth2ClientID == "" {
-			return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2))
+			return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2))
 		}
 		if remoteSource.Oauth2ClientSecret == "" {
-			return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2))
+			return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2))
 		}
 	}
 
+	return nil
+}
+
+func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) {
+	if err := h.ValidateRemoteSource(ctx, remoteSource); err != nil {
+		return nil, err
+	}
+
 	var cgt *datamanager.ChangeGroupsUpdateToken
 	// changegroup is the remotesource name
 	cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + remoteSource.Name)}
@@ -79,7 +87,7 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty
 			return err
 		}
 		if u != nil {
-			return util.NewErrBadRequest(errors.Errorf("remoteSource %q already exists", u.Name))
+			return util.NewErrBadRequest(errors.Errorf("remotesource %q already exists", u.Name))
 		}
 		return nil
 	})
@@ -106,6 +114,61 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty
 	return remoteSource, err
 }
 
+type UpdateRemoteSourceRequest struct {
+	RemoteSourceRef string
+
+	RemoteSource *types.RemoteSource
+}
+
+func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) {
+	if err := h.ValidateRemoteSource(ctx, req.RemoteSource); err != nil {
+		return nil, err
+	}
+
+	var cgt *datamanager.ChangeGroupsUpdateToken
+
+	// must do all the checks in a single transaction to avoid concurrent changes
+	err := h.readDB.Do(func(tx *db.Tx) error {
+		var err error
+		// check duplicate remoteSource name
+		curRemoteSource, err := h.readDB.GetRemoteSourceByName(tx, req.RemoteSourceRef)
+		if err != nil {
+			return err
+		}
+		if curRemoteSource == nil {
+			return util.NewErrBadRequest(errors.Errorf("remotesource with ref %q doesn't exist", req.RemoteSourceRef))
+		}
+
+		// changegroup is the remotesource id and also name since we could change the
+		// name so concurrently updating on the new name
+		cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + req.RemoteSource.Name), util.EncodeSha256Hex("remotesourceid-" + req.RemoteSource.ID)}
+		cgt, err = h.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	rsj, err := json.Marshal(req.RemoteSource)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal remotesource")
+	}
+	actions := []*datamanager.Action{
+		{
+			ActionType: datamanager.ActionTypePut,
+			DataType:   string(types.ConfigTypeRemoteSource),
+			ID:         req.RemoteSource.ID,
+			Data:       rsj,
+		},
+	}
+
+	_, err = h.dm.WriteWal(ctx, actions, cgt)
+	return req.RemoteSource, err
+}
+
 func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName string) error {
 	var remoteSource *types.RemoteSource
 	var cgt *datamanager.ChangeGroupsUpdateToken
@@ -144,7 +207,7 @@ func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName
 		},
 	}
 
-	// changegroup is all the remote sources
+	// changegroup is all the remotesources
 	_, err = h.dm.WriteWal(ctx, actions, cgt)
 	return err
 }

internal/services/configstore/api/client.go

@@ -446,16 +446,27 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int,
 }
 
 func (c *Client) CreateRemoteSource(ctx context.Context, rs *types.RemoteSource) (*types.RemoteSource, *http.Response, error) {
-	uj, err := json.Marshal(rs)
+	rsj, err := json.Marshal(rs)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	rs = new(types.RemoteSource)
-	resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs)
+	resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs)
 	return rs, resp, err
 }
 
+func (c *Client) UpdateRemoteSource(ctx context.Context, remoteSourceRef string, remoteSource *types.RemoteSource) (*types.RemoteSource, *http.Response, error) {
+	rsj, err := json.Marshal(remoteSource)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	resRemoteSource := new(types.RemoteSource)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", url.PathEscape(remoteSourceRef)), nil, jsonContent, bytes.NewReader(rsj), resRemoteSource)
+	return resRemoteSource, resp, err
+}
+
 func (c *Client) DeleteRemoteSource(ctx context.Context, rsRef string) (*http.Response, error) {
 	return c.getResponse(ctx, "DELETE", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil)
 }

internal/services/configstore/api/remotesource.go

@@ -96,6 +96,44 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 	}
 }
 
+type UpdateRemoteSourceHandler struct {
+	log    *zap.SugaredLogger
+	ah     *action.ActionHandler
+	readDB *readdb.ReadDB
+}
+
+func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler {
+	return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah}
+}
+
+func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	rsRef := vars["remotesourceref"]
+
+	var remoteSource *types.RemoteSource
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&remoteSource); err != nil {
+		httpError(w, util.NewErrBadRequest(err))
+		return
+	}
+
+	areq := &action.UpdateRemoteSourceRequest{
+		RemoteSourceRef: rsRef,
+		RemoteSource:    remoteSource,
+	}
+	remoteSource, err := h.ah.UpdateRemoteSource(ctx, areq)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	if err := httpResponse(w, http.StatusCreated, remoteSource); err != nil {
+		h.log.Errorf("err: %+v", err)
+	}
+}
+
 type DeleteRemoteSourceHandler struct {
 	log *zap.SugaredLogger
 	ah  *action.ActionHandler

internal/services/configstore/configstore.go

@@ -172,6 +172,7 @@ func (s *Configstore) Run(ctx context.Context) error {
 	remoteSourceHandler := api.NewRemoteSourceHandler(logger, s.readDB)
 	remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, s.readDB)
 	createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, s.ah)
+	updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, s.ah)
 	deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, s.ah)
 
 	router := mux.NewRouter()
@@ -228,6 +229,7 @@ func (s *Configstore) Run(ctx context.Context) error {
 	apirouter.Handle("/remotesources/{remotesourceref}", remoteSourceHandler).Methods("GET")
 	apirouter.Handle("/remotesources", remoteSourcesHandler).Methods("GET")
 	apirouter.Handle("/remotesources", createRemoteSourceHandler).Methods("POST")
+	apirouter.Handle("/remotesources/{remotesourceref}", updateRemoteSourceHandler).Methods("PUT")
 	apirouter.Handle("/remotesources/{remotesourceref}", deleteRemoteSourceHandler).Methods("DELETE")
 
 	mainrouter := mux.NewRouter()

internal/services/gateway/action/remotesource.go

@@ -115,6 +115,60 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, req *CreateRemot
 	return rs, nil
 }
 
+type UpdateRemoteSourceRequest struct {
+	RemoteSourceRef string
+
+	Name                *string
+	APIURL              *string
+	SkipVerify          *bool
+	Oauth2ClientID      *string
+	Oauth2ClientSecret  *string
+	SSHHostKey          *string
+	SkipSSHHostKeyCheck *bool
+}
+
+func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) {
+	if !h.IsUserAdmin(ctx) {
+		return nil, errors.Errorf("user not admin")
+	}
+
+	rs, resp, err := h.configstoreClient.GetRemoteSource(ctx, req.RemoteSourceRef)
+	if err != nil {
+		return nil, ErrFromRemote(resp, err)
+	}
+
+	if req.Name != nil {
+		rs.Name = *req.Name
+	}
+	if req.APIURL != nil {
+		rs.APIURL = *req.APIURL
+	}
+	if req.SkipVerify != nil {
+		rs.SkipVerify = *req.SkipVerify
+	}
+	if req.Oauth2ClientID != nil {
+		rs.Oauth2ClientID = *req.Oauth2ClientID
+	}
+	if req.Oauth2ClientSecret != nil {
+		rs.Oauth2ClientSecret = *req.Oauth2ClientSecret
+	}
+	if req.SSHHostKey != nil {
+		rs.SSHHostKey = *req.SSHHostKey
+	}
+	if req.SkipSSHHostKeyCheck != nil {
+		rs.SkipSSHHostKeyCheck = *req.SkipSSHHostKeyCheck
+	}
+
+	h.log.Infof("updating remotesource")
+	rs, resp, err = h.configstoreClient.UpdateRemoteSource(ctx, req.RemoteSourceRef, rs)
+	if err != nil {
+		return nil, ErrFromRemote(resp, errors.Wrapf(err, "failed to update remotesource"))
+	}
+	h.log.Infof("remotesource %s updated", rs.Name)
+
+	return rs, nil
+}
+
 func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, rsRef string) error {
 	if !h.IsUserAdmin(ctx) {
 		return errors.Errorf("user not admin")

internal/services/gateway/api/client.go

@@ -370,13 +370,24 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int,
 }
 
 func (c *Client) CreateRemoteSource(ctx context.Context, req *CreateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) {
-	uj, err := json.Marshal(req)
+	rsj, err := json.Marshal(req)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	rs := new(RemoteSourceResponse)
-	resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs)
+	resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs)
+	return rs, resp, err
+}
+
+func (c *Client) UpdateRemoteSource(ctx context.Context, rsRef string, req *UpdateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) {
+	rsj, err := json.Marshal(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	rs := new(RemoteSourceResponse)
+	resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, bytes.NewReader(rsj), rs)
 	return rs, resp, err
 }
 

internal/services/gateway/api/remotesource.go

@@ -82,6 +82,60 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 	}
 }
 
+type UpdateRemoteSourceRequest struct {
+	Name                *string `json:"name"`
+	APIURL              *string `json:"apiurl"`
+	SkipVerify          *bool   `json:"skip_verify"`
+	Oauth2ClientID      *string `json:"oauth_2_client_id"`
+	Oauth2ClientSecret  *string `json:"oauth_2_client_secret"`
+	SSHHostKey          *string `json:"ssh_host_key"`
+	SkipSSHHostKeyCheck *bool   `json:"skip_ssh_host_key_check"`
+}
+
+type UpdateRemoteSourceHandler struct {
+	log *zap.SugaredLogger
+	ah  *action.ActionHandler
+}
+
+func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler {
+	return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah}
+}
+
+func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	rsRef := vars["remotesourceref"]
+
+	var req UpdateRemoteSourceRequest
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&req); err != nil {
+		httpError(w, util.NewErrBadRequest(err))
+		return
+	}
+
+	creq := &action.UpdateRemoteSourceRequest{
+		RemoteSourceRef: rsRef,
+
+		Name:                req.Name,
+		APIURL:              req.APIURL,
+		SkipVerify:          req.SkipVerify,
+		Oauth2ClientID:      req.Oauth2ClientID,
+		Oauth2ClientSecret:  req.Oauth2ClientSecret,
+		SSHHostKey:          req.SSHHostKey,
+		SkipSSHHostKeyCheck: req.SkipSSHHostKeyCheck,
+	}
+	rs, err := h.ah.UpdateRemoteSource(ctx, creq)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+
+	res := createRemoteSourceResponse(rs)
+	if err := httpResponse(w, http.StatusCreated, res); err != nil {
+		h.log.Errorf("err: %+v", err)
+	}
+}
+
 type RemoteSourceResponse struct {
 	ID       string `json:"id"`
 	Name     string `json:"name"`

internal/services/gateway/gateway.go

@@ -184,6 +184,7 @@ func (g *Gateway) Run(ctx context.Context) error {
 
 	remoteSourceHandler := api.NewRemoteSourceHandler(logger, g.ah)
 	createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, g.ah)
+	updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, g.ah)
 	remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, g.ah)
 	deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, g.ah)
 
@@ -268,6 +269,7 @@ func (g *Gateway) Run(ctx context.Context) error {
 
 	apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(remoteSourceHandler)).Methods("GET")
 	apirouter.Handle("/remotesources", authForcedHandler(createRemoteSourceHandler)).Methods("POST")
+	apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(updateRemoteSourceHandler)).Methods("PUT")
 	apirouter.Handle("/remotesources", authOptionalHandler(remoteSourcesHandler)).Methods("GET")
 	apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(deleteRemoteSourceHandler)).Methods("DELETE")