From 8f1225da76f7731a78e167050c358dd6b0e930d4 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Thu, 23 May 2019 10:29:03 +0200 Subject: [PATCH] *: implement remote source update --- cmd/agola/cmd/remotesourceupdate.go | 103 ++++++++++++++++++ .../configstore/action/remotesource.go | 89 ++++++++++++--- internal/services/configstore/api/client.go | 15 ++- .../services/configstore/api/remotesource.go | 38 +++++++ internal/services/configstore/configstore.go | 2 + .../services/gateway/action/remotesource.go | 54 +++++++++ internal/services/gateway/api/client.go | 15 ++- internal/services/gateway/api/remotesource.go | 54 +++++++++ internal/services/gateway/gateway.go | 2 + 9 files changed, 355 insertions(+), 17 deletions(-) create mode 100644 cmd/agola/cmd/remotesourceupdate.go diff --git a/cmd/agola/cmd/remotesourceupdate.go b/cmd/agola/cmd/remotesourceupdate.go new file mode 100644 index 0000000..32b9d7e --- /dev/null +++ b/cmd/agola/cmd/remotesourceupdate.go @@ -0,0 +1,103 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + + "github.com/pkg/errors" + "github.com/sorintlab/agola/internal/services/gateway/api" + + "github.com/spf13/cobra" +) + +var cmdRemoteSourceUpdate = &cobra.Command{ + Use: "update", + Short: "update a remotesource", + Run: func(cmd *cobra.Command, args []string) { + if err := remoteSourceUpdate(cmd, args); err != nil { + log.Fatalf("err: %v", err) + } + }, +} + +type remoteSourceUpdateOptions struct { + ref string + + newName string + apiURL string + skipVerify bool + oauth2ClientID string + oauth2ClientSecret string + sshHostKey string + skipSSHHostKeyCheck bool +} + +var remoteSourceUpdateOpts remoteSourceUpdateOptions + +func init() { + flags := cmdRemoteSourceUpdate.Flags() + + flags.StringVarP(&remoteSourceUpdateOpts.ref, "ref", "", "", "current remotesource name or id") + flags.StringVarP(&remoteSourceUpdateOpts.newName, "new-name", "", "", "remotesource new name") + flags.StringVar(&remoteSourceUpdateOpts.apiURL, "api-url", "", "remotesource api url") + flags.BoolVarP(&remoteSourceUpdateOpts.skipVerify, "skip-verify", "", false, "skip remote source api tls certificate verification") + flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientID, "clientid", "", "remotesource oauth2 client id") + flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientSecret, "secret", "", "remotesource oauth2 secret") + flags.StringVar(&remoteSourceUpdateOpts.sshHostKey, "ssh-host-key", "", "remotesource ssh public host key") + flags.BoolVarP(&remoteSourceUpdateOpts.skipSSHHostKeyCheck, "skip-ssh-host-key-check", "s", false, "skip ssh host key check") + + cmdRemoteSourceUpdate.MarkFlagRequired("ref") + + cmdRemoteSource.AddCommand(cmdRemoteSourceUpdate) +} + +func remoteSourceUpdate(cmd *cobra.Command, args []string) error { + gwclient := api.NewClient(gatewayURL, token) + + req := &api.UpdateRemoteSourceRequest{} + + flags := cmd.Flags() + if flags.Changed("new-name") { + req.Name = &remoteSourceUpdateOpts.newName + } + if flags.Changed("api-url") { + req.APIURL = &remoteSourceUpdateOpts.apiURL + } + if flags.Changed("skip-verify") { + req.SkipVerify = &remoteSourceUpdateOpts.skipVerify + } + if flags.Changed("clientid") { + req.Oauth2ClientID = &remoteSourceUpdateOpts.oauth2ClientID + } + if flags.Changed("secret") { + req.Oauth2ClientSecret = &remoteSourceUpdateOpts.oauth2ClientSecret + } + if flags.Changed("ssh-host-key") { + req.SSHHostKey = &remoteSourceUpdateOpts.sshHostKey + } + if flags.Changed("skip-ssh-host-key-check") { + req.SkipSSHHostKeyCheck = &remoteSourceUpdateOpts.skipSSHHostKeyCheck + } + + log.Infof("updating remotesource") + remoteSource, _, err := gwclient.UpdateRemoteSource(context.TODO(), remoteSourceUpdateOpts.ref, req) + if err != nil { + return errors.Wrapf(err, "failed to update remotesource") + } + log.Infof("remotesource %s updated, ID: %s", remoteSource.Name, remoteSource.ID) + + return nil +} diff --git a/internal/services/configstore/action/remotesource.go b/internal/services/configstore/action/remotesource.go index 8ad22e3..9a86eea 100644 --- a/internal/services/configstore/action/remotesource.go +++ b/internal/services/configstore/action/remotesource.go @@ -27,40 +27,48 @@ import ( uuid "github.com/satori/go.uuid" ) -func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) { +func (h *ActionHandler) ValidateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) error { if remoteSource.Name == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required")) + return util.NewErrBadRequest(errors.Errorf("remotesource name required")) } if !util.ValidateName(remoteSource.Name) { - return nil, util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name)) + return util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name)) } if remoteSource.Name == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required")) + return util.NewErrBadRequest(errors.Errorf("remotesource name required")) } if remoteSource.APIURL == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource api url required")) + return util.NewErrBadRequest(errors.Errorf("remotesource api url required")) } if remoteSource.Type == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource type required")) + return util.NewErrBadRequest(errors.Errorf("remotesource type required")) } if remoteSource.AuthType == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource auth type required")) + return util.NewErrBadRequest(errors.Errorf("remotesource auth type required")) } - // validate if the remote source type supports the required auth type + // validate if the remotesource type supports the required auth type if !types.SourceSupportsAuthType(types.RemoteSourceType(remoteSource.Type), types.RemoteSourceAuthType(remoteSource.AuthType)) { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType)) + return util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType)) } if remoteSource.AuthType == types.RemoteSourceAuthTypeOauth2 { if remoteSource.Oauth2ClientID == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2)) + return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2)) } if remoteSource.Oauth2ClientSecret == "" { - return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2)) + return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2)) } } + return nil +} + +func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) { + if err := h.ValidateRemoteSource(ctx, remoteSource); err != nil { + return nil, err + } + var cgt *datamanager.ChangeGroupsUpdateToken // changegroup is the remotesource name cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + remoteSource.Name)} @@ -79,7 +87,7 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty return err } if u != nil { - return util.NewErrBadRequest(errors.Errorf("remoteSource %q already exists", u.Name)) + return util.NewErrBadRequest(errors.Errorf("remotesource %q already exists", u.Name)) } return nil }) @@ -106,6 +114,61 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty return remoteSource, err } +type UpdateRemoteSourceRequest struct { + RemoteSourceRef string + + RemoteSource *types.RemoteSource +} + +func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) { + if err := h.ValidateRemoteSource(ctx, req.RemoteSource); err != nil { + return nil, err + } + + var cgt *datamanager.ChangeGroupsUpdateToken + + // must do all the checks in a single transaction to avoid concurrent changes + err := h.readDB.Do(func(tx *db.Tx) error { + var err error + // check duplicate remoteSource name + curRemoteSource, err := h.readDB.GetRemoteSourceByName(tx, req.RemoteSourceRef) + if err != nil { + return err + } + if curRemoteSource == nil { + return util.NewErrBadRequest(errors.Errorf("remotesource with ref %q doesn't exist", req.RemoteSourceRef)) + } + + // changegroup is the remotesource id and also name since we could change the + // name so concurrently updating on the new name + cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + req.RemoteSource.Name), util.EncodeSha256Hex("remotesourceid-" + req.RemoteSource.ID)} + cgt, err = h.readDB.GetChangeGroupsUpdateTokens(tx, cgNames) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + + rsj, err := json.Marshal(req.RemoteSource) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal remotesource") + } + actions := []*datamanager.Action{ + { + ActionType: datamanager.ActionTypePut, + DataType: string(types.ConfigTypeRemoteSource), + ID: req.RemoteSource.ID, + Data: rsj, + }, + } + + _, err = h.dm.WriteWal(ctx, actions, cgt) + return req.RemoteSource, err +} + func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName string) error { var remoteSource *types.RemoteSource var cgt *datamanager.ChangeGroupsUpdateToken @@ -144,7 +207,7 @@ func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName }, } - // changegroup is all the remote sources + // changegroup is all the remotesources _, err = h.dm.WriteWal(ctx, actions, cgt) return err } diff --git a/internal/services/configstore/api/client.go b/internal/services/configstore/api/client.go index c5c7172..cc9cf90 100644 --- a/internal/services/configstore/api/client.go +++ b/internal/services/configstore/api/client.go @@ -446,16 +446,27 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, } func (c *Client) CreateRemoteSource(ctx context.Context, rs *types.RemoteSource) (*types.RemoteSource, *http.Response, error) { - uj, err := json.Marshal(rs) + rsj, err := json.Marshal(rs) if err != nil { return nil, nil, err } rs = new(types.RemoteSource) - resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs) + resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs) return rs, resp, err } +func (c *Client) UpdateRemoteSource(ctx context.Context, remoteSourceRef string, remoteSource *types.RemoteSource) (*types.RemoteSource, *http.Response, error) { + rsj, err := json.Marshal(remoteSource) + if err != nil { + return nil, nil, err + } + + resRemoteSource := new(types.RemoteSource) + resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", url.PathEscape(remoteSourceRef)), nil, jsonContent, bytes.NewReader(rsj), resRemoteSource) + return resRemoteSource, resp, err +} + func (c *Client) DeleteRemoteSource(ctx context.Context, rsRef string) (*http.Response, error) { return c.getResponse(ctx, "DELETE", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil) } diff --git a/internal/services/configstore/api/remotesource.go b/internal/services/configstore/api/remotesource.go index dd1fd46..6d30dfc 100644 --- a/internal/services/configstore/api/remotesource.go +++ b/internal/services/configstore/api/remotesource.go @@ -96,6 +96,44 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req } } +type UpdateRemoteSourceHandler struct { + log *zap.SugaredLogger + ah *action.ActionHandler + readDB *readdb.ReadDB +} + +func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler { + return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah} +} + +func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + rsRef := vars["remotesourceref"] + + var remoteSource *types.RemoteSource + d := json.NewDecoder(r.Body) + if err := d.Decode(&remoteSource); err != nil { + httpError(w, util.NewErrBadRequest(err)) + return + } + + areq := &action.UpdateRemoteSourceRequest{ + RemoteSourceRef: rsRef, + RemoteSource: remoteSource, + } + remoteSource, err := h.ah.UpdateRemoteSource(ctx, areq) + if httpError(w, err) { + h.log.Errorf("err: %+v", err) + return + } + + if err := httpResponse(w, http.StatusCreated, remoteSource); err != nil { + h.log.Errorf("err: %+v", err) + } +} + type DeleteRemoteSourceHandler struct { log *zap.SugaredLogger ah *action.ActionHandler diff --git a/internal/services/configstore/configstore.go b/internal/services/configstore/configstore.go index 46f50b5..353a202 100644 --- a/internal/services/configstore/configstore.go +++ b/internal/services/configstore/configstore.go @@ -172,6 +172,7 @@ func (s *Configstore) Run(ctx context.Context) error { remoteSourceHandler := api.NewRemoteSourceHandler(logger, s.readDB) remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, s.readDB) createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, s.ah) + updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, s.ah) deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, s.ah) router := mux.NewRouter() @@ -228,6 +229,7 @@ func (s *Configstore) Run(ctx context.Context) error { apirouter.Handle("/remotesources/{remotesourceref}", remoteSourceHandler).Methods("GET") apirouter.Handle("/remotesources", remoteSourcesHandler).Methods("GET") apirouter.Handle("/remotesources", createRemoteSourceHandler).Methods("POST") + apirouter.Handle("/remotesources/{remotesourceref}", updateRemoteSourceHandler).Methods("PUT") apirouter.Handle("/remotesources/{remotesourceref}", deleteRemoteSourceHandler).Methods("DELETE") mainrouter := mux.NewRouter() diff --git a/internal/services/gateway/action/remotesource.go b/internal/services/gateway/action/remotesource.go index 7c2e34f..b4c1c6b 100644 --- a/internal/services/gateway/action/remotesource.go +++ b/internal/services/gateway/action/remotesource.go @@ -115,6 +115,60 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, req *CreateRemot return rs, nil } +type UpdateRemoteSourceRequest struct { + RemoteSourceRef string + + Name *string + APIURL *string + SkipVerify *bool + Oauth2ClientID *string + Oauth2ClientSecret *string + SSHHostKey *string + SkipSSHHostKeyCheck *bool +} + +func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) { + if !h.IsUserAdmin(ctx) { + return nil, errors.Errorf("user not admin") + } + + rs, resp, err := h.configstoreClient.GetRemoteSource(ctx, req.RemoteSourceRef) + if err != nil { + return nil, ErrFromRemote(resp, err) + } + + if req.Name != nil { + rs.Name = *req.Name + } + if req.APIURL != nil { + rs.APIURL = *req.APIURL + } + if req.SkipVerify != nil { + rs.SkipVerify = *req.SkipVerify + } + if req.Oauth2ClientID != nil { + rs.Oauth2ClientID = *req.Oauth2ClientID + } + if req.Oauth2ClientSecret != nil { + rs.Oauth2ClientSecret = *req.Oauth2ClientSecret + } + if req.SSHHostKey != nil { + rs.SSHHostKey = *req.SSHHostKey + } + if req.SkipSSHHostKeyCheck != nil { + rs.SkipSSHHostKeyCheck = *req.SkipSSHHostKeyCheck + } + + h.log.Infof("updating remotesource") + rs, resp, err = h.configstoreClient.UpdateRemoteSource(ctx, req.RemoteSourceRef, rs) + if err != nil { + return nil, ErrFromRemote(resp, errors.Wrapf(err, "failed to update remotesource")) + } + h.log.Infof("remotesource %s updated", rs.Name) + + return rs, nil +} + func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, rsRef string) error { if !h.IsUserAdmin(ctx) { return errors.Errorf("user not admin") diff --git a/internal/services/gateway/api/client.go b/internal/services/gateway/api/client.go index d05b0f1..70ccee7 100644 --- a/internal/services/gateway/api/client.go +++ b/internal/services/gateway/api/client.go @@ -370,13 +370,24 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, } func (c *Client) CreateRemoteSource(ctx context.Context, req *CreateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) { - uj, err := json.Marshal(req) + rsj, err := json.Marshal(req) if err != nil { return nil, nil, err } rs := new(RemoteSourceResponse) - resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs) + resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs) + return rs, resp, err +} + +func (c *Client) UpdateRemoteSource(ctx context.Context, rsRef string, req *UpdateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) { + rsj, err := json.Marshal(req) + if err != nil { + return nil, nil, err + } + + rs := new(RemoteSourceResponse) + resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, bytes.NewReader(rsj), rs) return rs, resp, err } diff --git a/internal/services/gateway/api/remotesource.go b/internal/services/gateway/api/remotesource.go index 5c64812..90d814e 100644 --- a/internal/services/gateway/api/remotesource.go +++ b/internal/services/gateway/api/remotesource.go @@ -82,6 +82,60 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req } } +type UpdateRemoteSourceRequest struct { + Name *string `json:"name"` + APIURL *string `json:"apiurl"` + SkipVerify *bool `json:"skip_verify"` + Oauth2ClientID *string `json:"oauth_2_client_id"` + Oauth2ClientSecret *string `json:"oauth_2_client_secret"` + SSHHostKey *string `json:"ssh_host_key"` + SkipSSHHostKeyCheck *bool `json:"skip_ssh_host_key_check"` +} + +type UpdateRemoteSourceHandler struct { + log *zap.SugaredLogger + ah *action.ActionHandler +} + +func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler { + return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah} +} + +func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + rsRef := vars["remotesourceref"] + + var req UpdateRemoteSourceRequest + d := json.NewDecoder(r.Body) + if err := d.Decode(&req); err != nil { + httpError(w, util.NewErrBadRequest(err)) + return + } + + creq := &action.UpdateRemoteSourceRequest{ + RemoteSourceRef: rsRef, + + Name: req.Name, + APIURL: req.APIURL, + SkipVerify: req.SkipVerify, + Oauth2ClientID: req.Oauth2ClientID, + Oauth2ClientSecret: req.Oauth2ClientSecret, + SSHHostKey: req.SSHHostKey, + SkipSSHHostKeyCheck: req.SkipSSHHostKeyCheck, + } + rs, err := h.ah.UpdateRemoteSource(ctx, creq) + if httpError(w, err) { + h.log.Errorf("err: %+v", err) + return + } + + res := createRemoteSourceResponse(rs) + if err := httpResponse(w, http.StatusCreated, res); err != nil { + h.log.Errorf("err: %+v", err) + } +} + type RemoteSourceResponse struct { ID string `json:"id"` Name string `json:"name"` diff --git a/internal/services/gateway/gateway.go b/internal/services/gateway/gateway.go index a8c7d94..d88960b 100644 --- a/internal/services/gateway/gateway.go +++ b/internal/services/gateway/gateway.go @@ -184,6 +184,7 @@ func (g *Gateway) Run(ctx context.Context) error { remoteSourceHandler := api.NewRemoteSourceHandler(logger, g.ah) createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, g.ah) + updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, g.ah) remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, g.ah) deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, g.ah) @@ -268,6 +269,7 @@ func (g *Gateway) Run(ctx context.Context) error { apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(remoteSourceHandler)).Methods("GET") apirouter.Handle("/remotesources", authForcedHandler(createRemoteSourceHandler)).Methods("POST") + apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(updateRemoteSourceHandler)).Methods("PUT") apirouter.Handle("/remotesources", authOptionalHandler(remoteSourcesHandler)).Methods("GET") apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(deleteRemoteSourceHandler)).Methods("DELETE")