a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

*: implement remote source update

This commit is contained in:
Simone Gotti 2019-05-23 10:29:03 +02:00
parent 933dfae658
commit 8f1225da76
9 changed files with 355 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
cmd/agola/cmd/remotesourceupdate.go Normal file
View File

@ -0,0 +1,103 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/services/gateway/api"
"github.com/spf13/cobra"
)
var cmdRemoteSourceUpdate = &cobra.Command{
Use: "update",
Short: "update a remotesource",
Run: func(cmd *cobra.Command, args []string) {
if err := remoteSourceUpdate(cmd, args); err != nil {
log.Fatalf("err: %v", err)
}
},
}
type remoteSourceUpdateOptions struct {
ref string
newName string
apiURL string
skipVerify bool
oauth2ClientID string
oauth2ClientSecret string
sshHostKey string
skipSSHHostKeyCheck bool
}
var remoteSourceUpdateOpts remoteSourceUpdateOptions
func init() {
flags := cmdRemoteSourceUpdate.Flags()
flags.StringVarP(&remoteSourceUpdateOpts.ref, "ref", "", "", "current remotesource name or id")
flags.StringVarP(&remoteSourceUpdateOpts.newName, "new-name", "", "", "remotesource new name")
flags.StringVar(&remoteSourceUpdateOpts.apiURL, "api-url", "", "remotesource api url")
flags.BoolVarP(&remoteSourceUpdateOpts.skipVerify, "skip-verify", "", false, "skip remote source api tls certificate verification")
flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientID, "clientid", "", "remotesource oauth2 client id")
flags.StringVar(&remoteSourceUpdateOpts.oauth2ClientSecret, "secret", "", "remotesource oauth2 secret")
flags.StringVar(&remoteSourceUpdateOpts.sshHostKey, "ssh-host-key", "", "remotesource ssh public host key")
flags.BoolVarP(&remoteSourceUpdateOpts.skipSSHHostKeyCheck, "skip-ssh-host-key-check", "s", false, "skip ssh host key check")
cmdRemoteSourceUpdate.MarkFlagRequired("ref")
cmdRemoteSource.AddCommand(cmdRemoteSourceUpdate)
}
func remoteSourceUpdate(cmd *cobra.Command, args []string) error {
gwclient := api.NewClient(gatewayURL, token)
req := &api.UpdateRemoteSourceRequest{}
flags := cmd.Flags()
if flags.Changed("new-name") {
req.Name = &remoteSourceUpdateOpts.newName
}
if flags.Changed("api-url") {
req.APIURL = &remoteSourceUpdateOpts.apiURL
}
if flags.Changed("skip-verify") {
req.SkipVerify = &remoteSourceUpdateOpts.skipVerify
}
if flags.Changed("clientid") {
req.Oauth2ClientID = &remoteSourceUpdateOpts.oauth2ClientID
}
if flags.Changed("secret") {
req.Oauth2ClientSecret = &remoteSourceUpdateOpts.oauth2ClientSecret
}
if flags.Changed("ssh-host-key") {
req.SSHHostKey = &remoteSourceUpdateOpts.sshHostKey
}
if flags.Changed("skip-ssh-host-key-check") {
req.SkipSSHHostKeyCheck = &remoteSourceUpdateOpts.skipSSHHostKeyCheck
}
log.Infof("updating remotesource")
remoteSource, _, err := gwclient.UpdateRemoteSource(context.TODO(), remoteSourceUpdateOpts.ref, req)
if err != nil {
return errors.Wrapf(err, "failed to update remotesource")
}
log.Infof("remotesource %s updated, ID: %s", remoteSource.Name, remoteSource.ID)
return nil
}

89
internal/services/configstore/action/remotesource.go
View File

@ -27,40 +27,48 @@ import (
uuid "github.com/satori/go.uuid"
)
func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) {
func (h *ActionHandler) ValidateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) error {
if remoteSource.Name == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required"))
return util.NewErrBadRequest(errors.Errorf("remotesource name required"))
}
if !util.ValidateName(remoteSource.Name) {
return nil, util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name))
return util.NewErrBadRequest(errors.Errorf("invalid remotesource name %q", remoteSource.Name))
}
if remoteSource.Name == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource name required"))
return util.NewErrBadRequest(errors.Errorf("remotesource name required"))
}
if remoteSource.APIURL == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource api url required"))
return util.NewErrBadRequest(errors.Errorf("remotesource api url required"))
}
if remoteSource.Type == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource type required"))
return util.NewErrBadRequest(errors.Errorf("remotesource type required"))
}
if remoteSource.AuthType == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource auth type required"))
return util.NewErrBadRequest(errors.Errorf("remotesource auth type required"))
}
// validate if the remote source type supports the required auth type
// validate if the remotesource type supports the required auth type
if !types.SourceSupportsAuthType(types.RemoteSourceType(remoteSource.Type), types.RemoteSourceAuthType(remoteSource.AuthType)) {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType))
return util.NewErrBadRequest(errors.Errorf("remotesource type %q doesn't support auth type %q", remoteSource.Type, remoteSource.AuthType))
}
if remoteSource.AuthType == types.RemoteSourceAuthTypeOauth2 {
if remoteSource.Oauth2ClientID == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2))
return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientid required for auth type %q", types.RemoteSourceAuthTypeOauth2))
}
if remoteSource.Oauth2ClientSecret == "" {
return nil, util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2))
return util.NewErrBadRequest(errors.Errorf("remotesource oauth2clientsecret required for auth type %q", types.RemoteSourceAuthTypeOauth2))
}
}
return nil
}
func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *types.RemoteSource) (*types.RemoteSource, error) {
if err := h.ValidateRemoteSource(ctx, remoteSource); err != nil {
return nil, err
}
var cgt *datamanager.ChangeGroupsUpdateToken
// changegroup is the remotesource name
cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + remoteSource.Name)}
@ -79,7 +87,7 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty
return err
}
if u != nil {
return util.NewErrBadRequest(errors.Errorf("remoteSource %q already exists", u.Name))
return util.NewErrBadRequest(errors.Errorf("remotesource %q already exists", u.Name))
}
return nil
})
@ -106,6 +114,61 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, remoteSource *ty
return remoteSource, err
}
type UpdateRemoteSourceRequest struct {
RemoteSourceRef string
RemoteSource *types.RemoteSource
}
func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) {
if err := h.ValidateRemoteSource(ctx, req.RemoteSource); err != nil {
return nil, err
}
var cgt *datamanager.ChangeGroupsUpdateToken
// must do all the checks in a single transaction to avoid concurrent changes
err := h.readDB.Do(func(tx *db.Tx) error {
var err error
// check duplicate remoteSource name
curRemoteSource, err := h.readDB.GetRemoteSourceByName(tx, req.RemoteSourceRef)
if err != nil {
return err
}
if curRemoteSource == nil {
return util.NewErrBadRequest(errors.Errorf("remotesource with ref %q doesn't exist", req.RemoteSourceRef))
}
// changegroup is the remotesource id and also name since we could change the
// name so concurrently updating on the new name
cgNames := []string{util.EncodeSha256Hex("remotesourcename-" + req.RemoteSource.Name), util.EncodeSha256Hex("remotesourceid-" + req.RemoteSource.ID)}
cgt, err = h.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
if err != nil {
return err
}
return nil
})
if err != nil {
return nil, err
}
rsj, err := json.Marshal(req.RemoteSource)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal remotesource")
}
actions := []*datamanager.Action{
{
ActionType: datamanager.ActionTypePut,
DataType: string(types.ConfigTypeRemoteSource),
ID: req.RemoteSource.ID,
Data: rsj,
},
}
_, err = h.dm.WriteWal(ctx, actions, cgt)
return req.RemoteSource, err
}
func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName string) error {
var remoteSource *types.RemoteSource
var cgt *datamanager.ChangeGroupsUpdateToken
@ -144,7 +207,7 @@ func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, remoteSourceName
},
}
// changegroup is all the remote sources
// changegroup is all the remotesources
_, err = h.dm.WriteWal(ctx, actions, cgt)
return err
}

15
internal/services/configstore/api/client.go
View File

@ -446,16 +446,27 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int,
}
func (c *Client) CreateRemoteSource(ctx context.Context, rs *types.RemoteSource) (*types.RemoteSource, *http.Response, error) {
uj, err := json.Marshal(rs)
rsj, err := json.Marshal(rs)
if err != nil {
return nil, nil, err
}
rs = new(types.RemoteSource)
resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs)
resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs)
return rs, resp, err
}
func (c *Client) UpdateRemoteSource(ctx context.Context, remoteSourceRef string, remoteSource *types.RemoteSource) (*types.RemoteSource, *http.Response, error) {
rsj, err := json.Marshal(remoteSource)
if err != nil {
return nil, nil, err
}
resRemoteSource := new(types.RemoteSource)
resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", url.PathEscape(remoteSourceRef)), nil, jsonContent, bytes.NewReader(rsj), resRemoteSource)
return resRemoteSource, resp, err
}
func (c *Client) DeleteRemoteSource(ctx context.Context, rsRef string) (*http.Response, error) {
return c.getResponse(ctx, "DELETE", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil)
}

38
internal/services/configstore/api/remotesource.go
View File

@ -96,6 +96,44 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
}
}
type UpdateRemoteSourceHandler struct {
log *zap.SugaredLogger
ah *action.ActionHandler
readDB *readdb.ReadDB
}
func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler {
return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah}
}
func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
rsRef := vars["remotesourceref"]
var remoteSource *types.RemoteSource
d := json.NewDecoder(r.Body)
if err := d.Decode(&remoteSource); err != nil {
httpError(w, util.NewErrBadRequest(err))
return
}
areq := &action.UpdateRemoteSourceRequest{
RemoteSourceRef: rsRef,
RemoteSource: remoteSource,
}
remoteSource, err := h.ah.UpdateRemoteSource(ctx, areq)
if httpError(w, err) {
h.log.Errorf("err: %+v", err)
return
}
if err := httpResponse(w, http.StatusCreated, remoteSource); err != nil {
h.log.Errorf("err: %+v", err)
}
}
type DeleteRemoteSourceHandler struct {
log *zap.SugaredLogger
ah *action.ActionHandler

2
internal/services/configstore/configstore.go
View File

@ -172,6 +172,7 @@ func (s *Configstore) Run(ctx context.Context) error {
remoteSourceHandler := api.NewRemoteSourceHandler(logger, s.readDB)
remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, s.readDB)
createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, s.ah)
updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, s.ah)
deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, s.ah)
router := mux.NewRouter()
@ -228,6 +229,7 @@ func (s *Configstore) Run(ctx context.Context) error {
apirouter.Handle("/remotesources/{remotesourceref}", remoteSourceHandler).Methods("GET")
apirouter.Handle("/remotesources", remoteSourcesHandler).Methods("GET")
apirouter.Handle("/remotesources", createRemoteSourceHandler).Methods("POST")
apirouter.Handle("/remotesources/{remotesourceref}", updateRemoteSourceHandler).Methods("PUT")
apirouter.Handle("/remotesources/{remotesourceref}", deleteRemoteSourceHandler).Methods("DELETE")
mainrouter := mux.NewRouter()

54
internal/services/gateway/action/remotesource.go
View File

@ -115,6 +115,60 @@ func (h *ActionHandler) CreateRemoteSource(ctx context.Context, req *CreateRemot
return rs, nil
}
type UpdateRemoteSourceRequest struct {
RemoteSourceRef string
Name *string
APIURL *string
SkipVerify *bool
Oauth2ClientID *string
Oauth2ClientSecret *string
SSHHostKey *string
SkipSSHHostKeyCheck *bool
}
func (h *ActionHandler) UpdateRemoteSource(ctx context.Context, req *UpdateRemoteSourceRequest) (*types.RemoteSource, error) {
if !h.IsUserAdmin(ctx) {
return nil, errors.Errorf("user not admin")
}
rs, resp, err := h.configstoreClient.GetRemoteSource(ctx, req.RemoteSourceRef)
if err != nil {
return nil, ErrFromRemote(resp, err)
}
if req.Name != nil {
rs.Name = *req.Name
}
if req.APIURL != nil {
rs.APIURL = *req.APIURL
}
if req.SkipVerify != nil {
rs.SkipVerify = *req.SkipVerify
}
if req.Oauth2ClientID != nil {
rs.Oauth2ClientID = *req.Oauth2ClientID
}
if req.Oauth2ClientSecret != nil {
rs.Oauth2ClientSecret = *req.Oauth2ClientSecret
}
if req.SSHHostKey != nil {
rs.SSHHostKey = *req.SSHHostKey
}
if req.SkipSSHHostKeyCheck != nil {
rs.SkipSSHHostKeyCheck = *req.SkipSSHHostKeyCheck
}
h.log.Infof("updating remotesource")
rs, resp, err = h.configstoreClient.UpdateRemoteSource(ctx, req.RemoteSourceRef, rs)
if err != nil {
return nil, ErrFromRemote(resp, errors.Wrapf(err, "failed to update remotesource"))
}
h.log.Infof("remotesource %s updated", rs.Name)
return rs, nil
}
func (h *ActionHandler) DeleteRemoteSource(ctx context.Context, rsRef string) error {
if !h.IsUserAdmin(ctx) {
return errors.Errorf("user not admin")

15
internal/services/gateway/api/client.go
View File

@ -370,13 +370,24 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int,
}
func (c *Client) CreateRemoteSource(ctx context.Context, req *CreateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) {
uj, err := json.Marshal(req)
rsj, err := json.Marshal(req)
if err != nil {
return nil, nil, err
}
rs := new(RemoteSourceResponse)
resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(uj), rs)
resp, err := c.getParsedResponse(ctx, "POST", "/remotesources", nil, jsonContent, bytes.NewReader(rsj), rs)
return rs, resp, err
}
func (c *Client) UpdateRemoteSource(ctx context.Context, rsRef string, req *UpdateRemoteSourceRequest) (*RemoteSourceResponse, *http.Response, error) {
rsj, err := json.Marshal(req)
if err != nil {
return nil, nil, err
}
rs := new(RemoteSourceResponse)
resp, err := c.getParsedResponse(ctx, "PUT", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, bytes.NewReader(rsj), rs)
return rs, resp, err
}

54
internal/services/gateway/api/remotesource.go
View File

@ -82,6 +82,60 @@ func (h *CreateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
}
}
type UpdateRemoteSourceRequest struct {
Name *string `json:"name"`
APIURL *string `json:"apiurl"`
SkipVerify *bool `json:"skip_verify"`
Oauth2ClientID *string `json:"oauth_2_client_id"`
Oauth2ClientSecret *string `json:"oauth_2_client_secret"`
SSHHostKey *string `json:"ssh_host_key"`
SkipSSHHostKeyCheck *bool `json:"skip_ssh_host_key_check"`
}
type UpdateRemoteSourceHandler struct {
log *zap.SugaredLogger
ah *action.ActionHandler
}
func NewUpdateRemoteSourceHandler(logger *zap.Logger, ah *action.ActionHandler) *UpdateRemoteSourceHandler {
return &UpdateRemoteSourceHandler{log: logger.Sugar(), ah: ah}
}
func (h *UpdateRemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
rsRef := vars["remotesourceref"]
var req UpdateRemoteSourceRequest
d := json.NewDecoder(r.Body)
if err := d.Decode(&req); err != nil {
httpError(w, util.NewErrBadRequest(err))
return
}
creq := &action.UpdateRemoteSourceRequest{
RemoteSourceRef: rsRef,
Name: req.Name,
APIURL: req.APIURL,
SkipVerify: req.SkipVerify,
Oauth2ClientID: req.Oauth2ClientID,
Oauth2ClientSecret: req.Oauth2ClientSecret,
SSHHostKey: req.SSHHostKey,
SkipSSHHostKeyCheck: req.SkipSSHHostKeyCheck,
}
rs, err := h.ah.UpdateRemoteSource(ctx, creq)
if httpError(w, err) {
h.log.Errorf("err: %+v", err)
return
}
res := createRemoteSourceResponse(rs)
if err := httpResponse(w, http.StatusCreated, res); err != nil {
h.log.Errorf("err: %+v", err)
}
}
type RemoteSourceResponse struct {
ID string `json:"id"`
Name string `json:"name"`

2
internal/services/gateway/gateway.go
View File

@ -184,6 +184,7 @@ func (g *Gateway) Run(ctx context.Context) error {
remoteSourceHandler := api.NewRemoteSourceHandler(logger, g.ah)
createRemoteSourceHandler := api.NewCreateRemoteSourceHandler(logger, g.ah)
updateRemoteSourceHandler := api.NewUpdateRemoteSourceHandler(logger, g.ah)
remoteSourcesHandler := api.NewRemoteSourcesHandler(logger, g.ah)
deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, g.ah)
@ -268,6 +269,7 @@ func (g *Gateway) Run(ctx context.Context) error {
apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(remoteSourceHandler)).Methods("GET")
apirouter.Handle("/remotesources", authForcedHandler(createRemoteSourceHandler)).Methods("POST")
apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(updateRemoteSourceHandler)).Methods("PUT")
apirouter.Handle("/remotesources", authOptionalHandler(remoteSourcesHandler)).Methods("GET")
apirouter.Handle("/remotesources/{remotesourceref}", authForcedHandler(deleteRemoteSourceHandler)).Methods("DELETE")