a/agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial project group impl

Browse Source 
and related api updated
This commit is contained in:
Simone Gotti 2019-03-14 14:36:18 +01:00
parent 4cacae198f
commit 8b92b6f55c
35 changed files with 1599 additions and 685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/agola/cmd/projectcreate.go
View File

@ -19,7 +19,6 @@ import (
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/services/gateway/api"
"github.com/sorintlab/agola/internal/services/types"
"github.com/spf13/cobra"
)
@ -36,7 +35,7 @@ var cmdProjectCreate = &cobra.Command{
type projectCreateOptions struct {
name string
organizationName string
parentPath string
repoURL string
remoteSourceName string
skipSSHHostKeyCheck bool
@ -51,9 +50,10 @@ func init() {
flags.StringVar(&projectCreateOpts.repoURL, "repo-url", "", "repository url")
flags.StringVar(&projectCreateOpts.remoteSourceName, "remote-source", "", "remote source name")
flags.BoolVarP(&projectCreateOpts.skipSSHHostKeyCheck, "skip-ssh-host-key-check", "s", false, "skip ssh host key check")
flags.StringVar(&projectCreateOpts.organizationName, "orgname", "", "organization name where the project should be created")
flags.StringVar(&projectCreateOpts.parentPath, "parent", "", `parent project group path (i.e "org/org01" for root project group in org01, "/user/user01/group01/subgroub01") or project group id where the project should be created`)
cmdProjectCreate.MarkFlagRequired("name")
cmdProjectCreate.MarkFlagRequired("parent")
cmdProjectCreate.MarkFlagRequired("repo-url")
cmdProjectCreate.MarkFlagRequired("remote-source")
@ -65,6 +65,7 @@ func projectCreate(cmd *cobra.Command, args []string) error {
req := &api.CreateProjectRequest{
Name: projectCreateOpts.name,
ParentID: projectCreateOpts.parentPath,
RepoURL: projectCreateOpts.repoURL,
RemoteSourceName: projectCreateOpts.remoteSourceName,
SkipSSHHostKeyCheck: projectCreateOpts.skipSSHHostKeyCheck,
@ -72,13 +73,7 @@ func projectCreate(cmd *cobra.Command, args []string) error {
log.Infof("creating project")
var project *types.Project
var err error
if projectCreateOpts.organizationName != "" {
project, _, err = gwclient.CreateOrgProject(context.TODO(), projectCreateOpts.organizationName, req)
} else {
project, _, err = gwclient.CreateCurrentUserProject(context.TODO(), req)
}
project, _, err := gwclient.CreateProject(context.TODO(), req)
if err != nil {
return errors.Wrapf(err, "failed to create project")
}

28
cmd/agola/cmd/projectgroup.go Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"github.com/spf13/cobra"
)
var cmdProjectGroup = &cobra.Command{
Use: "projectgroup",
Short: "projectgroup",
}
func init() {
cmdAgola.AddCommand(cmdProjectGroup)
}

72
cmd/agola/cmd/projectgroupcreate.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/services/gateway/api"
"github.com/spf13/cobra"
)
var cmdProjectGroupCreate = &cobra.Command{
Use: "create",
Short: "create a project",
Run: func(cmd *cobra.Command, args []string) {
if err := projectGroupCreate(cmd, args); err != nil {
log.Fatalf("err: %v", err)
}
},
}
type projectGroupCreateOptions struct {
name string
parentPath string
}
var projectGroupCreateOpts projectGroupCreateOptions
func init() {
flags := cmdProjectGroupCreate.Flags()
flags.StringVarP(&projectGroupCreateOpts.name, "name", "n", "", "project name")
flags.StringVar(&projectGroupCreateOpts.parentPath, "parent", "", `parent project group path (i.e "org/org01" for root project group in org01, "/user/user01/group01/subgroub01") or project group id where the project group should be created`)
cmdProjectGroupCreate.MarkFlagRequired("name")
cmdProjectGroupCreate.MarkFlagRequired("parent")
cmdProjectGroup.AddCommand(cmdProjectGroupCreate)
}
func projectGroupCreate(cmd *cobra.Command, args []string) error {
gwclient := api.NewClient(gatewayURL, token)
req := &api.CreateProjectGroupRequest{
Name: projectGroupCreateOpts.name,
ParentID: projectGroupCreateOpts.parentPath,
}
log.Infof("creating project group")
project, _, err := gwclient.CreateProjectGroup(context.TODO(), req)
if err != nil {
return errors.Wrapf(err, "failed to create project")
}
log.Infof("project %s created, ID: %s", project.Name, project.ID)
return nil
}

16
cmd/agola/cmd/projectlist.go
View File

@ -33,8 +33,7 @@ var cmdProjectList = &cobra.Command{
}
type projectListOptions struct {
limit int
start string
parentPath string
}
var projectListOpts projectListOptions
@ -42,14 +41,15 @@ var projectListOpts projectListOptions
func init() {
flags := cmdProjectList.PersistentFlags()
flags.IntVar(&projectListOpts.limit, "limit", 10, "max number of runs to show")
flags.StringVar(&projectListOpts.start, "start", "", "starting project name (excluded) to fetch")
flags.StringVar(&projectListOpts.parentPath, "parent", "", `project group path (i.e "org/org01" for root project group in org01, "/user/user01/group01/subgroub01") or project group id`)
cmdProjectList.MarkFlagRequired("parent")
cmdProject.AddCommand(cmdProjectList)
}
func printProjects(projectsResponse *api.GetProjectsResponse) {
for _, project := range projectsResponse.Projects {
func printProjects(projects []*api.ProjectResponse) {
for _, project := range projects {
fmt.Printf("%s: Name: %s\n", project.ID, project.Name)
}
}
@ -57,12 +57,12 @@ func printProjects(projectsResponse *api.GetProjectsResponse) {
func projectList(cmd *cobra.Command, args []string) error {
gwclient := api.NewClient(gatewayURL, token)
projectsResponse, _, err := gwclient.GetCurrentUserProjects(context.TODO(), projectListOpts.start, projectListOpts.limit, false)
projects, _, err := gwclient.GetProjectGroupProjects(context.TODO(), projectListOpts.parentPath)
if err != nil {
return err
}
printProjects(projectsResponse)
printProjects(projects)
return nil
}

8
cmd/agola/cmd/remotesourcelist.go
View File

@ -48,8 +48,8 @@ func init() {
cmdRemoteSource.AddCommand(cmdRemoteSourceList)
}
func printRemoteSources(rssResponse *api.RemoteSourcesResponse) {
for _, rs := range rssResponse.RemoteSources {
func printRemoteSources(remoteSources []*api.RemoteSourceResponse) {
for _, rs := range remoteSources {
fmt.Printf("%s: Name: %s\n", rs.ID, rs.Name)
}
}
@ -57,12 +57,12 @@ func printRemoteSources(rssResponse *api.RemoteSourcesResponse) {
func remoteSourceList(cmd *cobra.Command, args []string) error {
gwclient := api.NewClient(gatewayURL, token)
rssResponse, _, err := gwclient.GetRemoteSources(context.TODO(), remoteSourceListOpts.start, remoteSourceListOpts.limit, false)
remouteSources, _, err := gwclient.GetRemoteSources(context.TODO(), remoteSourceListOpts.start, remoteSourceListOpts.limit, false)
if err != nil {
return err
}
printRemoteSources(rssResponse)
printRemoteSources(remouteSources)
return nil
}

6
cmd/agola/cmd/runlist.go
View File

@ -70,9 +70,9 @@ func runList(cmd *cobra.Command, args []string) error {
return err
}
runs := make([]*api.RunResponse, len(runsResp.Runs))
for i, runsResponse := range runsResp.Runs {
run, _, err := gwclient.GetRun(context.TODO(), runsResponse.ID)
runs := make([]*api.RunResponse, len(runsResp))
for i, runResponse := range runsResp {
run, _, err := gwclient.GetRun(context.TODO(), runResponse.ID)
if err != nil {
return err
}

8
cmd/agola/cmd/userlist.go
View File

@ -48,8 +48,8 @@ func init() {
cmdUser.AddCommand(cmdUserList)
}
func printUsers(usersResponse *api.UsersResponse) {
for _, user := range usersResponse.Users {
func printUsers(users []*api.UserResponse) {
for _, user := range users {
fmt.Printf("%s: Name: %s\n", user.ID, user.UserName)
}
}
@ -57,12 +57,12 @@ func printUsers(usersResponse *api.UsersResponse) {
func userList(cmd *cobra.Command, args []string) error {
gwclient := api.NewClient(gatewayURL, token)
usersResponse, _, err := gwclient.GetUsers(context.TODO(), userListOpts.start, userListOpts.limit, false)
users, _, err := gwclient.GetUsers(context.TODO(), userListOpts.start, userListOpts.limit, false)
if err != nil {
return err
}
printUsers(usersResponse)
printUsers(users)
return nil
}

1
go.mod
View File

@ -19,7 +19,6 @@ require (
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/google/go-cmp v0.3.0
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/go-sockaddr v1.0.1

4
go.sum
View File

@ -70,12 +70,8 @@ github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA=
github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8=

8
internal/config/config.go
View File

@ -167,7 +167,6 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
case "run":
var rs RunStep
rs.Type = stepType
switch stepSpec.(type) {
case string:
rs.Command = stepSpec.(string)
@ -176,22 +175,23 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
return err
}
}
rs.Type = stepType
steps[i] = &rs
case "save_to_workspace":
var sws SaveToWorkspaceStep
sws.Type = stepType
if err := yaml.Unmarshal(o, &sws); err != nil {
return err
}
sws.Type = stepType
steps[i] = &sws
case "restore_workspace":
var rws RestoreWorkspaceStep
rws.Type = stepType
if err := yaml.Unmarshal(o, &rws); err != nil {
return err
}
rws.Type = stepType
steps[i] = &rws
default:
return errors.Errorf("unknown step type: %s", stepType)
@ -452,7 +452,7 @@ var DefaultConfig = Config{}
func ParseConfig(configData []byte) (*Config, error) {
config := DefaultConfig
if err := yaml.Unmarshal(configData, &config); err != nil {
return nil, err
return nil, errors.Wrapf(err, "failed to unmarshal config")
}
if len(config.Pipelines) == 0 {

54
internal/services/configstore/api/client.go
View File

@ -108,15 +108,38 @@ func (c *Client) getParsedResponse(ctx context.Context, method, path string, que
return resp, d.Decode(obj)
}
func (c *Client) GetProject(ctx context.Context, projectID string) (*types.Project, *http.Response, error) {
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/project/%s", projectID), nil, jsonContent, nil, project)
return project, resp, err
func (c *Client) GetProjectGroup(ctx context.Context, projectGroupID string) (*types.ProjectGroup, *http.Response, error) {
projectGroup := new(types.ProjectGroup)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s", url.PathEscape(projectGroupID)), nil, jsonContent, nil, projectGroup)
return projectGroup, resp, err
}
func (c *Client) GetProjectByName(ctx context.Context, ownerid, projectName string) (*types.Project, *http.Response, error) {
func (c *Client) GetProjectGroupSubgroups(ctx context.Context, projectGroupID string) ([]*types.ProjectGroup, *http.Response, error) {
projectGroups := []*types.ProjectGroup{}
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/subgroups", url.PathEscape(projectGroupID)), nil, jsonContent, nil, &projectGroups)
return projectGroups, resp, err
}
func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupID string) ([]*types.Project, *http.Response, error) {
projects := []*types.Project{}
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupID)), nil, jsonContent, nil, &projects)
return projects, resp, err
}
func (c *Client) CreateProjectGroup(ctx context.Context, projectGroup *types.ProjectGroup) (*types.ProjectGroup, *http.Response, error) {
pj, err := json.Marshal(projectGroup)
if err != nil {
return nil, nil, err
}
projectGroup = new(types.ProjectGroup)
resp, err := c.getParsedResponse(ctx, "PUT", "/projectgroups", nil, jsonContent, bytes.NewReader(pj), projectGroup)
return projectGroup, resp, err
}
func (c *Client) GetProject(ctx context.Context, projectID string) (*types.Project, *http.Response, error) {
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/%s", ownerid, projectName), nil, jsonContent, nil, project)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s", url.PathEscape(projectID)), nil, jsonContent, nil, project)
return project, resp, err
}
@ -132,24 +155,7 @@ func (c *Client) CreateProject(ctx context.Context, project *types.Project) (*ty
}
func (c *Client) DeleteProject(ctx context.Context, projectID string) (*http.Response, error) {
return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s", projectID), nil, jsonContent, nil)
}
func (c *Client) GetOwnerProjects(ctx context.Context, ownerid, start string, limit int, asc bool) ([]*types.Project, *http.Response, error) {
q := url.Values{}
if start != "" {
q.Add("start", start)
}
if limit > 0 {
q.Add("limit", strconv.Itoa(limit))
}
if asc {
q.Add("asc", "")
}
projects := []*types.Project{}
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/owner/%s/projects", ownerid), q, jsonContent, nil, &projects)
return projects, resp, err
return c.getResponse(ctx, "DELETE", fmt.Sprintf("/projects/%s", url.PathEscape(projectID)), nil, jsonContent, nil)
}
func (c *Client) GetUser(ctx context.Context, userID string) (*types.User, *http.Response, error) {

112
internal/services/configstore/api/project.go
View File

@ -17,10 +17,11 @@ package api
import (
"encoding/json"
"net/http"
"strconv"
"net/url"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/configstore/command"
"github.com/sorintlab/agola/internal/services/configstore/common"
"github.com/sorintlab/agola/internal/services/configstore/readdb"
"github.com/sorintlab/agola/internal/services/types"
@ -39,48 +40,27 @@ func NewProjectHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectHandle
func (h *ProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
projectID := vars["projectid"]
var project *types.Project
err := h.readDB.Do(func(tx *db.Tx) error {
var err error
project, err = h.readDB.GetProject(tx, projectID)
return err
})
projectRef, err := url.PathUnescape(vars["projectref"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if project == nil {
http.Error(w, "", http.StatusNotFound)
projectRefType, err := common.ParseRef(projectRef)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode(project); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectByNameHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB
}
func NewProjectByNameHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectByNameHandler {
return &ProjectByNameHandler{log: logger.Sugar(), readDB: readDB}
}
func (h *ProjectByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
ownerID := vars["ownerid"]
projectName := vars["projectname"]
var project *types.Project
err := h.readDB.Do(func(tx *db.Tx) error {
err = h.readDB.Do(func(tx *db.Tx) error {
var err error
project, err = h.readDB.GetOwnerProjectByName(tx, ownerID, projectName)
switch projectRefType {
case common.RefTypeID:
project, err = h.readDB.GetProject(tx, projectRef)
case common.RefTypePath:
project, err = h.readDB.GetProjectByPath(tx, projectRef)
}
return err
})
if err != nil {
@ -144,9 +124,13 @@ func (h *DeleteProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
ctx := r.Context()
vars := mux.Vars(r)
projectID := vars["projectid"]
projectRef, err := url.PathUnescape(vars["projectref"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
err := h.ch.DeleteProject(ctx, projectID)
err = h.ch.DeleteProject(ctx, projectRef)
if httpError(w, err) {
h.log.Errorf("err: %+v", err)
}
@ -156,59 +140,3 @@ const (
DefaultProjectsLimit = 10
MaxProjectsLimit = 20
)
type ProjectsHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB
}
func NewProjectsHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectsHandler {
return &ProjectsHandler{log: logger.Sugar(), readDB: readDB}
}
func (h *ProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
ownerID := vars["ownerid"]
query := r.URL.Query()
limitS := query.Get("limit")
limit := DefaultProjectsLimit
if limitS != "" {
var err error
limit, err = strconv.Atoi(limitS)
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
}
if limit < 0 {
http.Error(w, "limit must be greater or equal than 0", http.StatusBadRequest)
return
}
if limit > MaxProjectsLimit {
limit = MaxProjectsLimit
}
asc := false
if _, ok := query["asc"]; ok {
asc = true
}
start := query.Get("start")
var projects []*types.Project
err := h.readDB.Do(func(tx *db.Tx) error {
var err error
projects, err = h.readDB.GetOwnerProjects(tx, ownerID, start, limit, asc)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if err := json.NewEncoder(w).Encode(projects); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

198
internal/services/configstore/api/projectgroup.go Normal file
View File

@ -0,0 +1,198 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"encoding/json"
"net/http"
"net/url"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/configstore/command"
"github.com/sorintlab/agola/internal/services/configstore/readdb"
"github.com/sorintlab/agola/internal/services/types"
"github.com/gorilla/mux"
"go.uber.org/zap"
)
type ProjectGroupHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB
}
func NewProjectGroupHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectGroupHandler {
return &ProjectGroupHandler{log: logger.Sugar(), readDB: readDB}
}
func (h *ProjectGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
projectGroupRef, err := url.PathUnescape(vars["projectgroupref"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var projectGroup *types.ProjectGroup
err = h.readDB.Do(func(tx *db.Tx) error {
var err error
projectGroup, err = h.readDB.GetProjectGroup(tx, projectGroupRef)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if projectGroup == nil {
http.Error(w, "", http.StatusNotFound)
return
}
if err := json.NewEncoder(w).Encode(projectGroup); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupProjectsHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB
}
func NewProjectGroupProjectsHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectGroupProjectsHandler {
return &ProjectGroupProjectsHandler{log: logger.Sugar(), readDB: readDB}
}
func (h *ProjectGroupProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
projectGroupRef, err := url.PathUnescape(vars["projectgroupref"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var projectGroup *types.ProjectGroup
err = h.readDB.Do(func(tx *db.Tx) error {
projectGroup, err = h.readDB.GetProjectGroup(tx, projectGroupRef)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if projectGroup == nil {
http.Error(w, "", http.StatusNotFound)
return
}
var projects []*types.Project
err = h.readDB.Do(func(tx *db.Tx) error {
var err error
projects, err = h.readDB.GetProjectGroupProjects(tx, projectGroup.ID)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if err := json.NewEncoder(w).Encode(projects); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupSubgroupsHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB
}
func NewProjectGroupSubgroupsHandler(logger *zap.Logger, readDB *readdb.ReadDB) *ProjectGroupSubgroupsHandler {
return &ProjectGroupSubgroupsHandler{log: logger.Sugar(), readDB: readDB}
}
func (h *ProjectGroupSubgroupsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
projectGroupRef, err := url.PathUnescape(vars["projectgroupref"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var projectGroup *types.ProjectGroup
err = h.readDB.Do(func(tx *db.Tx) error {
projectGroup, err = h.readDB.GetProjectGroup(tx, projectGroupRef)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if projectGroup == nil {
http.Error(w, "", http.StatusNotFound)
return
}
var projectGroups []*types.ProjectGroup
err = h.readDB.Do(func(tx *db.Tx) error {
var err error
projectGroups, err = h.readDB.GetProjectGroupSubgroups(tx, projectGroup.ID)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if err := json.NewEncoder(w).Encode(projectGroups); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type CreateProjectGroupHandler struct {
log *zap.SugaredLogger
ch *command.CommandHandler
readDB *readdb.ReadDB
}
func NewCreateProjectGroupHandler(logger *zap.Logger, ch *command.CommandHandler) *CreateProjectGroupHandler {
return &CreateProjectGroupHandler{log: logger.Sugar(), ch: ch}
}
func (h *CreateProjectGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req types.ProjectGroup
d := json.NewDecoder(r.Body)
if err := d.Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
projectGroup, err := h.ch.CreateProjectGroup(ctx, &req)
if httpError(w, err) {
h.log.Errorf("err: %+v", err)
return
}
if err := json.NewEncoder(w).Encode(projectGroup); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

1
internal/services/configstore/api/user.go
View File

@ -145,7 +145,6 @@ func NewDeleteUserHandler(logger *zap.Logger, ch *command.CommandHandler) *Delet
}
func (h *DeleteUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.log.Infof("deleteuserhandler")
ctx := r.Context()
vars := mux.Vars(r)

213
internal/services/configstore/command/command.go
View File

@ -17,6 +17,7 @@ package command
import (
"context"
"encoding/json"
"path"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/configstore/common"
@ -44,57 +45,129 @@ func NewCommandHandler(logger *zap.Logger, readDB *readdb.ReadDB, wal *wal.WalMa
}
}
func (s *CommandHandler) CreateProject(ctx context.Context, project *types.Project) (*types.Project, error) {
if project.Name == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project name required"))
func (s *CommandHandler) CreateProjectGroup(ctx context.Context, projectGroup *types.ProjectGroup) (*types.ProjectGroup, error) {
if projectGroup.Name == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project group name required"))
}
if project.OwnerType == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project ownertype required"))
}
if project.OwnerID == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project ownerid required"))
}
if !types.IsValidOwnerType(project.OwnerType) {
return nil, util.NewErrBadRequest(errors.Errorf("invalid project ownertype %q", project.OwnerType))
if projectGroup.Parent.ID == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project group parent id required"))
}
var cgt *wal.ChangeGroupsUpdateToken
cgNames := []string{project.OwnerID}
// must do all the check in a single transaction to avoid concurrent changes
err := s.readDB.Do(func(tx *db.Tx) error {
var err error
parentProjectGroup, err := s.readDB.GetProjectGroup(tx, projectGroup.Parent.ID)
if err != nil {
return err
}
if parentProjectGroup == nil {
return util.NewErrBadRequest(errors.Errorf("project group with id %q doesn't exist", projectGroup.Parent.ID))
}
projectGroup.Parent.ID = parentProjectGroup.ID
groupPath, err := s.readDB.GetProjectGroupPath(tx, parentProjectGroup)
if err != nil {
return err
}
pp := path.Join(groupPath, projectGroup.Name)
cgNames := []string{pp}
cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
if err != nil {
return err
}
// check owner exists
switch project.OwnerType {
case types.OwnerTypeUser:
user, err := s.readDB.GetUser(tx, project.OwnerID)
if err != nil {
return err
}
if user == nil {
return util.NewErrBadRequest(errors.Errorf("user id %q doesn't exist", project.OwnerID))
}
case types.OwnerTypeOrganization:
org, err := s.readDB.GetOrg(tx, project.OwnerID)
if err != nil {
return err
}
if org == nil {
return util.NewErrBadRequest(errors.Errorf("organization id %q doesn't exist", project.OwnerID))
}
}
// check duplicate project name
p, err := s.readDB.GetOwnerProjectByName(tx, project.OwnerID, project.Name)
p, err := s.readDB.GetProjectByName(tx, projectGroup.Parent.ID, projectGroup.Name)
if err != nil {
return err
}
if p != nil {
return util.NewErrBadRequest(errors.Errorf("project with name %q for %s with id %q already exists", p.Name, project.OwnerType, project.OwnerID))
return util.NewErrBadRequest(errors.Errorf("project with name %q, path %q already exists", p.Name, pp))
}
// check duplicate project group name
pg, err := s.readDB.GetProjectGroupByName(tx, projectGroup.Parent.ID, projectGroup.Name)
if err != nil {
return err
}
if pg != nil {
return util.NewErrBadRequest(errors.Errorf("project group with name %q, path %q already exists", pg.Name, pp))
}
return nil
})
if err != nil {
return nil, err
}
projectGroup.ID = uuid.NewV4().String()
projectGroup.Parent.Type = types.ConfigTypeProjectGroup
pcj, err := json.Marshal(projectGroup)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal projectGroup")
}
actions := []*wal.Action{
{
ActionType: wal.ActionTypePut,
Path: common.StorageProjectGroupFile(projectGroup.ID),
Data: pcj,
},
}
_, err = s.wal.WriteWal(ctx, actions, cgt)
return projectGroup, err
}
func (s *CommandHandler) CreateProject(ctx context.Context, project *types.Project) (*types.Project, error) {
if project.Name == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project name required"))
}
if project.Parent.ID == "" {
return nil, util.NewErrBadRequest(errors.Errorf("project parent id required"))
}
var cgt *wal.ChangeGroupsUpdateToken
// must do all the check in a single transaction to avoid concurrent changes
err := s.readDB.Do(func(tx *db.Tx) error {
var err error
group, err := s.readDB.GetProjectGroup(tx, project.Parent.ID)
if err != nil {
return err
}
if group == nil {
return util.NewErrBadRequest(errors.Errorf("project group with id %q doesn't exist", project.Parent.ID))
}
project.Parent.ID = group.ID
groupPath, err := s.readDB.GetProjectGroupPath(tx, group)
if err != nil {
return err
}
pp := path.Join(groupPath, project.Name)
cgNames := []string{pp}
cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
if err != nil {
return err
}
// check duplicate project name
p, err := s.readDB.GetProjectByName(tx, project.Parent.ID, project.Name)
if err != nil {
return err
}
if p != nil {
return util.NewErrBadRequest(errors.Errorf("project with name %q, path %q already exists", p.Name, pp))
}
// check duplicate project group name
pg, err := s.readDB.GetProjectGroupByName(tx, project.Parent.ID, project.Name)
if err != nil {
return err
}
if pg != nil {
return util.NewErrBadRequest(errors.Errorf("project group with name %q, path %q already exists", pg.Name, pp))
}
return nil
})
@ -103,6 +176,7 @@ func (s *CommandHandler) CreateProject(ctx context.Context, project *types.Proje
}
project.ID = uuid.NewV4().String()
project.Parent.Type = types.ConfigTypeProjectGroup
pcj, err := json.Marshal(project)
if err != nil {
@ -120,28 +194,34 @@ func (s *CommandHandler) CreateProject(ctx context.Context, project *types.Proje
return project, err
}
func (s *CommandHandler) DeleteProject(ctx context.Context, projectID string) error {
func (s *CommandHandler) DeleteProject(ctx context.Context, projectRef string) error {
var project *types.Project
var cgt *wal.ChangeGroupsUpdateToken
cgNames := []string{project.OwnerID}
// must do all the check in a single transaction to avoid concurrent changes
err := s.readDB.Do(func(tx *db.Tx) error {
var err error
// check project existance
project, err := s.readDB.GetProject(tx, projectRef)
if err != nil {
return err
}
if project == nil {
return util.NewErrBadRequest(errors.Errorf("project %q doesn't exist", projectRef))
}
group, err := s.readDB.GetProjectGroup(tx, project.Parent.ID)
if err != nil {
return err
}
cgNames := []string{group.ID}
cgt, err = s.readDB.GetChangeGroupsUpdateTokens(tx, cgNames)
if err != nil {
return err
}
// check project existance
project, err = s.readDB.GetProject(tx, projectID)
if err != nil {
return err
}
if project == nil {
return util.NewErrBadRequest(errors.Errorf("project %q doesn't exist", projectID))
}
return nil
})
if err != nil {
@ -190,17 +270,34 @@ func (s *CommandHandler) CreateUser(ctx context.Context, user *types.User) (*typ
}
user.ID = uuid.NewV4().String()
userj, err := json.Marshal(user)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal user")
}
pg := &types.ProjectGroup{
ID: uuid.NewV4().String(),
Parent: types.Parent{
Type: types.ConfigTypeUser,
ID: user.ID,
},
}
pgj, err := json.Marshal(pg)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal project group")
}
actions := []*wal.Action{
{
ActionType: wal.ActionTypePut,
Path: common.StorageUserFile(user.ID),
Data: userj,
},
{
ActionType: wal.ActionTypePut,
Path: common.StorageProjectGroupFile(pg.ID),
Data: pgj,
},
}
_, err = s.wal.WriteWal(ctx, actions, cgt)
@ -645,17 +742,33 @@ func (s *CommandHandler) CreateOrg(ctx context.Context, org *types.Organization)
}
org.ID = uuid.NewV4().String()
orgj, err := json.Marshal(org)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal org")
}
pg := &types.ProjectGroup{
ID: uuid.NewV4().String(),
Parent: types.Parent{
Type: types.ConfigTypeOrg,
ID: org.ID,
},
}
pgj, err := json.Marshal(pg)
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal project group")
}
actions := []*wal.Action{
{
ActionType: wal.ActionTypePut,
Path: common.StorageOrgFile(org.ID),
Data: orgj,
},
{
ActionType: wal.ActionTypePut,
Path: common.StorageProjectGroupFile(pg.ID),
Data: pgj,
},
}
_, err = s.wal.WriteWal(ctx, actions, cgt)
@ -667,7 +780,7 @@ func (s *CommandHandler) DeleteOrg(ctx context.Context, orgName string) error {
var projects []*types.Project
var cgt *wal.ChangeGroupsUpdateToken
cgNames := []string{org.ID}
cgNames := []string{orgName}
// must do all the check in a single transaction to avoid concurrent changes
err := s.readDB.Do(func(tx *db.Tx) error {
@ -685,11 +798,7 @@ func (s *CommandHandler) DeleteOrg(ctx context.Context, orgName string) error {
if org == nil {
return util.NewErrBadRequest(errors.Errorf("org %q doesn't exist", orgName))
}
// get org projects
projects, err = s.readDB.GetOwnerProjects(tx, org.ID, "", 0, false)
if err != nil {
return err
}
// TODO(sgotti) delete all project groups, projects etc...
return nil
})
if err != nil {

64
internal/services/configstore/common/common.go
View File

@ -16,15 +16,20 @@ package common
import (
"fmt"
"net/url"
"path"
"strings"
"github.com/sorintlab/agola/internal/services/types"
)
var (
// Storage paths. Always use path (not filepath) to use the "/" separator
StorageDataDir = "data"
StorageProjectsDir = path.Join(StorageDataDir, "projects")
StorageUsersDir = path.Join(StorageDataDir, "users")
StorageOrgsDir = path.Join(StorageDataDir, "orgs")
StorageProjectsDir = path.Join(StorageDataDir, "projects")
StorageProjectGroupsDir = path.Join(StorageDataDir, "projectgroups")
StorageRemoteSourcesDir = path.Join(StorageDataDir, "remotesources")
)
@ -32,10 +37,6 @@ const (
etcdWalsMinRevisionRange = 100
)
func StorageProjectFile(projectID string) string {
return path.Join(StorageProjectsDir, projectID)
}
func StorageUserFile(userID string) string {
return path.Join(StorageUsersDir, userID)
}
@ -44,33 +45,54 @@ func StorageOrgFile(orgID string) string {
return path.Join(StorageOrgsDir, orgID)
}
func StorageProjectGroupFile(projectGroupID string) string {
return path.Join(StorageProjectGroupsDir, projectGroupID)
}
func StorageProjectFile(projectID string) string {
return path.Join(StorageProjectsDir, projectID)
}
func StorageRemoteSourceFile(userID string) string {
return path.Join(StorageRemoteSourcesDir, userID)
}
type ConfigType string
const (
ConfigTypeProject ConfigType = "project"
ConfigTypeUser ConfigType = "user"
ConfigTypeOrg ConfigType = "org"
ConfigTypeRemoteSource ConfigType = "remotesource"
)
func PathToTypeID(p string) (ConfigType, string) {
var configType ConfigType
func PathToTypeID(p string) (types.ConfigType, string) {
var configType types.ConfigType
switch path.Dir(p) {
case StorageProjectsDir:
configType = ConfigTypeProject
case StorageUsersDir:
configType = ConfigTypeUser
configType = types.ConfigTypeUser
case StorageOrgsDir:
configType = ConfigTypeOrg
configType = types.ConfigTypeOrg
case StorageProjectGroupsDir:
configType = types.ConfigTypeProjectGroup
case StorageProjectsDir:
configType = types.ConfigTypeProject
case StorageRemoteSourcesDir:
configType = ConfigTypeRemoteSource
configType = types.ConfigTypeRemoteSource
default:
panic(fmt.Errorf("cannot determine configtype for path: %q", p))
}
return configType, path.Base(p)
}
type RefType int
const (
RefTypeID RefType = iota
RefTypePath
)
// ParseRef parses the api call to determine if the provided ref is
// an ID or a path
func ParseRef(projectRef string) (RefType, error) {
projectRef, err := url.PathUnescape(projectRef)
if err != nil {
return -1, err
}
if strings.Contains(projectRef, "/") {
return RefTypePath, nil
}
return RefTypeID, nil
}

18
internal/services/configstore/configstore.go
View File

@ -109,9 +109,12 @@ func (s *ConfigStore) Run(ctx context.Context) error {
corsAllowedOriginsOptions := ghandlers.AllowedOrigins([]string{"*"})
corsHandler = ghandlers.CORS(corsAllowedMethodsOptions, corsAllowedHeadersOptions, corsAllowedOriginsOptions)
projectGroupHandler := api.NewProjectGroupHandler(logger, s.readDB)
projectGroupSubgroupsHandler := api.NewProjectGroupSubgroupsHandler(logger, s.readDB)
projectGroupProjectsHandler := api.NewProjectGroupProjectsHandler(logger, s.readDB)
createProjectGroupHandler := api.NewCreateProjectGroupHandler(logger, s.ch)
projectHandler := api.NewProjectHandler(logger, s.readDB)
projectsHandler := api.NewProjectsHandler(logger, s.readDB)
projectByNameHandler := api.NewProjectByNameHandler(logger, s.readDB)
createProjectHandler := api.NewCreateProjectHandler(logger, s.ch)
deleteProjectHandler := api.NewDeleteProjectHandler(logger, s.ch)
@ -140,11 +143,14 @@ func (s *ConfigStore) Run(ctx context.Context) error {
deleteRemoteSourceHandler := api.NewDeleteRemoteSourceHandler(logger, s.ch)
router := mux.NewRouter()
apirouter := router.PathPrefix("/api/v1alpha").Subrouter()
apirouter := router.PathPrefix("/api/v1alpha").Subrouter().UseEncodedPath()
apirouter.Handle("/project/{projectid}", projectHandler).Methods("GET")
apirouter.Handle("/owner/{ownerid}/projects", projectsHandler).Methods("GET")
apirouter.Handle("/projects/{ownerid}/{projectname}", projectByNameHandler).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupref}", projectGroupHandler).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupref}/subgroups", projectGroupSubgroupsHandler).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupref}/projects", projectGroupProjectsHandler).Methods("GET")
apirouter.Handle("/projectgroups", createProjectGroupHandler).Methods("PUT")
apirouter.Handle("/projects/{projectref}", projectHandler).Methods("GET")
apirouter.Handle("/projects", createProjectHandler).Methods("PUT")
apirouter.Handle("/projects/{projectid}", deleteProjectHandler).Methods("DELETE")

88
internal/services/configstore/configstore_test.go
View File

@ -20,6 +20,7 @@ import (
"io/ioutil"
"net"
"os"
"path"
"reflect"
"sync"
"testing"
@ -90,7 +91,7 @@ func getProjects(cs *ConfigStore) ([]*types.Project, error) {
var projects []*types.Project
err := cs.readDB.Do(func(tx *db.Tx) error {
var err error
projects, err = cs.readDB.GetProjects(tx, "", 0, true)
projects, err = cs.readDB.GetAllProjects(tx)
return err
})
return projects, err
@ -356,7 +357,7 @@ func TestUser(t *testing.T) {
})
}
func TestProject(t *testing.T) {
func TestProjectGroupsAndProjects(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
@ -390,60 +391,91 @@ func TestProject(t *testing.T) {
// TODO(sgotti) change the sleep with a real check that user is in readdb
time.Sleep(2 * time.Second)
t.Run("create project with owner type user", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "user", OwnerID: user.ID})
t.Run("create a project in user root project group", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create project with owner type org", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "organization", OwnerID: org.ID})
t.Run("create a project in org root project group", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create duplicated project for user", func(t *testing.T) {
expectedErr := fmt.Sprintf("bad request: project with name %q for user with id %q already exists", "project01", user.ID)
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "user", OwnerID: user.ID})
t.Run("create a projectgroup in user root project group", func(t *testing.T) {
_, err := cs.ch.CreateProjectGroup(ctx, &types.ProjectGroup{Name: "projectgroup01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a projectgroup in org root project group", func(t *testing.T) {
_, err := cs.ch.CreateProjectGroup(ctx, &types.ProjectGroup{Name: "projectgroup01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create a project in user non root project group with same name as a root project", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName, "projectgroup01")}})
if err != nil {
t.Fatalf("unexpected err: %+#v", err)
}
})
t.Run("create a project in org non root project group with same name as a root project", func(t *testing.T) {
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name, "projectgroup01")}})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
})
t.Run("create duplicated project in user root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("bad request: project with name %q, path %q already exists", projectName, path.Join("user", user.UserName, projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create duplicated project for org", func(t *testing.T) {
expectedErr := fmt.Sprintf("bad request: project with name %q for organization with id %q already exists", "project01", org.ID)
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "organization", OwnerID: org.ID})
t.Run("create duplicated project in org root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("bad request: project with name %q, path %q already exists", projectName, path.Join("org", org.Name, projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name)}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project with owner as unexistent user", func(t *testing.T) {
expectedErr := `bad request: user id "unexistentid" doesn't exist`
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "user", OwnerID: "unexistentid"})
t.Run("create duplicated project in user non root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("bad request: project with name %q, path %q already exists", projectName, path.Join("user", user.UserName, "projectgroup01", projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName, "projectgroup01")}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project with owner as unexistent org", func(t *testing.T) {
expectedErr := `bad request: organization id "unexistentid" doesn't exist`
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "organization", OwnerID: "unexistentid"})
t.Run("create duplicated project in org non root project group", func(t *testing.T) {
projectName := "project01"
expectedErr := fmt.Sprintf("bad request: project with name %q, path %q already exists", projectName, path.Join("org", org.Name, "projectgroup01", projectName))
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: projectName, Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("org", org.Name, "projectgroup01")}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project without ownertype specified", func(t *testing.T) {
expectedErr := "bad request: project ownertype required"
t.Run("create project in unexistent project group", func(t *testing.T) {
expectedErr := `bad request: project group with id "unexistentid" doesn't exist`
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: "unexistentid"}})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project without parent id specified", func(t *testing.T) {
expectedErr := "bad request: project parent id required"
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01"})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("create project without ownerid specified", func(t *testing.T) {
expectedErr := "bad request: project ownerid required"
_, err := cs.ch.CreateProject(ctx, &types.Project{Name: "project01", OwnerType: "organization"})
if err.Error() != expectedErr {
t.Fatalf("expected err %v, got err: %v", expectedErr, err)
}
})
t.Run("concurrent project with same name creation", func(t *testing.T) {
prevProjects, err := getProjects(cs)
@ -454,7 +486,7 @@ func TestProject(t *testing.T) {
wg := sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go cs.ch.CreateProject(ctx, &types.Project{Name: "project02", OwnerType: "user", OwnerID: user.ID})
go cs.ch.CreateProject(ctx, &types.Project{Name: "project02", Parent: types.Parent{Type: types.ConfigTypeProjectGroup, ID: path.Join("user", user.UserName)}})
wg.Done()
}
wg.Wait()

12
internal/services/configstore/readdb/migration.go → internal/services/configstore/readdb/create.go
View File

@ -15,6 +15,7 @@
package readdb
var Stmts = []string{
// last processed etcd event revision
"create table revision (revision bigint, PRIMARY KEY(revision))",
@ -24,7 +25,10 @@ var Stmts = []string{
// changegrouprevision stores the current revision of the changegroup for optimistic locking
"create table changegrouprevision (id varchar, revision varchar, PRIMARY KEY (id, revision))",
"create table project (id uuid, name varchar, ownerid varchar, data bytea, PRIMARY KEY (id))",
"create table projectgroup (id uuid, name varchar, parentid varchar, data bytea, PRIMARY KEY (id))",
"create index projectgroup_name on projectgroup(name)",
"create table project (id uuid, name varchar, parentid varchar, data bytea, PRIMARY KEY (id))",
"create index project_name on project(name)",
"create table user (id uuid, name varchar, data bytea, PRIMARY KEY (id))",
@ -39,4 +43,10 @@ var Stmts = []string{
"create table linkedaccount_user (id uuid, remotesourceid uuid, userid uuid, remoteuserid uuid, PRIMARY KEY (id), FOREIGN KEY(userid) REFERENCES user(id))",
"create table linkedaccount_project (id uuid, projectid uuid, PRIMARY KEY (id), FOREIGN KEY(projectid) REFERENCES user(id))",
"create table secret (id uuid, name varchar, containerid varchar, data bytea, PRIMARY KEY (id))",
"create index secret_name on secret(name)",
"create table variable (id uuid, name varchar, containerid varchar, data bytea, PRIMARY KEY (id))",
"create index variable_name on variable(name)",
}

49
internal/services/configstore/readdb/parent.go Normal file
View File

@ -0,0 +1,49 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package readdb
import (
"github.com/pkg/errors"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
)
func (r *ReadDB) ResolveConfigID(tx *db.Tx, configType types.ConfigType, ref string) (string, error) {
switch configType {
case types.ConfigTypeProjectGroup:
group, err := r.GetProjectGroup(tx, ref)
if err != nil {
return "", err
}
if group == nil {
return "", util.NewErrBadRequest(errors.Errorf("group with ref %q doesn't exists", ref))
}
return group.ID, nil
case types.ConfigTypeProject:
project, err := r.GetProject(tx, ref)
if err != nil {
return "", err
}
if project == nil {
return "", util.NewErrBadRequest(errors.Errorf("project with ref %q doesn't exists", ref))
}
return project.ID, nil
default:
return "", util.NewErrBadRequest(errors.Errorf("unknown config type %q", configType))
}
}

136
internal/services/configstore/readdb/project.go
View File

@ -17,8 +17,11 @@ package readdb
import (
"database/sql"
"encoding/json"
"path"
"strings"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/configstore/common"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
@ -28,11 +31,11 @@ import (
var (
projectSelect = sb.Select("id", "data").From("project")
projectInsert = sb.Insert("project").Columns("id", "name", "ownerid", "data")
projectInsert = sb.Insert("project").Columns("id", "name", "parentid", "data")
)
func (r *ReadDB) insertProject(tx *db.Tx, data []byte) error {
project := types.Project{}
var project *types.Project
if err := json.Unmarshal(data, &project); err != nil {
return errors.Wrap(err, "failed to unmarshal project")
}
@ -40,7 +43,7 @@ func (r *ReadDB) insertProject(tx *db.Tx, data []byte) error {
if err := r.deleteProject(tx, project.ID); err != nil {
return err
}
q, args, err := projectInsert.Values(project.ID, project.Name, project.OwnerID, data).ToSql()
q, args, err := projectInsert.Values(project.ID, project.Name, project.Parent.ID, data).ToSql()
if err != nil {
return errors.Wrap(err, "failed to build query")
}
@ -56,7 +59,42 @@ func (r *ReadDB) deleteProject(tx *db.Tx, id string) error {
return nil
}
func (r *ReadDB) GetProject(tx *db.Tx, projectID string) (*types.Project, error) {
func (r *ReadDB) GetProjectPath(tx *db.Tx, project *types.Project) (string, error) {
pgroup, err := r.GetProjectGroup(tx, project.Parent.ID)
if err != nil {
return "", err
}
if pgroup == nil {
return "", errors.Errorf("parent group %q for project %q doesn't exist", project.Parent.ID, project.ID)
}
p, err := r.GetProjectGroupPath(tx, pgroup)
if err != nil {
return "", err
}
p = path.Join(p, project.Name)
return p, nil
}
func (r *ReadDB) GetProject(tx *db.Tx, projectRef string) (*types.Project, error) {
projectRefType, err := common.ParseRef(projectRef)
if err != nil {
return nil, err
}
var project *types.Project
switch projectRefType {
case common.RefTypeID:
project, err = r.GetProjectByID(tx, projectRef)
case common.RefTypePath:
project, err = r.GetProjectByPath(tx, projectRef)
}
return project, err
}
func (r *ReadDB) GetProjectByID(tx *db.Tx, projectID string) (*types.Project, error) {
q, args, err := projectSelect.Where(sq.Eq{"id": projectID}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
@ -76,8 +114,8 @@ func (r *ReadDB) GetProject(tx *db.Tx, projectID string) (*types.Project, error)
return projects[0], nil
}
func (r *ReadDB) GetOwnerProjectByName(tx *db.Tx, ownerid, name string) (*types.Project, error) {
q, args, err := projectSelect.Where(sq.Eq{"ownerid": ownerid, "name": name}).ToSql()
func (r *ReadDB) GetProjectByName(tx *db.Tx, parentID, name string) (*types.Project, error) {
q, args, err := projectSelect.Where(sq.Eq{"parentid": parentID, "name": name}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
@ -96,67 +134,38 @@ func (r *ReadDB) GetOwnerProjectByName(tx *db.Tx, ownerid, name string) (*types.
return projects[0], nil
}
func getProjectsFilteredQuery(ownerid, startProjectName string, limit int, asc bool) sq.SelectBuilder {
fields := []string{"id", "data"}
s := sb.Select(fields...).From("project as project")
if asc {
s = s.OrderBy("project.name asc")
} else {
s = s.OrderBy("project.name desc")
}
if ownerid != "" {
s = s.Where(sq.Eq{"project.ownerid": ownerid})
}
if startProjectName != "" {
if asc {
s = s.Where(sq.Gt{"project.name": startProjectName})
} else {
s = s.Where(sq.Lt{"project.name": startProjectName})
}
}
if limit > 0 {
s = s.Limit(uint64(limit))
func (r *ReadDB) GetProjectByPath(tx *db.Tx, projectPath string) (*types.Project, error) {
if len(strings.Split(projectPath, "/")) < 3 {
return nil, errors.Errorf("wrong project path: %q", projectPath)
}
return s
projectGroupPath := path.Dir(projectPath)
projectName := path.Base(projectPath)
projectGroup, err := r.GetProjectGroupByPath(tx, projectGroupPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to get project group %q", projectGroupPath)
}
if projectGroup == nil {
return nil, nil
}
project, err := r.GetProjectByName(tx, projectGroup.ID, projectName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get project group %q", projectName)
}
return project, nil
}
func (r *ReadDB) GetOwnerProjects(tx *db.Tx, ownerid, startProjectName string, limit int, asc bool) ([]*types.Project, error) {
func (r *ReadDB) GetProjectGroupProjects(tx *db.Tx, parentID string) ([]*types.Project, error) {
var projects []*types.Project
s := getProjectsFilteredQuery(ownerid, startProjectName, limit, asc)
q, args, err := s.ToSql()
q, args, err := projectSelect.Where(sq.Eq{"parentid": parentID}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
rows, err := tx.Query(q, args...)
if err != nil {
return nil, err
}
projects, _, err = scanProjects(rows)
return projects, err
}
func (r *ReadDB) GetProjects(tx *db.Tx, startProjectName string, limit int, asc bool) ([]*types.Project, error) {
var projects []*types.Project
s := getProjectsFilteredQuery("", startProjectName, limit, asc)
q, args, err := s.ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
rows, err := tx.Query(q, args...)
if err != nil {
return nil, err
}
projects, _, err = scanProjects(rows)
projects, _, err = fetchProjects(tx, q, args...)
return projects, err
}
@ -202,3 +211,18 @@ func scanProjects(rows *sql.Rows) ([]*types.Project, []string, error) {
}
return projects, ids, nil
}
// Test only functions
func (r *ReadDB) GetAllProjects(tx *db.Tx) ([]*types.Project, error) {
var projects []*types.Project
q, args, err := projectSelect.ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
projects, _, err = fetchProjects(tx, q, args...)
return projects, err
}

301
internal/services/configstore/readdb/projectgroup.go Normal file
View File

@ -0,0 +1,301 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package readdb
import (
"database/sql"
"encoding/json"
"path"
"strings"
"github.com/sorintlab/agola/internal/db"
"github.com/sorintlab/agola/internal/services/configstore/common"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
sq "github.com/Masterminds/squirrel"
"github.com/pkg/errors"
)
var (
projectgroupSelect = sb.Select("id", "data").From("projectgroup")
projectgroupInsert = sb.Insert("projectgroup").Columns("id", "name", "parentid", "data")
)
func (r *ReadDB) insertProjectGroup(tx *db.Tx, data []byte) error {
var group *types.ProjectGroup
if err := json.Unmarshal(data, &group); err != nil {
return errors.Wrap(err, "failed to unmarshal group")
}
// poor man insert or update...
if err := r.deleteProjectGroup(tx, group.ID); err != nil {
return err
}
q, args, err := projectgroupInsert.Values(group.ID, group.Name, group.Parent.ID, data).ToSql()
if err != nil {
return errors.Wrap(err, "failed to build query")
}
_, err = tx.Exec(q, args...)
return errors.Wrap(err, "failed to insert group")
}
func (r *ReadDB) deleteProjectGroup(tx *db.Tx, id string) error {
// poor man insert or update...
if _, err := tx.Exec("delete from projectgroup where id = $1", id); err != nil {
return errors.Wrap(err, "failed to delete group")
}
return nil
}
type Element struct {
ID string
Name string
Type types.ConfigType
ParentType types.ConfigType
ParentID string
}
func (r *ReadDB) GetProjectGroupHierarchy(tx *db.Tx, projectGroup *types.ProjectGroup) ([]*Element, error) {
projectGroupID := projectGroup.Parent.ID
elements := []*Element{
{
ID: projectGroup.ID,
Name: projectGroup.Name,
Type: types.ConfigTypeProjectGroup,
ParentType: projectGroup.Parent.Type,
ParentID: projectGroup.Parent.ID,
},
}
for projectGroup.Parent.Type == types.ConfigTypeProjectGroup {
var err error
projectGroup, err = r.GetProjectGroup(tx, projectGroupID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get project group %q", projectGroupID)
}
if projectGroup == nil {
return nil, errors.Errorf("project group %q doesn't exist", projectGroupID)
}
elements = append([]*Element{
{
ID: projectGroup.ID,
Name: projectGroup.Name,
Type: types.ConfigTypeProjectGroup,
ParentType: projectGroup.Parent.Type,
ParentID: projectGroup.Parent.ID,
},
}, elements...)
projectGroupID = projectGroup.Parent.ID
}
return elements, nil
}
func (r *ReadDB) GetProjectGroupPath(tx *db.Tx, group *types.ProjectGroup) (string, error) {
var p string
groups, err := r.GetProjectGroupHierarchy(tx, group)
if err != nil {
return "", err
}
rootGroupType := groups[0].ParentType
rootGroupID := groups[0].ParentID
switch rootGroupType {
case types.ConfigTypeOrg:
org, err := r.GetOrg(tx, rootGroupID)
if err != nil {
return "", errors.Wrapf(err, "failed to get org %q", rootGroupID)
}
if org == nil {
return "", errors.Errorf("cannot find org with id %q", rootGroupID)
}
p = path.Join("org", org.Name)
case types.ConfigTypeUser:
user, err := r.GetUser(tx, rootGroupID)
if err != nil {
return "", errors.Wrapf(err, "failed to get user %q", rootGroupID)
}
if user == nil {
return "", errors.Errorf("cannot find user with id %q", rootGroupID)
}
p = path.Join("user", user.UserName)
}
for _, group := range groups {
p = path.Join(p, group.Name)
}
return p, nil
}
func (r *ReadDB) GetProjectGroup(tx *db.Tx, projectGroupRef string) (*types.ProjectGroup, error) {
groupRef, err := common.ParseRef(projectGroupRef)
if err != nil {
return nil, err
}
var group *types.ProjectGroup
switch groupRef {
case common.RefTypeID:
group, err = r.GetProjectGroupByID(tx, projectGroupRef)
case common.RefTypePath:
group, err = r.GetProjectGroupByPath(tx, projectGroupRef)
}
return group, err
}
func (r *ReadDB) GetProjectGroupByID(tx *db.Tx, projectGroupID string) (*types.ProjectGroup, error) {
q, args, err := projectgroupSelect.Where(sq.Eq{"id": projectGroupID}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
projectGroups, _, err := fetchProjectGroups(tx, q, args...)
if err != nil {
return nil, errors.WithStack(err)
}
if len(projectGroups) > 1 {
return nil, errors.Errorf("too many rows returned")
}
if len(projectGroups) == 0 {
return nil, nil
}
return projectGroups[0], nil
}
func (r *ReadDB) GetProjectGroupByName(tx *db.Tx, parentID, name string) (*types.ProjectGroup, error) {
q, args, err := projectgroupSelect.Where(sq.Eq{"parentid": parentID, "name": name}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
projectGroups, _, err := fetchProjectGroups(tx, q, args...)
if err != nil {
return nil, errors.WithStack(err)
}
if len(projectGroups) > 1 {
return nil, errors.Errorf("too many rows returned")
}
if len(projectGroups) == 0 {
return nil, nil
}
return projectGroups[0], nil
}
func (r *ReadDB) GetProjectGroupByPath(tx *db.Tx, projectGroupPath string) (*types.ProjectGroup, error) {
parts := strings.Split(projectGroupPath, "/")
if len(parts) < 2 {
return nil, errors.Errorf("wrong project group path: %q", projectGroupPath)
}
var parentID string
switch parts[0] {
case "org":
org, err := r.GetOrgByName(tx, parts[1])
if err != nil {
return nil, errors.Wrapf(err, "failed to get org %q", parts[1])
}
if org == nil {
return nil, errors.Errorf("cannot find org with name %q", parts[1])
}
parentID = org.ID
case "user":
user, err := r.GetUserByName(tx, parts[1])
if err != nil {
return nil, errors.Wrapf(err, "failed to get user %q", parts[1])
}
if user == nil {
return nil, errors.Errorf("cannot find user with name %q", parts[1])
}
parentID = user.ID
default:
return nil, errors.Errorf("wrong project group path: %q", projectGroupPath)
}
var projectGroup *types.ProjectGroup
// add root project group (empty name)
for _, projectGroupName := range append([]string{""}, parts[2:]...) {
var err error
projectGroup, err = r.GetProjectGroupByName(tx, parentID, projectGroupName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get project group %q", projectGroupName)
}
if projectGroup == nil {
return nil, nil
}
parentID = projectGroup.ID
}
return projectGroup, nil
}
func (r *ReadDB) GetProjectGroupSubgroups(tx *db.Tx, parentID string) ([]*types.ProjectGroup, error) {
var projectGroups []*types.ProjectGroup
q, args, err := projectgroupSelect.Where(sq.Eq{"parentid": parentID}).ToSql()
r.log.Debugf("q: %s, args: %s", q, util.Dump(args))
if err != nil {
return nil, errors.Wrap(err, "failed to build query")
}
projectGroups, _, err = fetchProjectGroups(tx, q, args...)
return projectGroups, err
}
func fetchProjectGroups(tx *db.Tx, q string, args ...interface{}) ([]*types.ProjectGroup, []string, error) {
rows, err := tx.Query(q, args...)
if err != nil {
return nil, nil, err
}
defer rows.Close()
return scanProjectGroups(rows)
}
func scanProjectGroup(rows *sql.Rows, additionalFields ...interface{}) (*types.ProjectGroup, string, error) {
var id string
var data []byte
if err := rows.Scan(&id, &data); err != nil {
return nil, "", errors.Wrap(err, "failed to scan rows")
}
group := types.ProjectGroup{}
if len(data) > 0 {
if err := json.Unmarshal(data, &group); err != nil {
return nil, "", errors.Wrap(err, "failed to unmarshal group")
}
}
return &group, id, nil
}
func scanProjectGroups(rows *sql.Rows) ([]*types.ProjectGroup, []string, error) {
projectGroups := []*types.ProjectGroup{}
ids := []string{}
for rows.Next() {
p, id, err := scanProjectGroup(rows)
if err != nil {
rows.Close()
return nil, nil, err
}
projectGroups = append(projectGroups, p)
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, nil, err
}
return projectGroups, ids, nil
}

41
internal/services/configstore/readdb/readdb.go
View File

@ -30,6 +30,7 @@ import (
"github.com/sorintlab/agola/internal/objectstorage"
"github.com/sorintlab/agola/internal/sequence"
"github.com/sorintlab/agola/internal/services/configstore/common"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
"github.com/sorintlab/agola/internal/wal"
@ -440,6 +441,7 @@ func (r *ReadDB) Run(ctx context.Context) error {
break
}
r.log.Errorf("initialize err: %+v", err)
time.Sleep(1 * time.Second)
}
}
@ -619,19 +621,23 @@ func (r *ReadDB) applyAction(tx *db.Tx, action *wal.Action) error {
switch action.ActionType {
case wal.ActionTypePut:
switch configType {
case common.ConfigTypeProject:
if err := r.insertProject(tx, action.Data); err != nil {
return err
}
case common.ConfigTypeUser:
case types.ConfigTypeUser:
if err := r.insertUser(tx, action.Data); err != nil {
return err
}
case common.ConfigTypeOrg:
case types.ConfigTypeOrg:
if err := r.insertOrg(tx, action.Data); err != nil {
return err
}
case common.ConfigTypeRemoteSource:
case types.ConfigTypeProjectGroup:
if err := r.insertProjectGroup(tx, action.Data); err != nil {
return err
}
case types.ConfigTypeProject:
if err := r.insertProject(tx, action.Data); err != nil {
return err
}
case types.ConfigTypeRemoteSource:
if err := r.insertRemoteSource(tx, action.Data); err != nil {
return err
}
@ -639,22 +645,27 @@ func (r *ReadDB) applyAction(tx *db.Tx, action *wal.Action) error {
case wal.ActionTypeDelete:
switch configType {
case common.ConfigTypeProject:
r.log.Debugf("deleting project with id: %s", ID)
if err := r.deleteProject(tx, ID); err != nil {
return err
}
case common.ConfigTypeUser:
case types.ConfigTypeUser:
r.log.Debugf("deleting user with id: %s", ID)
if err := r.deleteUser(tx, ID); err != nil {
return err
}
case common.ConfigTypeOrg:
case types.ConfigTypeOrg:
r.log.Debugf("deleting org with id: %s", ID)
if err := r.deleteOrg(tx, ID); err != nil {
return err
}
case common.ConfigTypeRemoteSource:
case types.ConfigTypeProjectGroup:
r.log.Debugf("deleting project group with id: %s", ID)
if err := r.deleteProjectGroup(tx, ID); err != nil {
return err
}
case types.ConfigTypeProject:
r.log.Debugf("deleting project with id: %s", ID)
if err := r.deleteProject(tx, ID); err != nil {
return err
}
case types.ConfigTypeRemoteSource:
r.log.Debugf("deleting remote source with id: %s", ID)
if err := r.deleteRemoteSource(tx, ID); err != nil {
return err

34
internal/services/gateway/api/api.go Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"github.com/sorintlab/agola/internal/util"
)
func httpError(w http.ResponseWriter, err error) bool {
if err != nil {
if util.IsErrBadRequest(err) {
http.Error(w, err.Error(), http.StatusBadRequest)
} else {
http.Error(w, "", http.StatusInternalServerError)
}
return true
}
return false
}

80
internal/services/gateway/api/client.go
View File

@ -113,61 +113,49 @@ func (c *Client) getParsedResponse(ctx context.Context, method, path string, que
return resp, d.Decode(obj)
}
func (c *Client) GetProject(ctx context.Context, projectID string) (*types.Project, *http.Response, error) {
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/project/%s", projectID), nil, jsonContent, nil, project)
return project, resp, err
func (c *Client) GetProjectGroup(ctx context.Context, projectGroupID string) (*ProjectGroupResponse, *http.Response, error) {
projectGroup := new(ProjectGroupResponse)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s", url.PathEscape(projectGroupID)), nil, jsonContent, nil, projectGroup)
return projectGroup, resp, err
}
func (c *Client) GetCurrentUserProjects(ctx context.Context, start string, limit int, asc bool) (*GetProjectsResponse, *http.Response, error) {
return c.getProjects(ctx, "user", "", start, limit, asc)
func (c *Client) GetProjectGroupSubgroups(ctx context.Context, projectGroupID string) ([]*ProjectGroupResponse, *http.Response, error) {
projectGroups := []*ProjectGroupResponse{}
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/subgroups", url.PathEscape(projectGroupID)), nil, jsonContent, nil, &projectGroups)
return projectGroups, resp, err
}
func (c *Client) GetUserProjects(ctx context.Context, username, start string, limit int, asc bool) (*GetProjectsResponse, *http.Response, error) {
return c.getProjects(ctx, "user", username, start, limit, asc)
}
func (c *Client) GetOrgProjects(ctx context.Context, orgname, start string, limit int, asc bool) (*GetProjectsResponse, *http.Response, error) {
return c.getProjects(ctx, "org", orgname, start, limit, asc)
}
func (c *Client) getProjects(ctx context.Context, ownertype, ownername, start string, limit int, asc bool) (*GetProjectsResponse, *http.Response, error) {
q := url.Values{}
if start != "" {
q.Add("start", start)
}
if limit > 0 {
q.Add("limit", strconv.Itoa(limit))
}
if asc {
q.Add("asc", "")
}
projects := new(GetProjectsResponse)
resp, err := c.getParsedResponse(ctx, "GET", path.Join("/", ownertype, ownername, "projects"), q, jsonContent, nil, &projects)
func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupID string) ([]*ProjectResponse, *http.Response, error) {
projects := []*ProjectResponse{}
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupID)), nil, jsonContent, nil, &projects)
return projects, resp, err
}
func (c *Client) CreateCurrentUserProject(ctx context.Context, req *CreateProjectRequest) (*types.Project, *http.Response, error) {
return c.createProject(ctx, "user", "", req)
func (c *Client) GetProject(ctx context.Context, projectID string) (*types.Project, *http.Response, error) {
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s", url.PathEscape(projectID)), nil, jsonContent, nil, project)
return project, resp, err
}
func (c *Client) CreateUserProject(ctx context.Context, username string, req *CreateProjectRequest) (*types.Project, *http.Response, error) {
return c.createProject(ctx, "user", username, req)
}
func (c *Client) CreateOrgProject(ctx context.Context, orgname string, req *CreateProjectRequest) (*types.Project, *http.Response, error) {
return c.createProject(ctx, "org", orgname, req)
}
func (c *Client) createProject(ctx context.Context, ownertype, ownername string, req *CreateProjectRequest) (*types.Project, *http.Response, error) {
func (c *Client) CreateProjectGroup(ctx context.Context, req *CreateProjectGroupRequest) (*types.Project, *http.Response, error) {
reqj, err := json.Marshal(req)
if err != nil {
return nil, nil, err
}
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "PUT", path.Join("/", ownertype, ownername, "projects"), nil, jsonContent, bytes.NewReader(reqj), project)
resp, err := c.getParsedResponse(ctx, "PUT", "/projectgroups", nil, jsonContent, bytes.NewReader(reqj), project)
return project, resp, err
}
func (c *Client) CreateProject(ctx context.Context, req *CreateProjectRequest) (*types.Project, *http.Response, error) {
reqj, err := json.Marshal(req)
if err != nil {
return nil, nil, err
}
project := new(types.Project)
resp, err := c.getParsedResponse(ctx, "PUT", "/projects", nil, jsonContent, bytes.NewReader(reqj), project)
return project, resp, err
}
@ -197,7 +185,7 @@ func (c *Client) GetUser(ctx context.Context, userID string) (*types.User, *http
return user, resp, err
}
func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) (*UsersResponse, *http.Response, error) {
func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) ([]*UserResponse, *http.Response, error) {
q := url.Values{}
if start != "" {
q.Add("start", start)
@ -209,7 +197,7 @@ func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool
q.Add("asc", "")
}
users := new(UsersResponse)
users := []*UserResponse{}
resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users)
return users, resp, err
}
@ -261,7 +249,7 @@ func (c *Client) GetRun(ctx context.Context, runID string) (*RunResponse, *http.
return run, resp, err
}
func (c *Client) GetRuns(ctx context.Context, phaseFilter, groups, runGroups []string, start string, limit int, asc bool) (*GetRunsResponse, *http.Response, error) {
func (c *Client) GetRuns(ctx context.Context, phaseFilter, groups, runGroups []string, start string, limit int, asc bool) ([]*RunsResponse, *http.Response, error) {
q := url.Values{}
for _, phase := range phaseFilter {
q.Add("phase", phase)
@ -282,7 +270,7 @@ func (c *Client) GetRuns(ctx context.Context, phaseFilter, groups, runGroups []s
q.Add("asc", "")
}
getRunsResponse := new(GetRunsResponse)
getRunsResponse := []*RunsResponse{}
resp, err := c.getParsedResponse(ctx, "GET", "/runs", q, jsonContent, nil, getRunsResponse)
return getRunsResponse, resp, err
}
@ -293,7 +281,7 @@ func (c *Client) GetRemoteSource(ctx context.Context, rsID string) (*RemoteSourc
return rs, resp, err
}
func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) (*RemoteSourcesResponse, *http.Response, error) {
func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) ([]*RemoteSourceResponse, *http.Response, error) {
q := url.Values{}
if start != "" {
q.Add("start", start)
@ -305,7 +293,7 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int,
q.Add("asc", "")
}
rss := new(RemoteSourcesResponse)
rss := []*RemoteSourceResponse{}
resp, err := c.getParsedResponse(ctx, "GET", "/remotesources", q, jsonContent, nil, &rss)
return rss, resp, err
}

224
internal/services/gateway/api/project.go
View File

@ -15,13 +15,11 @@
package api
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strconv"
"net/url"
"github.com/pkg/errors"
csapi "github.com/sorintlab/agola/internal/services/configstore/api"
"github.com/sorintlab/agola/internal/services/gateway/command"
"github.com/sorintlab/agola/internal/services/types"
@ -31,10 +29,11 @@ import (
)
type CreateProjectRequest struct {
Name string `json:"name"`
RepoURL string `json:"repo_url"`
RemoteSourceName string `json:"remote_source_name"`
SkipSSHHostKeyCheck bool `json:"skip_ssh_host_key_check"`
Name string `json:"name,omitempty"`
ParentID string `json:"parent_id,omitempty"`
RepoURL string `json:"repo_url,omitempty"`
RemoteSourceName string `json:"remote_source_name,omitempty"`
SkipSSHHostKeyCheck bool `json:"skip_ssh_host_key_check,omitempty"`
}
type CreateProjectHandler struct {
@ -50,8 +49,6 @@ func NewCreateProjectHandler(logger *zap.Logger, ch *command.CommandHandler, con
func (h *CreateProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
orgname := vars["orgname"]
var req CreateProjectRequest
d := json.NewDecoder(r.Body)
@ -70,23 +67,13 @@ func (h *CreateProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
creq := &command.CreateProjectRequest{
Name: req.Name,
ParentID: req.ParentID,
RepoURL: req.RepoURL,
RemoteSourceName: req.RemoteSourceName,
UserID: userID,
CurrentUserID: userID,
SkipSSHHostKeyCheck: req.SkipSSHHostKeyCheck,
}
ownerID, code, userErr, err := getOwnerID(ctx, h.configstoreClient, "", orgname, true)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, userErr, code)
return
}
if orgname != "" {
creq.OwnerType = types.OwnerTypeOrganization
creq.OwnerID = ownerID
}
project, err := h.ch.CreateProject(ctx, creq)
if err != nil {
h.log.Errorf("err: %+v", err)
@ -115,18 +102,13 @@ func NewProjectReconfigHandler(logger *zap.Logger, ch *command.CommandHandler, c
func (h *ProjectReconfigHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectName := vars["projectname"]
username := vars["username"]
orgname := vars["orgname"]
ownerID, code, userErr, err := getOwnerID(ctx, h.configstoreClient, username, orgname, false)
projectID, err := url.PathUnescape(vars["projectid"])
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, userErr, code)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.ch.ReconfigProject(ctx, ownerID, projectName); err != nil {
if err := h.ch.ReconfigProject(ctx, projectID); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -145,21 +127,16 @@ func NewDeleteProjectHandler(logger *zap.Logger, configstoreClient *csapi.Client
func (h *DeleteProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectName := vars["projectname"]
username := vars["username"]
orgname := vars["orgname"]
ownerID, code, userErr, err := getOwnerID(ctx, h.configstoreClient, username, orgname, true)
projectID, err := url.PathUnescape(vars["projectid"])
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, userErr, code)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
project, resp, err := h.configstoreClient.GetProjectByName(ctx, ownerID, projectName)
project, resp, err := h.configstoreClient.GetProject(ctx, projectID)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, fmt.Sprintf("project with name %q doesn't exist", projectName), http.StatusNotFound)
http.Error(w, fmt.Sprintf("project with id %q doesn't exist", projectID), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
@ -191,7 +168,11 @@ func NewProjectHandler(logger *zap.Logger, configstoreClient *csapi.Client) *Pro
func (h *ProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectID := vars["projectid"]
projectID, err := url.PathUnescape(vars["projectid"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
project, resp, err := h.configstoreClient.GetProject(ctx, projectID)
if err != nil {
@ -212,177 +193,16 @@ func (h *ProjectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
type ProjectByNameHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
}
func NewProjectByNameHandler(logger *zap.Logger, configstoreClient *csapi.Client) *ProjectByNameHandler {
return &ProjectByNameHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
}
func (h *ProjectByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectName := vars["projectname"]
username := vars["username"]
orgname := vars["orgname"]
ownerID, code, userErr, err := getOwnerID(ctx, h.configstoreClient, username, orgname, false)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, userErr, code)
return
}
project, resp, err := h.configstoreClient.GetProjectByName(ctx, ownerID, projectName)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
res := createProjectResponse(project)
if err := json.NewEncoder(w).Encode(res); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type GetProjectsResponse struct {
Projects []*ProjectResponse `json:"projects"`
}
type ProjectResponse struct {
ID string `json:"id"`
Name string `json:"name"`
}
func createProjectResponse(r *types.Project) *ProjectResponse {
run := &ProjectResponse{
res := &ProjectResponse{
ID: r.ID,
Name: r.Name,
}
return run
}
type ProjectsHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
}
func NewProjectsHandler(logger *zap.Logger, configstoreClient *csapi.Client) *ProjectsHandler {
return &ProjectsHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
}
func (h *ProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
query := r.URL.Query()
username := vars["username"]
orgname := vars["orgname"]
ownerID, code, userErr, err := getOwnerID(ctx, h.configstoreClient, username, orgname, true)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, userErr, code)
return
}
limitS := query.Get("limit")
limit := DefaultRunsLimit
if limitS != "" {
var err error
limit, err = strconv.Atoi(limitS)
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
}
if limit < 0 {
http.Error(w, "limit must be greater or equal than 0", http.StatusBadRequest)
return
}
if limit > MaxRunsLimit {
limit = MaxRunsLimit
}
asc := false
if _, ok := query["asc"]; ok {
asc = true
}
start := query.Get("start")
csprojects, resp, err := h.configstoreClient.GetOwnerProjects(ctx, ownerID, start, limit, asc)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
projects := make([]*ProjectResponse, len(csprojects))
for i, p := range csprojects {
projects[i] = createProjectResponse(p)
}
getProjectsResponse := &GetProjectsResponse{
Projects: projects,
}
if err := json.NewEncoder(w).Encode(getProjectsResponse); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func getOwnerID(ctx context.Context, configstoreClient *csapi.Client, username, orgname string, useAuthUser bool) (string, int, string, error) {
var ownerID string
switch {
case username != "":
user, resp, err := configstoreClient.GetUserByName(ctx, username)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
err = errors.Errorf("user %q doens't exist", username)
return "", http.StatusNotFound, err.Error(), err
}
return "", http.StatusInternalServerError, "", err
}
ownerID = user.ID
case orgname != "":
org, resp, err := configstoreClient.GetOrgByName(ctx, orgname)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
err = errors.Errorf("organization %q doens't exist", orgname)
return "", http.StatusNotFound, err.Error(), err
}
return "", http.StatusInternalServerError, "", err
}
ownerID = org.ID
default:
if useAuthUser {
// use the current authenticated user
ctxUserID := ctx.Value("userid")
if ctxUserID == nil {
err := errors.New("no authenticated user")
return "", http.StatusBadRequest, err.Error(), err
}
ownerID = ctxUserID.(string)
} else {
err := errors.New("no user or org name specified")
return "", http.StatusBadRequest, err.Error(), err
}
}
return ownerID, 0, "", nil
return res
}

218
internal/services/gateway/api/projectgroup.go Normal file
View File

@ -0,0 +1,218 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"encoding/json"
"net/http"
"net/url"
csapi "github.com/sorintlab/agola/internal/services/configstore/api"
"github.com/sorintlab/agola/internal/services/gateway/command"
"github.com/sorintlab/agola/internal/services/types"
"github.com/gorilla/mux"
"go.uber.org/zap"
)
type CreateProjectGroupRequest struct {
Name string `json:"name,omitempty"`
ParentID string `json:"parent_id,omitempty"`
}
type CreateProjectGroupHandler struct {
log *zap.SugaredLogger
ch *command.CommandHandler
configstoreClient *csapi.Client
exposedURL string
}
func NewCreateProjectGroupHandler(logger *zap.Logger, ch *command.CommandHandler, configstoreClient *csapi.Client, exposedURL string) *CreateProjectGroupHandler {
return &CreateProjectGroupHandler{log: logger.Sugar(), ch: ch, configstoreClient: configstoreClient, exposedURL: exposedURL}
}
func (h *CreateProjectGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req CreateProjectGroupRequest
d := json.NewDecoder(r.Body)
if err := d.Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
ctxUserID := ctx.Value("userid")
if ctxUserID == nil {
http.Error(w, "no authenticated user", http.StatusBadRequest)
return
}
userID := ctxUserID.(string)
h.log.Infof("userID: %q", userID)
creq := &command.CreateProjectGroupRequest{
Name: req.Name,
ParentID: req.ParentID,
CurrentUserID: userID,
}
projectGroup, err := h.ch.CreateProjectGroup(ctx, creq)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode(projectGroup); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
}
func NewProjectGroupHandler(logger *zap.Logger, configstoreClient *csapi.Client) *ProjectGroupHandler {
return &ProjectGroupHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
}
func (h *ProjectGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectGroupID, err := url.PathUnescape(vars["projectgroupid"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
h.log.Infof("projectGroupID: %s", projectGroupID)
projectGroup, resp, err := h.configstoreClient.GetProjectGroup(ctx, projectGroupID)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
res := createProjectGroupResponse(projectGroup)
if err := json.NewEncoder(w).Encode(res); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupProjectsHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
}
func NewProjectGroupProjectsHandler(logger *zap.Logger, configstoreClient *csapi.Client) *ProjectGroupProjectsHandler {
return &ProjectGroupProjectsHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
}
func (h *ProjectGroupProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectGroupID, err := url.PathUnescape(vars["projectgroupid"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
h.log.Infof("projectGroupID: %s", projectGroupID)
csprojects, resp, err := h.configstoreClient.GetProjectGroupProjects(ctx, projectGroupID)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
projects := make([]*ProjectResponse, len(csprojects))
for i, p := range csprojects {
projects[i] = createProjectResponse(p)
}
if err := json.NewEncoder(w).Encode(projects); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupSubgroupsHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
}
func NewProjectGroupSubgroupsHandler(logger *zap.Logger, configstoreClient *csapi.Client) *ProjectGroupSubgroupsHandler {
return &ProjectGroupSubgroupsHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
}
func (h *ProjectGroupSubgroupsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
projectGroupID, err := url.PathUnescape(vars["projectgroupid"])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
h.log.Infof("projectGroupID: %s", projectGroupID)
cssubgroups, resp, err := h.configstoreClient.GetProjectGroupSubgroups(ctx, projectGroupID)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
subgroups := make([]*ProjectGroupResponse, len(cssubgroups))
for i, g := range cssubgroups {
subgroups[i] = createProjectGroupResponse(g)
}
if err := json.NewEncoder(w).Encode(subgroups); err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
type ProjectGroupResponse struct {
ID string `json:"id"`
Name string `json:"name"`
}
func createProjectGroupResponse(r *types.ProjectGroup) *ProjectGroupResponse {
run := &ProjectGroupResponse{
ID: r.ID,
Name: r.Name,
}
return run
}

9
internal/services/gateway/api/remotesource.go
View File

@ -122,10 +122,6 @@ func (h *CreateRemoteSourceHandler) createRemoteSource(ctx context.Context, req
return rs, nil
}
type RemoteSourcesResponse struct {
RemoteSources []*RemoteSourceResponse `json:"remote_sources"`
}
type RemoteSourceResponse struct {
ID string `json:"id"`
Name string `json:"name"`
@ -224,11 +220,8 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
for i, rs := range csRemoteSources {
remoteSources[i] = createRemoteSourceResponse(rs)
}
remoteSourcesResponse := &RemoteSourcesResponse{
RemoteSources: remoteSources,
}
if err := json.NewEncoder(w).Encode(remoteSourcesResponse); err != nil {
if err := json.NewEncoder(w).Encode(remoteSources); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

9
internal/services/gateway/api/run.go
View File

@ -260,10 +260,6 @@ const (
MaxRunsLimit = 40
)
type GetRunsResponse struct {
Runs []*RunsResponse `json:"runs"`
}
func createRunsResponse(r *rstypes.Run) *RunsResponse {
run := &RunsResponse{
ID: r.ID,
@ -339,11 +335,8 @@ func (h *RunsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
for i, r := range runsResp.Runs {
runs[i] = createRunsResponse(r)
}
getRunsResponse := &GetRunsResponse{
Runs: runs,
}
if err := json.NewEncoder(w).Encode(getRunsResponse); err != nil {
if err := json.NewEncoder(w).Encode(runs); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

49
internal/services/gateway/api/user.go
View File

@ -36,11 +36,12 @@ type CreateUserRequest struct {
type CreateUserHandler struct {
log *zap.SugaredLogger
ch *command.CommandHandler
configstoreClient *csapi.Client
}
func NewCreateUserHandler(logger *zap.Logger, configstoreClient *csapi.Client) *CreateUserHandler {
return &CreateUserHandler{log: logger.Sugar(), configstoreClient: configstoreClient}
func NewCreateUserHandler(logger *zap.Logger, ch *command.CommandHandler, configstoreClient *csapi.Client) *CreateUserHandler {
return &CreateUserHandler{log: logger.Sugar(), ch: ch, configstoreClient: configstoreClient}
}
func (h *CreateUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -53,40 +54,24 @@ func (h *CreateUserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
user, err := h.createUser(ctx, &req)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
creq := &command.CreateUserRequest{
UserName: req.UserName,
}
u, err := h.ch.CreateUser(ctx, creq)
if httpError(w, err) {
return
}
if err := json.NewEncoder(w).Encode(user); err != nil {
res := createUserResponse(u)
if err := json.NewEncoder(w).Encode(res); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func (h *CreateUserHandler) createUser(ctx context.Context, req *CreateUserRequest) (*UserResponse, error) {
if !util.ValidateName(req.UserName) {
return nil, errors.Errorf("invalid user name %q", req.UserName)
}
u := &types.User{
UserName: req.UserName,
}
h.log.Infof("creating user")
u, _, err := h.configstoreClient.CreateUser(ctx, u)
if err != nil {
return nil, errors.Wrapf(err, "failed to create user")
}
h.log.Infof("user %s created, ID: %s", u.UserName, u.ID)
res := createUserResponse(u)
return res, nil
}
type DeleteUserHandler struct {
log *zap.SugaredLogger
configstoreClient *csapi.Client
@ -210,10 +195,6 @@ func (h *UserByNameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
type UsersResponse struct {
Users []*UserResponse `json:"users"`
}
type UserResponse struct {
ID string `json:"id"`
UserName string `json:"username"`
@ -279,11 +260,8 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
for i, p := range csusers {
users[i] = createUserResponse(p)
}
usersResponse := &UsersResponse{
Users: users,
}
if err := json.NewEncoder(w).Encode(usersResponse); err != nil {
if err := json.NewEncoder(w).Encode(users); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -440,6 +418,7 @@ func (h *CreateUserTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
h.log.Infof("creating user %q token", userName)
cresp, _, err := h.configstoreClient.CreateUserToken(ctx, userName, creq)
if err != nil {
h.log.Errorf("err: %+v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

41
internal/services/gateway/command/project.go
View File

@ -30,11 +30,10 @@ import (
type CreateProjectRequest struct {
Name string
ParentID string
RemoteSourceName string
RepoURL string
UserID string
OwnerType types.OwnerType
OwnerID string
CurrentUserID string
SkipSSHHostKeyCheck bool
}
@ -65,9 +64,9 @@ func (c *CommandHandler) CreateProject(ctx context.Context, req *CreateProjectRe
return nil, errors.Wrapf(err, "failed to generate ssh key pair")
}
user, _, err := c.configstoreClient.GetUser(ctx, req.UserID)
user, _, err := c.configstoreClient.GetUser(ctx, req.CurrentUserID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get user %q", req.UserID)
return nil, errors.Wrapf(err, "failed to get user %q", req.CurrentUserID)
}
rs, _, err := c.configstoreClient.GetRemoteSourceByName(ctx, req.RemoteSourceName)
if err != nil {
@ -86,25 +85,25 @@ func (c *CommandHandler) CreateProject(ctx context.Context, req *CreateProjectRe
return nil, errors.Errorf("user doesn't have a linked account for remote source %q", rs.Name)
}
parentID := req.ParentID
if parentID == "" {
// create project in current user namespace
parentID = path.Join("user", user.UserName)
}
p := &types.Project{
Name: req.Name,
OwnerType: types.OwnerTypeUser,
OwnerID: user.ID,
Name: req.Name,
Parent: types.Parent{
Type: types.ConfigTypeProjectGroup,
ID: parentID,
},
LinkedAccountID: la.ID,
Path: fmt.Sprintf("%s/%s", repoOwner, repoName),
RepoPath: fmt.Sprintf("%s/%s", repoOwner, repoName),
CloneURL: cloneURL,
SkipSSHHostKeyCheck: req.SkipSSHHostKeyCheck,
SSHPrivateKey: string(privateKey),
}
if req.OwnerType == types.OwnerTypeOrganization {
if req.OwnerID == "" {
return nil, errors.Errorf("ownerid must be specified when adding a project outside the current user")
}
p.OwnerType = req.OwnerType
p.OwnerID = req.OwnerID
}
c.log.Infof("creating project")
p, _, err = c.configstoreClient.CreateProject(ctx, p)
if err != nil {
@ -157,8 +156,8 @@ func (c *CommandHandler) SetupProject(ctx context.Context, rs *types.RemoteSourc
return nil
}
func (c *CommandHandler) ReconfigProject(ctx context.Context, ownerID, projectName string) error {
p, _, err := c.configstoreClient.GetProjectByName(ctx, ownerID, projectName)
func (c *CommandHandler) ReconfigProject(ctx context.Context, projectID string) error {
p, _, err := c.configstoreClient.GetProject(ctx, projectID)
if err != nil {
return err
}
@ -179,8 +178,8 @@ func (c *CommandHandler) ReconfigProject(ctx context.Context, ownerID, projectNa
return errors.Wrapf(err, "failed to get remote source %q", la.RemoteSourceID)
}
repoOwner := strings.TrimPrefix(path.Dir(p.Path), "/")
repoName := path.Base(p.Path)
repoOwner := strings.TrimPrefix(path.Dir(p.RepoPath), "/")
repoName := path.Base(p.RepoPath)
return c.SetupProject(ctx, rs, la, &SetupProjectRequest{
Project: p,

65
internal/services/gateway/command/projectgroup.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.
package command
import (
"context"
"path"
"github.com/sorintlab/agola/internal/services/types"
"github.com/sorintlab/agola/internal/util"
"github.com/pkg/errors"
)
type CreateProjectGroupRequest struct {
Name string
ParentID string
CurrentUserID string
}
func (c *CommandHandler) CreateProjectGroup(ctx context.Context, req *CreateProjectGroupRequest) (*types.ProjectGroup, error) {
if !util.ValidateName(req.Name) {
return nil, errors.Errorf("invalid projectGroup name %q", req.Name)
}
user, _, err := c.configstoreClient.GetUser(ctx, req.CurrentUserID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get user %q", req.CurrentUserID)
}
parentID := req.ParentID
if parentID == "" {
// create projectGroup in current user namespace
parentID = path.Join("user", user.UserName)
}
p := &types.ProjectGroup{
Name: req.Name,
Parent: types.Parent{
Type: types.ConfigTypeProjectGroup,
ID: parentID,
},
}
c.log.Infof("creating projectGroup")
p, _, err = c.configstoreClient.CreateProjectGroup(ctx, p)
if err != nil {
return nil, errors.Wrapf(err, "failed to create projectGroup")
}
c.log.Infof("projectGroup %s created, ID: %s", p.Name, p.ID)
return p, nil
}

26
internal/services/gateway/command/user.go
View File

@ -27,6 +27,32 @@ import (
"github.com/pkg/errors"
)
type CreateUserRequest struct {
UserName string
}
func (c *CommandHandler) CreateUser(ctx context.Context, req *CreateUserRequest) (*types.User, error) {
if req.UserName == "" {
return nil, util.NewErrBadRequest(errors.Errorf("user name required"))
}
if !util.ValidateName(req.UserName) {
return nil, errors.Errorf("invalid user name %q", req.UserName)
}
u := &types.User{
UserName: req.UserName,
}
c.log.Infof("creating user")
u, _, err := c.configstoreClient.CreateUser(ctx, u)
if err != nil {
return nil, errors.Wrapf(err, "failed to create user")
}
c.log.Infof("user %s created, ID: %s", u.UserName, u.ID)
return u, nil
}
type CreateUserLARequest struct {
UserName string
RemoteSourceName string

35
internal/services/gateway/gateway.go
View File

@ -143,9 +143,12 @@ func (g *Gateway) Run(ctx context.Context) error {
webhooksHandler := &webhooksHandler{log: log, configstoreClient: g.configstoreClient, runserviceClient: g.runserviceClient, apiExposedURL: g.c.APIExposedURL}
projectGroupHandler := api.NewProjectGroupHandler(logger, g.configstoreClient)
projectGroupSubgroupsHandler := api.NewProjectGroupSubgroupsHandler(logger, g.configstoreClient)
projectGroupProjectsHandler := api.NewProjectGroupProjectsHandler(logger, g.configstoreClient)
createProjectGroupHandler := api.NewCreateProjectGroupHandler(logger, g.ch, g.configstoreClient, g.c.APIExposedURL)
projectHandler := api.NewProjectHandler(logger, g.configstoreClient)
projectByNameHandler := api.NewProjectByNameHandler(logger, g.configstoreClient)
projectsHandler := api.NewProjectsHandler(logger, g.configstoreClient)
createProjectHandler := api.NewCreateProjectHandler(logger, g.ch, g.configstoreClient, g.c.APIExposedURL)
deleteProjectHandler := api.NewDeleteProjectHandler(logger, g.configstoreClient)
projectReconfigHandler := api.NewProjectReconfigHandler(logger, g.ch, g.configstoreClient, g.c.APIExposedURL)
@ -154,7 +157,7 @@ func (g *Gateway) Run(ctx context.Context) error {
userHandler := api.NewUserHandler(logger, g.configstoreClient)
userByNameHandler := api.NewUserByNameHandler(logger, g.configstoreClient)
usersHandler := api.NewUsersHandler(logger, g.configstoreClient)
createUserHandler := api.NewCreateUserHandler(logger, g.configstoreClient)
createUserHandler := api.NewCreateUserHandler(logger, g.ch, g.configstoreClient)
deleteUserHandler := api.NewDeleteUserHandler(logger, g.configstoreClient)
createUserLAHandler := api.NewCreateUserLAHandler(logger, g.ch, g.configstoreClient)
@ -185,7 +188,7 @@ func (g *Gateway) Run(ctx context.Context) error {
router := mux.NewRouter()
apirouter := mux.NewRouter().PathPrefix("/api/v1alpha").Subrouter()
apirouter := mux.NewRouter().PathPrefix("/api/v1alpha").Subrouter().UseEncodedPath()
authForcedHandler := handlers.NewAuthHandler(logger, g.configstoreClient, g.c.AdminToken, g.sd, true)
authOptionalHandler := handlers.NewAuthHandler(logger, g.configstoreClient, g.c.AdminToken, g.sd, false)
@ -194,19 +197,17 @@ func (g *Gateway) Run(ctx context.Context) error {
apirouter.Handle("/logs", logsHandler).Methods("GET")
apirouter.Handle("/project/{projectid}", authForcedHandler(projectHandler)).Methods("GET")
apirouter.Handle("/user/projects", authForcedHandler(projectsHandler)).Methods("GET")
apirouter.Handle("/user/{username}/projects", authForcedHandler(projectsHandler)).Methods("GET")
apirouter.Handle("/org/{orgname}/projects", authForcedHandler(projectsHandler)).Methods("GET")
apirouter.Handle("/user/projects", authForcedHandler(createProjectHandler)).Methods("PUT")
apirouter.Handle("/org/{orgname}/projects", authForcedHandler(createProjectHandler)).Methods("PUT")
apirouter.Handle("/projects/user/{username}/{projectname}", authForcedHandler(projectByNameHandler)).Methods("GET")
apirouter.Handle("/projects/org/{orgname}/{projectname}", authForcedHandler(projectByNameHandler)).Methods("GET")
apirouter.Handle("/projects/user/{projectname}", authForcedHandler(deleteProjectHandler)).Methods("DELETE")
apirouter.Handle("/projects/user/{username}/{projectname}", authForcedHandler(deleteProjectHandler)).Methods("DELETE")
apirouter.Handle("/projects/org/{orgname}/{projectname}", authForcedHandler(deleteProjectHandler)).Methods("DELETE")
apirouter.Handle("/projects/user/{username}/{projectname}/reconfig", authForcedHandler(projectReconfigHandler)).Methods("POST")
apirouter.Handle("/projects/org/{orgname}/{projectname}/reconfig", authForcedHandler(projectReconfigHandler)).Methods("POST")
//apirouter.Handle("/projectgroups", authForcedHandler(projectsHandler)).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupid}", authForcedHandler(projectGroupHandler)).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupid}/subgroups", authForcedHandler(projectGroupSubgroupsHandler)).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupid}/projects", authForcedHandler(projectGroupProjectsHandler)).Methods("GET")
apirouter.Handle("/projectgroups", authForcedHandler(createProjectGroupHandler)).Methods("PUT")
//apirouter.Handle("/projectgroups/{projectgroupid}", authForcedHandler(deleteProjectGroupHandler)).Methods("DELETE")
apirouter.Handle("/projects/{projectid}", authForcedHandler(projectHandler)).Methods("GET")
apirouter.Handle("/projects", authForcedHandler(createProjectHandler)).Methods("PUT")
apirouter.Handle("/projects/{projectid}", authForcedHandler(deleteProjectHandler)).Methods("DELETE")
apirouter.Handle("/projects/{projectid}/reconfig", authForcedHandler(projectReconfigHandler)).Methods("POST")
apirouter.Handle("/user", authForcedHandler(currentUserHandler)).Methods("GET")
apirouter.Handle("/user/{userid}", authForcedHandler(userHandler)).Methods("GET")

41
internal/services/types/types.go
View File

@ -21,6 +21,21 @@ import (
// Configstore types
type ConfigType string
const (
ConfigTypeUser ConfigType = "user"
ConfigTypeOrg ConfigType = "org"
ConfigTypeProjectGroup ConfigType = "projectgroup"
ConfigTypeProject ConfigType = "project"
ConfigTypeRemoteSource ConfigType = "remotesource"
)
type Parent struct {
Type ConfigType `json:"type,omitempty"`
ID string `json:"id,omitempty"`
}
type User struct {
// The type version. Increase when a breaking change is done. Usually not
// needed when adding fields.
@ -48,6 +63,16 @@ type Organization struct {
Name string `json:"name,omitempty"`
}
type ProjectGroup struct {
Version string `json:"version,omitempty"`
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Parent Parent `json:"parent,omitempty"`
}
type RemoteSourceType string
const (
@ -102,17 +127,6 @@ type LinkedAccount struct {
Oauth2Expire time.Duration `json:"oauth2_expire,omitempty"`
}
type OwnerType string
const (
OwnerTypeUser OwnerType = "user"
OwnerTypeOrganization OwnerType = "organization"
)
func IsValidOwnerType(ownerType OwnerType) bool {
return ownerType == OwnerTypeUser || ownerType == OwnerTypeOrganization
}
type Project struct {
// The type version. Increase when a breaking change is done. Usually not
// needed when adding fields.
@ -121,14 +135,13 @@ type Project struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
OwnerType OwnerType `json:"owner_type,omitempty"`
OwnerID string `json:"owner_id,omitempty"`
Parent Parent `json:"parent,omitempty"`
// Project repository path. It may be different for every kind of git source.
// It's needed to get git source needed information like the repo owner and
// repo user
// Examples: sgotti/agola (for github, gitea etc... sources)
Path string `json:"path,omitempty"`
RepoPath string `json:"repo_path,omitempty"`
LinkedAccountID string `json:"linked_account_id,omitempty"`