a
/
agola
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #157 from camandel/api_delete_logs 

gateway/runservice: add api to delete step logs
This commit is contained in:
Simone Gotti 2019-11-18 13:58:25 +01:00 committed by GitHub
parent 3ff18510a6 3e47bc601a
commit f5998a78d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 401 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
internal/services/gateway/action/run.go
View File

@ -145,6 +145,34 @@ func (h *ActionHandler) GetLogs(ctx context.Context, req *GetLogsRequest) (*http
return resp, nil
}
type DeleteLogsRequest struct {
RunID string
TaskID string
Setup bool
Step int
}
func (h *ActionHandler) DeleteLogs(ctx context.Context, req *DeleteLogsRequest) error {
runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
if err != nil {
return ErrFromRemote(resp, err)
}
canDoRunActions, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group)
if err != nil {
return errors.Errorf("failed to determine permissions: %w", err)
}
if !canDoRunActions {
return util.NewErrForbidden(errors.Errorf("user not authorized"))
}
resp, err = h.runserviceClient.DeleteLogs(ctx, req.RunID, req.TaskID, req.Setup, req.Step)
if err != nil {
return ErrFromRemote(resp, err)
}
return nil
}
type RunActionType string
const (

60
internal/services/gateway/api/run.go
View File

@ -502,3 +502,63 @@ func sendLogs(w io.Writer, r io.Reader) error {
}
}
}
type LogsDeleteHandler struct {
log *zap.SugaredLogger
ah *action.ActionHandler
}
func NewLogsDeleteHandler(logger *zap.Logger, ah *action.ActionHandler) *LogsDeleteHandler {
return &LogsDeleteHandler{log: logger.Sugar(), ah: ah}
}
func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
q := r.URL.Query()
runID := q.Get("runID")
if runID == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("empty run id")))
return
}
taskID := q.Get("taskID")
if taskID == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("empty task id")))
return
}
_, setup := q["setup"]
stepStr := q.Get("step")
if !setup && stepStr == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("no setup or step number provided")))
return
}
if setup && stepStr != "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("both setup and step number provided")))
return
}
var step int
if stepStr != "" {
var err error
step, err = strconv.Atoi(stepStr)
if err != nil {
httpError(w, util.NewErrBadRequest(errors.Errorf("cannot parse step number: %w", err)))
return
}
}
areq := &action.DeleteLogsRequest{
RunID: runID,
TaskID: taskID,
Setup: setup,
Step: step,
}
err := h.ah.DeleteLogs(ctx, areq)
if httpError(w, err) {
h.log.Errorf("err: %+v", err)
return
}
}

2
internal/services/gateway/gateway.go
View File

@ -210,6 +210,7 @@ func (g *Gateway) Run(ctx context.Context) error {
runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, g.ah)
logsHandler := api.NewLogsHandler(logger, g.ah)
logsDeleteHandler := api.NewLogsDeleteHandler(logger, g.ah)
userRemoteReposHandler := api.NewUserRemoteReposHandler(logger, g.ah, g.configstoreClient)
@ -235,6 +236,7 @@ func (g *Gateway) Run(ctx context.Context) error {
router.PathPrefix("/api/v1alpha").Handler(apirouter)
apirouter.Handle("/logs", authOptionalHandler(logsHandler)).Methods("GET")
apirouter.Handle("/logs", authForcedHandler(logsDeleteHandler)).Methods("DELETE")
//apirouter.Handle("/projectgroups", authForcedHandler(projectsHandler)).Methods("GET")
apirouter.Handle("/projectgroups/{projectgroupref}", authForcedHandler(projectGroupHandler)).Methods("GET")

100
internal/services/runservice/api/api.go
View File

@ -323,6 +323,106 @@ func sendLogs(w http.ResponseWriter, r io.Reader) error {
}
}
type LogsDeleteHandler struct {
log *zap.SugaredLogger
e *etcd.Store
ost *objectstorage.ObjStorage
dm *datamanager.DataManager
}
func NewLogsDeleteHandler(logger *zap.Logger, e *etcd.Store, ost *objectstorage.ObjStorage, dm *datamanager.DataManager) *LogsDeleteHandler {
return &LogsDeleteHandler{
log: logger.Sugar(),
e: e,
ost: ost,
dm: dm,
}
}
func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
q := r.URL.Query()
runID := q.Get("runid")
if runID == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("runid is empty")))
return
}
taskID := q.Get("taskid")
if taskID == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("taskid is empty")))
return
}
_, setup := q["setup"]
stepStr := q.Get("step")
if !setup && stepStr == "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("setup is false and step is empty")))
return
}
if setup && stepStr != "" {
httpError(w, util.NewErrBadRequest(errors.Errorf("setup is true and step is %s", stepStr)))
return
}
var step int
if stepStr != "" {
var err error
step, err = strconv.Atoi(stepStr)
if err != nil {
httpError(w, util.NewErrBadRequest(errors.Errorf("step %s is not a valid number", stepStr)))
return
}
}
if err := h.deleteTaskLogs(ctx, runID, taskID, setup, step, w); err != nil {
h.log.Errorf("err: %+v", err)
switch {
case util.IsNotExist(err):
httpError(w, util.NewErrNotExist(errors.Errorf("log doesn't exist: %w", err)))
default:
httpError(w, err)
}
}
}
func (h *LogsDeleteHandler) deleteTaskLogs(ctx context.Context, runID, taskID string, setup bool, step int, w http.ResponseWriter) error {
r, err := store.GetRunEtcdOrOST(ctx, h.e, h.dm, runID)
if err != nil {
return err
}
if r == nil {
return util.NewErrNotExist(errors.Errorf("no such run with id: %s", runID))
}
task, ok := r.Tasks[taskID]
if !ok {
return util.NewErrNotExist(errors.Errorf("no such task with ID %s in run %s", taskID, runID))
}
if len(task.Steps) <= step {
return util.NewErrNotExist(errors.Errorf("no such step for task %s in run %s", taskID, runID))
}
if task.Steps[step].LogPhase == types.RunTaskFetchPhaseFinished {
var logPath string
if setup {
logPath = store.OSTRunTaskSetupLogPath(task.ID)
} else {
logPath = store.OSTRunTaskStepLogPath(task.ID, step)
}
err := h.ost.DeleteObject(logPath)
if err != nil {
if objectstorage.IsNotExist(err) {
return util.NewErrNotExist(err)
}
return err
}
return nil
}
return util.NewErrBadRequest(errors.Errorf("Log for task %s in run %s is not yet archived", taskID, runID))
}
type ChangeGroupsUpdateTokensHandler struct {
log *zap.SugaredLogger
readDB *readdb.ReadDB

2
internal/services/runservice/runservice.go
View File

@ -221,6 +221,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand
executorDeleteHandler := api.NewExecutorDeleteHandler(logger, s.ah)
logsHandler := api.NewLogsHandler(logger, s.e, s.ost, s.dm)
logsDeleteHandler := api.NewLogsDeleteHandler(logger, s.e, s.ost, s.dm)
runHandler := api.NewRunHandler(logger, s.e, s.dm, s.readDB)
runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, s.ah)
@ -248,6 +249,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand
apirouter.Handle("/executor/caches/{key}", cacheCreateHandler).Methods("POST")
apirouter.Handle("/logs", logsHandler).Methods("GET")
apirouter.Handle("/logs", logsDeleteHandler).Methods("DELETE")
apirouter.Handle("/runs/events", runEventsHandler).Methods("GET")
apirouter.Handle("/runs/{runid}", runHandler).Methods("GET")

19
services/gateway/client/client.go
View File

@ -455,6 +455,12 @@ func (c *Client) GetRun(ctx context.Context, runID string) (*gwapitypes.RunRespo
return run, resp, err
}
func (c *Client) GetRunTask(ctx context.Context, runID, taskID string) (*gwapitypes.RunTaskResponse, *http.Response, error) {
task := new(gwapitypes.RunTaskResponse)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/run/%s/task/%s", runID, taskID), nil, jsonContent, nil, task)
return task, resp, err
}
func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups, runGroups []string, start string, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) {
q := url.Values{}
for _, phase := range phaseFilter {
@ -497,6 +503,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool,
return c.getResponse(ctx, "GET", "/logs", q, nil, nil)
}
func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
q := url.Values{}
q.Add("runID", runID)
q.Add("taskID", taskID)
if setup {
q.Add("setup", "")
} else {
q.Add("step", strconv.Itoa(step))
}
return c.getResponse(ctx, "DELETE", "/logs", q, nil, nil)
}
func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) {
rs := new(gwapitypes.RemoteSourceResponse)
resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs)

13
services/runservice/client/client.go
View File

@ -304,6 +304,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool,
return c.getResponse(ctx, "GET", "/logs", q, -1, nil, nil)
}
func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
q := url.Values{}
q.Add("runid", runID)
q.Add("taskid", taskID)
if setup {
q.Add("setup", "")
} else {
q.Add("step", strconv.Itoa(step))
}
return c.getResponse(ctx, "DELETE", "/logs", q, -1, nil, nil)
}
func (c *Client) GetRunEvents(ctx context.Context, startRunEventID string) (*http.Response, error) {
q := url.Values{}
q.Add("startruneventid", startRunEventID)

177
tests/setup_test.go
View File

@ -23,6 +23,7 @@ import (
"os/exec"
"path"
"path/filepath"
"strings"
"testing"
"time"
@ -1104,3 +1105,179 @@ func TestDirectRunVariables(t *testing.T) {
})
}
}
func TestDirectRunLogs(t *testing.T) {
config := `
{
runs: [
{
name: 'run01',
tasks: [
{
name: 'task01',
runtime: {
containers: [
{
image: 'alpine/git',
},
],
},
steps: [
{ type: 'clone' },
{ type: 'run', command: 'echo STEPLOG' },
],
},
],
},
],
}
`
tests := []struct {
name string
setup bool
step int
delete bool
err error
}{
{
name: "test get log step 1",
step: 1,
},
{
name: "test get log setup",
setup: true,
},
{
name: "test get log with unexisting step",
step: 99,
err: errors.Errorf("log doesn't exist"),
},
{
name: "test delete log step 1",
step: 1,
delete: true,
},
{
name: "test delete log setup",
setup: true,
delete: true,
},
{
name: "test delete log with unexisting step",
step: 99,
delete: true,
err: errors.Errorf("log doesn't exist"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir, err := ioutil.TempDir("", "agola")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
tetcd, tgitea, c := setup(ctx, t, dir)
defer shutdownGitea(tgitea)
defer shutdownEtcd(tetcd)
gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("created agola user: %s", user.UserName)
token := createAgolaUserToken(ctx, t, c)
// From now use the user token
gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
directRun(t, dir, config, c.Gateway.APIExposedURL, token)
_ = testutil.Wait(30*time.Second, func() (bool, error) {
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
if err != nil {
return false, nil
}
if len(runs) != 1 {
return false, nil
}
run := runs[0]
if run.Phase != rstypes.RunPhaseFinished {
return false, nil
}
return true, nil
})
runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("runs: %s", util.Dump(runs))
if len(runs) != 1 {
t.Fatalf("expected 1 run got: %d", len(runs))
}
run, _, err := gwClient.GetRun(ctx, runs[0].ID)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if run.Phase != rstypes.RunPhaseFinished {
t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
}
if run.Result != rstypes.RunResultSuccess {
t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
}
var task *gwapitypes.RunResponseTask
for _, t := range run.Tasks {
if t.Name == "task01" {
task = t
break
}
}
_ = testutil.Wait(30*time.Second, func() (bool, error) {
t, _, err := gwClient.GetRunTask(ctx, runs[0].ID, task.ID)
if err != nil {
return false, nil
}
if !t.Steps[tt.step].LogArchived {
return false, nil
}
return true, nil
})
if tt.delete {
_, err = gwClient.DeleteLogs(ctx, run.ID, task.ID, tt.setup, tt.step)
} else {
_, err = gwClient.GetLogs(ctx, run.ID, task.ID, tt.setup, tt.step)
}
if err != nil {
if tt.err == nil {
t.Fatalf("got error: %v, expected no error", err)
}
if !strings.HasPrefix(err.Error(), tt.err.Error()) {
t.Fatalf("got error: %v, want error: %v", err, tt.err)
}
} else {
if tt.err != nil {
t.Fatalf("got nil error, want error: %v", tt.err)
}
}
})
}
}