From 3e47bc601a3ae597797a99c5d90c8d5cd520e644 Mon Sep 17 00:00:00 2001 From: Carlo Mandelli Date: Sat, 9 Nov 2019 11:18:00 +0100 Subject: [PATCH] gateway/runservice: add api to delete step logs --- internal/services/gateway/action/run.go | 28 ++++ internal/services/gateway/api/run.go | 60 +++++++ internal/services/gateway/gateway.go | 2 + internal/services/runservice/api/api.go | 100 ++++++++++++ internal/services/runservice/runservice.go | 2 + services/gateway/client/client.go | 19 +++ services/runservice/client/client.go | 13 ++ tests/setup_test.go | 177 +++++++++++++++++++++ 8 files changed, 401 insertions(+) diff --git a/internal/services/gateway/action/run.go b/internal/services/gateway/action/run.go index b0adc21..ec7d662 100644 --- a/internal/services/gateway/action/run.go +++ b/internal/services/gateway/action/run.go @@ -145,6 +145,34 @@ func (h *ActionHandler) GetLogs(ctx context.Context, req *GetLogsRequest) (*http return resp, nil } +type DeleteLogsRequest struct { + RunID string + TaskID string + Setup bool + Step int +} + +func (h *ActionHandler) DeleteLogs(ctx context.Context, req *DeleteLogsRequest) error { + runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil) + if err != nil { + return ErrFromRemote(resp, err) + } + canDoRunActions, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group) + if err != nil { + return errors.Errorf("failed to determine permissions: %w", err) + } + if !canDoRunActions { + return util.NewErrForbidden(errors.Errorf("user not authorized")) + } + + resp, err = h.runserviceClient.DeleteLogs(ctx, req.RunID, req.TaskID, req.Setup, req.Step) + if err != nil { + return ErrFromRemote(resp, err) + } + + return nil +} + type RunActionType string const ( diff --git a/internal/services/gateway/api/run.go b/internal/services/gateway/api/run.go index d735b17..8c19f43 100644 --- a/internal/services/gateway/api/run.go +++ b/internal/services/gateway/api/run.go @@ -502,3 +502,63 @@ func sendLogs(w io.Writer, r io.Reader) error { } } } + +type LogsDeleteHandler struct { + log *zap.SugaredLogger + ah *action.ActionHandler +} + +func NewLogsDeleteHandler(logger *zap.Logger, ah *action.ActionHandler) *LogsDeleteHandler { + return &LogsDeleteHandler{log: logger.Sugar(), ah: ah} +} + +func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + q := r.URL.Query() + + runID := q.Get("runID") + if runID == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("empty run id"))) + return + } + taskID := q.Get("taskID") + if taskID == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("empty task id"))) + return + } + + _, setup := q["setup"] + stepStr := q.Get("step") + if !setup && stepStr == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("no setup or step number provided"))) + return + } + if setup && stepStr != "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("both setup and step number provided"))) + return + } + + var step int + if stepStr != "" { + var err error + step, err = strconv.Atoi(stepStr) + if err != nil { + httpError(w, util.NewErrBadRequest(errors.Errorf("cannot parse step number: %w", err))) + return + } + } + + areq := &action.DeleteLogsRequest{ + RunID: runID, + TaskID: taskID, + Setup: setup, + Step: step, + } + + err := h.ah.DeleteLogs(ctx, areq) + if httpError(w, err) { + h.log.Errorf("err: %+v", err) + return + } +} diff --git a/internal/services/gateway/gateway.go b/internal/services/gateway/gateway.go index 95a9e76..9632479 100644 --- a/internal/services/gateway/gateway.go +++ b/internal/services/gateway/gateway.go @@ -210,6 +210,7 @@ func (g *Gateway) Run(ctx context.Context) error { runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, g.ah) logsHandler := api.NewLogsHandler(logger, g.ah) + logsDeleteHandler := api.NewLogsDeleteHandler(logger, g.ah) userRemoteReposHandler := api.NewUserRemoteReposHandler(logger, g.ah, g.configstoreClient) @@ -235,6 +236,7 @@ func (g *Gateway) Run(ctx context.Context) error { router.PathPrefix("/api/v1alpha").Handler(apirouter) apirouter.Handle("/logs", authOptionalHandler(logsHandler)).Methods("GET") + apirouter.Handle("/logs", authForcedHandler(logsDeleteHandler)).Methods("DELETE") //apirouter.Handle("/projectgroups", authForcedHandler(projectsHandler)).Methods("GET") apirouter.Handle("/projectgroups/{projectgroupref}", authForcedHandler(projectGroupHandler)).Methods("GET") diff --git a/internal/services/runservice/api/api.go b/internal/services/runservice/api/api.go index f74c444..08a2630 100644 --- a/internal/services/runservice/api/api.go +++ b/internal/services/runservice/api/api.go @@ -323,6 +323,106 @@ func sendLogs(w http.ResponseWriter, r io.Reader) error { } } +type LogsDeleteHandler struct { + log *zap.SugaredLogger + e *etcd.Store + ost *objectstorage.ObjStorage + dm *datamanager.DataManager +} + +func NewLogsDeleteHandler(logger *zap.Logger, e *etcd.Store, ost *objectstorage.ObjStorage, dm *datamanager.DataManager) *LogsDeleteHandler { + return &LogsDeleteHandler{ + log: logger.Sugar(), + e: e, + ost: ost, + dm: dm, + } +} + +func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + q := r.URL.Query() + + runID := q.Get("runid") + if runID == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("runid is empty"))) + return + } + taskID := q.Get("taskid") + if taskID == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("taskid is empty"))) + return + } + + _, setup := q["setup"] + stepStr := q.Get("step") + if !setup && stepStr == "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("setup is false and step is empty"))) + return + } + if setup && stepStr != "" { + httpError(w, util.NewErrBadRequest(errors.Errorf("setup is true and step is %s", stepStr))) + return + } + + var step int + if stepStr != "" { + var err error + step, err = strconv.Atoi(stepStr) + if err != nil { + httpError(w, util.NewErrBadRequest(errors.Errorf("step %s is not a valid number", stepStr))) + return + } + } + + if err := h.deleteTaskLogs(ctx, runID, taskID, setup, step, w); err != nil { + h.log.Errorf("err: %+v", err) + switch { + case util.IsNotExist(err): + httpError(w, util.NewErrNotExist(errors.Errorf("log doesn't exist: %w", err))) + default: + httpError(w, err) + } + } +} + +func (h *LogsDeleteHandler) deleteTaskLogs(ctx context.Context, runID, taskID string, setup bool, step int, w http.ResponseWriter) error { + r, err := store.GetRunEtcdOrOST(ctx, h.e, h.dm, runID) + if err != nil { + return err + } + if r == nil { + return util.NewErrNotExist(errors.Errorf("no such run with id: %s", runID)) + } + + task, ok := r.Tasks[taskID] + if !ok { + return util.NewErrNotExist(errors.Errorf("no such task with ID %s in run %s", taskID, runID)) + } + if len(task.Steps) <= step { + return util.NewErrNotExist(errors.Errorf("no such step for task %s in run %s", taskID, runID)) + } + + if task.Steps[step].LogPhase == types.RunTaskFetchPhaseFinished { + var logPath string + if setup { + logPath = store.OSTRunTaskSetupLogPath(task.ID) + } else { + logPath = store.OSTRunTaskStepLogPath(task.ID, step) + } + err := h.ost.DeleteObject(logPath) + if err != nil { + if objectstorage.IsNotExist(err) { + return util.NewErrNotExist(err) + } + return err + } + return nil + } + return util.NewErrBadRequest(errors.Errorf("Log for task %s in run %s is not yet archived", taskID, runID)) +} + type ChangeGroupsUpdateTokensHandler struct { log *zap.SugaredLogger readDB *readdb.ReadDB diff --git a/internal/services/runservice/runservice.go b/internal/services/runservice/runservice.go index 1f229a5..bfdcbf1 100644 --- a/internal/services/runservice/runservice.go +++ b/internal/services/runservice/runservice.go @@ -221,6 +221,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand executorDeleteHandler := api.NewExecutorDeleteHandler(logger, s.ah) logsHandler := api.NewLogsHandler(logger, s.e, s.ost, s.dm) + logsDeleteHandler := api.NewLogsDeleteHandler(logger, s.e, s.ost, s.dm) runHandler := api.NewRunHandler(logger, s.e, s.dm, s.readDB) runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, s.ah) @@ -248,6 +249,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand apirouter.Handle("/executor/caches/{key}", cacheCreateHandler).Methods("POST") apirouter.Handle("/logs", logsHandler).Methods("GET") + apirouter.Handle("/logs", logsDeleteHandler).Methods("DELETE") apirouter.Handle("/runs/events", runEventsHandler).Methods("GET") apirouter.Handle("/runs/{runid}", runHandler).Methods("GET") diff --git a/services/gateway/client/client.go b/services/gateway/client/client.go index 0ff4c64..a3a4214 100644 --- a/services/gateway/client/client.go +++ b/services/gateway/client/client.go @@ -455,6 +455,12 @@ func (c *Client) GetRun(ctx context.Context, runID string) (*gwapitypes.RunRespo return run, resp, err } +func (c *Client) GetRunTask(ctx context.Context, runID, taskID string) (*gwapitypes.RunTaskResponse, *http.Response, error) { + task := new(gwapitypes.RunTaskResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/run/%s/task/%s", runID, taskID), nil, jsonContent, nil, task) + return task, resp, err +} + func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups, runGroups []string, start string, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) { q := url.Values{} for _, phase := range phaseFilter { @@ -497,6 +503,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool, return c.getResponse(ctx, "GET", "/logs", q, nil, nil) } +func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) { + q := url.Values{} + q.Add("runID", runID) + q.Add("taskID", taskID) + if setup { + q.Add("setup", "") + } else { + q.Add("step", strconv.Itoa(step)) + } + + return c.getResponse(ctx, "DELETE", "/logs", q, nil, nil) +} + func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) { rs := new(gwapitypes.RemoteSourceResponse) resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs) diff --git a/services/runservice/client/client.go b/services/runservice/client/client.go index 0e0331c..355c3d7 100644 --- a/services/runservice/client/client.go +++ b/services/runservice/client/client.go @@ -304,6 +304,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool, return c.getResponse(ctx, "GET", "/logs", q, -1, nil, nil) } +func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) { + q := url.Values{} + q.Add("runid", runID) + q.Add("taskid", taskID) + if setup { + q.Add("setup", "") + } else { + q.Add("step", strconv.Itoa(step)) + } + + return c.getResponse(ctx, "DELETE", "/logs", q, -1, nil, nil) +} + func (c *Client) GetRunEvents(ctx context.Context, startRunEventID string) (*http.Response, error) { q := url.Values{} q.Add("startruneventid", startRunEventID) diff --git a/tests/setup_test.go b/tests/setup_test.go index 0a8fd30..a471e79 100644 --- a/tests/setup_test.go +++ b/tests/setup_test.go @@ -23,6 +23,7 @@ import ( "os/exec" "path" "path/filepath" + "strings" "testing" "time" @@ -1104,3 +1105,179 @@ func TestDirectRunVariables(t *testing.T) { }) } } + +func TestDirectRunLogs(t *testing.T) { + config := ` + { + runs: [ + { + name: 'run01', + tasks: [ + { + name: 'task01', + runtime: { + containers: [ + { + image: 'alpine/git', + }, + ], + }, + steps: [ + { type: 'clone' }, + { type: 'run', command: 'echo STEPLOG' }, + ], + }, + ], + }, + ], + } + ` + + tests := []struct { + name string + setup bool + step int + delete bool + err error + }{ + { + name: "test get log step 1", + step: 1, + }, + { + name: "test get log setup", + setup: true, + }, + { + name: "test get log with unexisting step", + step: 99, + err: errors.Errorf("log doesn't exist"), + }, + { + name: "test delete log step 1", + step: 1, + delete: true, + }, + { + name: "test delete log setup", + setup: true, + delete: true, + }, + { + name: "test delete log with unexisting step", + step: 99, + delete: true, + err: errors.Errorf("log doesn't exist"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := ioutil.TempDir("", "agola") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tetcd, tgitea, c := setup(ctx, t, dir) + defer shutdownGitea(tgitea) + defer shutdownEtcd(tetcd) + + gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken") + user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + t.Logf("created agola user: %s", user.UserName) + + token := createAgolaUserToken(ctx, t, c) + + // From now use the user token + gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token) + + directRun(t, dir, config, c.Gateway.APIExposedURL, token) + + _ = testutil.Wait(30*time.Second, func() (bool, error) { + runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false) + if err != nil { + return false, nil + } + + if len(runs) != 1 { + return false, nil + } + + run := runs[0] + if run.Phase != rstypes.RunPhaseFinished { + return false, nil + } + + return true, nil + }) + + runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + t.Logf("runs: %s", util.Dump(runs)) + + if len(runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs)) + } + + run, _, err := gwClient.GetRun(ctx, runs[0].ID) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + if run.Phase != rstypes.RunPhaseFinished { + t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) + } + if run.Result != rstypes.RunResultSuccess { + t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result) + } + + var task *gwapitypes.RunResponseTask + for _, t := range run.Tasks { + if t.Name == "task01" { + task = t + break + } + } + + _ = testutil.Wait(30*time.Second, func() (bool, error) { + t, _, err := gwClient.GetRunTask(ctx, runs[0].ID, task.ID) + if err != nil { + return false, nil + } + if !t.Steps[tt.step].LogArchived { + return false, nil + } + return true, nil + }) + + if tt.delete { + _, err = gwClient.DeleteLogs(ctx, run.ID, task.ID, tt.setup, tt.step) + } else { + _, err = gwClient.GetLogs(ctx, run.ID, task.ID, tt.setup, tt.step) + } + + if err != nil { + if tt.err == nil { + t.Fatalf("got error: %v, expected no error", err) + } + if !strings.HasPrefix(err.Error(), tt.err.Error()) { + t.Fatalf("got error: %v, want error: %v", err, tt.err) + } + } else { + if tt.err != nil { + t.Fatalf("got nil error, want error: %v", tt.err) + } + } + }) + } +}