gateway/runservice: add api to delete step logs

internal/services/gateway/action/run.go

@@ -145,6 +145,34 @@ func (h *ActionHandler) GetLogs(ctx context.Context, req *GetLogsRequest) (*http
 	return resp, nil
 }
 
+type DeleteLogsRequest struct {
+	RunID  string
+	TaskID string
+	Setup  bool
+	Step   int
+}
+
+func (h *ActionHandler) DeleteLogs(ctx context.Context, req *DeleteLogsRequest) error {
+	runResp, resp, err := h.runserviceClient.GetRun(ctx, req.RunID, nil)
+	if err != nil {
+		return ErrFromRemote(resp, err)
+	}
+	canDoRunActions, err := h.CanDoRunActions(ctx, runResp.RunConfig.Group)
+	if err != nil {
+		return errors.Errorf("failed to determine permissions: %w", err)
+	}
+	if !canDoRunActions {
+		return util.NewErrForbidden(errors.Errorf("user not authorized"))
+	}
+
+	resp, err = h.runserviceClient.DeleteLogs(ctx, req.RunID, req.TaskID, req.Setup, req.Step)
+	if err != nil {
+		return ErrFromRemote(resp, err)
+	}
+
+	return nil
+}
+
 type RunActionType string
 
 const (

internal/services/gateway/api/run.go

@@ -502,3 +502,63 @@ func sendLogs(w io.Writer, r io.Reader) error {
 		}
 	}
 }
+
+type LogsDeleteHandler struct {
+	log *zap.SugaredLogger
+	ah  *action.ActionHandler
+}
+
+func NewLogsDeleteHandler(logger *zap.Logger, ah *action.ActionHandler) *LogsDeleteHandler {
+	return &LogsDeleteHandler{log: logger.Sugar(), ah: ah}
+}
+
+func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	q := r.URL.Query()
+
+	runID := q.Get("runID")
+	if runID == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("empty run id")))
+		return
+	}
+	taskID := q.Get("taskID")
+	if taskID == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("empty task id")))
+		return
+	}
+
+	_, setup := q["setup"]
+	stepStr := q.Get("step")
+	if !setup && stepStr == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("no setup or step number provided")))
+		return
+	}
+	if setup && stepStr != "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("both setup and step number provided")))
+		return
+	}
+
+	var step int
+	if stepStr != "" {
+		var err error
+		step, err = strconv.Atoi(stepStr)
+		if err != nil {
+			httpError(w, util.NewErrBadRequest(errors.Errorf("cannot parse step number: %w", err)))
+			return
+		}
+	}
+
+	areq := &action.DeleteLogsRequest{
+		RunID:  runID,
+		TaskID: taskID,
+		Setup:  setup,
+		Step:   step,
+	}
+
+	err := h.ah.DeleteLogs(ctx, areq)
+	if httpError(w, err) {
+		h.log.Errorf("err: %+v", err)
+		return
+	}
+}

internal/services/gateway/gateway.go

@@ -210,6 +210,7 @@ func (g *Gateway) Run(ctx context.Context) error {
 	runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, g.ah)
 
 	logsHandler := api.NewLogsHandler(logger, g.ah)
+	logsDeleteHandler := api.NewLogsDeleteHandler(logger, g.ah)
 
 	userRemoteReposHandler := api.NewUserRemoteReposHandler(logger, g.ah, g.configstoreClient)
 
@@ -235,6 +236,7 @@ func (g *Gateway) Run(ctx context.Context) error {
 	router.PathPrefix("/api/v1alpha").Handler(apirouter)
 
 	apirouter.Handle("/logs", authOptionalHandler(logsHandler)).Methods("GET")
+	apirouter.Handle("/logs", authForcedHandler(logsDeleteHandler)).Methods("DELETE")
 
 	//apirouter.Handle("/projectgroups", authForcedHandler(projectsHandler)).Methods("GET")
 	apirouter.Handle("/projectgroups/{projectgroupref}", authForcedHandler(projectGroupHandler)).Methods("GET")

internal/services/runservice/api/api.go

@@ -323,6 +323,106 @@ func sendLogs(w http.ResponseWriter, r io.Reader) error {
 	}
 }
 
+type LogsDeleteHandler struct {
+	log *zap.SugaredLogger
+	e   *etcd.Store
+	ost *objectstorage.ObjStorage
+	dm  *datamanager.DataManager
+}
+
+func NewLogsDeleteHandler(logger *zap.Logger, e *etcd.Store, ost *objectstorage.ObjStorage, dm *datamanager.DataManager) *LogsDeleteHandler {
+	return &LogsDeleteHandler{
+		log: logger.Sugar(),
+		e:   e,
+		ost: ost,
+		dm:  dm,
+	}
+}
+
+func (h *LogsDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	q := r.URL.Query()
+
+	runID := q.Get("runid")
+	if runID == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("runid is empty")))
+		return
+	}
+	taskID := q.Get("taskid")
+	if taskID == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("taskid is empty")))
+		return
+	}
+
+	_, setup := q["setup"]
+	stepStr := q.Get("step")
+	if !setup && stepStr == "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("setup is false and step is empty")))
+		return
+	}
+	if setup && stepStr != "" {
+		httpError(w, util.NewErrBadRequest(errors.Errorf("setup is true and step is %s", stepStr)))
+		return
+	}
+
+	var step int
+	if stepStr != "" {
+		var err error
+		step, err = strconv.Atoi(stepStr)
+		if err != nil {
+			httpError(w, util.NewErrBadRequest(errors.Errorf("step %s is not a valid number", stepStr)))
+			return
+		}
+	}
+
+	if err := h.deleteTaskLogs(ctx, runID, taskID, setup, step, w); err != nil {
+		h.log.Errorf("err: %+v", err)
+		switch {
+		case util.IsNotExist(err):
+			httpError(w, util.NewErrNotExist(errors.Errorf("log doesn't exist: %w", err)))
+		default:
+			httpError(w, err)
+		}
+	}
+}
+
+func (h *LogsDeleteHandler) deleteTaskLogs(ctx context.Context, runID, taskID string, setup bool, step int, w http.ResponseWriter) error {
+	r, err := store.GetRunEtcdOrOST(ctx, h.e, h.dm, runID)
+	if err != nil {
+		return err
+	}
+	if r == nil {
+		return util.NewErrNotExist(errors.Errorf("no such run with id: %s", runID))
+	}
+
+	task, ok := r.Tasks[taskID]
+	if !ok {
+		return util.NewErrNotExist(errors.Errorf("no such task with ID %s in run %s", taskID, runID))
+	}
+	if len(task.Steps) <= step {
+		return util.NewErrNotExist(errors.Errorf("no such step for task %s in run %s", taskID, runID))
+	}
+
+	if task.Steps[step].LogPhase == types.RunTaskFetchPhaseFinished {
+		var logPath string
+		if setup {
+			logPath = store.OSTRunTaskSetupLogPath(task.ID)
+		} else {
+			logPath = store.OSTRunTaskStepLogPath(task.ID, step)
+		}
+		err := h.ost.DeleteObject(logPath)
+		if err != nil {
+			if objectstorage.IsNotExist(err) {
+				return util.NewErrNotExist(err)
+			}
+			return err
+		}
+		return nil
+	}
+	return util.NewErrBadRequest(errors.Errorf("Log for task %s in run %s is not yet archived", taskID, runID))
+}
+
 type ChangeGroupsUpdateTokensHandler struct {
 	log    *zap.SugaredLogger
 	readDB *readdb.ReadDB

internal/services/runservice/runservice.go

@@ -221,6 +221,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand
 	executorDeleteHandler := api.NewExecutorDeleteHandler(logger, s.ah)
 
 	logsHandler := api.NewLogsHandler(logger, s.e, s.ost, s.dm)
+	logsDeleteHandler := api.NewLogsDeleteHandler(logger, s.e, s.ost, s.dm)
 
 	runHandler := api.NewRunHandler(logger, s.e, s.dm, s.readDB)
 	runTaskActionsHandler := api.NewRunTaskActionsHandler(logger, s.ah)
@@ -248,6 +249,7 @@ func (s *Runservice) setupDefaultRouter(etCh chan *types.ExecutorTask) http.Hand
 	apirouter.Handle("/executor/caches/{key}", cacheCreateHandler).Methods("POST")
 
 	apirouter.Handle("/logs", logsHandler).Methods("GET")
+	apirouter.Handle("/logs", logsDeleteHandler).Methods("DELETE")
 
 	apirouter.Handle("/runs/events", runEventsHandler).Methods("GET")
 	apirouter.Handle("/runs/{runid}", runHandler).Methods("GET")

services/gateway/client/client.go

@@ -455,6 +455,12 @@ func (c *Client) GetRun(ctx context.Context, runID string) (*gwapitypes.RunRespo
 	return run, resp, err
 }
 
+func (c *Client) GetRunTask(ctx context.Context, runID, taskID string) (*gwapitypes.RunTaskResponse, *http.Response, error) {
+	task := new(gwapitypes.RunTaskResponse)
+	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/run/%s/task/%s", runID, taskID), nil, jsonContent, nil, task)
+	return task, resp, err
+}
+
 func (c *Client) GetRuns(ctx context.Context, phaseFilter, resultFilter, groups, runGroups []string, start string, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) {
 	q := url.Values{}
 	for _, phase := range phaseFilter {
@@ -497,6 +503,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool,
 	return c.getResponse(ctx, "GET", "/logs", q, nil, nil)
 }
 
+func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
+	q := url.Values{}
+	q.Add("runID", runID)
+	q.Add("taskID", taskID)
+	if setup {
+		q.Add("setup", "")
+	} else {
+		q.Add("step", strconv.Itoa(step))
+	}
+
+	return c.getResponse(ctx, "DELETE", "/logs", q, nil, nil)
+}
+
 func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes.RemoteSourceResponse, *http.Response, error) {
 	rs := new(gwapitypes.RemoteSourceResponse)
 	resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/remotesources/%s", rsRef), nil, jsonContent, nil, rs)

services/runservice/client/client.go

@@ -304,6 +304,19 @@ func (c *Client) GetLogs(ctx context.Context, runID, taskID string, setup bool,
 	return c.getResponse(ctx, "GET", "/logs", q, -1, nil, nil)
 }
 
+func (c *Client) DeleteLogs(ctx context.Context, runID, taskID string, setup bool, step int) (*http.Response, error) {
+	q := url.Values{}
+	q.Add("runid", runID)
+	q.Add("taskid", taskID)
+	if setup {
+		q.Add("setup", "")
+	} else {
+		q.Add("step", strconv.Itoa(step))
+	}
+
+	return c.getResponse(ctx, "DELETE", "/logs", q, -1, nil, nil)
+}
+
 func (c *Client) GetRunEvents(ctx context.Context, startRunEventID string) (*http.Response, error) {
 	q := url.Values{}
 	q.Add("startruneventid", startRunEventID)

tests/setup_test.go

@@ -23,6 +23,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -1104,3 +1105,179 @@ func TestDirectRunVariables(t *testing.T) {
 		})
 	}
 }
+
+func TestDirectRunLogs(t *testing.T) {
+	config := `
+      {
+        runs: [
+          {
+            name: 'run01',
+            tasks: [
+              {
+                name: 'task01',
+                runtime: {
+                  containers: [
+                    {
+                      image: 'alpine/git',
+                    },
+                  ],
+                },
+                steps: [
+                  { type: 'clone' },
+                  { type: 'run', command: 'echo STEPLOG' },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `
+
+	tests := []struct {
+		name   string
+		setup  bool
+		step   int
+		delete bool
+		err    error
+	}{
+		{
+			name: "test get log step 1",
+			step: 1,
+		},
+		{
+			name:  "test get log setup",
+			setup: true,
+		},
+		{
+			name: "test get log with unexisting step",
+			step: 99,
+			err:  errors.Errorf("log doesn't exist"),
+		},
+		{
+			name:   "test delete log step 1",
+			step:   1,
+			delete: true,
+		},
+		{
+			name:   "test delete log setup",
+			setup:  true,
+			delete: true,
+		},
+		{
+			name:   "test delete log with unexisting step",
+			step:   99,
+			delete: true,
+			err:    errors.Errorf("log doesn't exist"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir, err := ioutil.TempDir("", "agola")
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			defer os.RemoveAll(dir)
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			tetcd, tgitea, c := setup(ctx, t, dir)
+			defer shutdownGitea(tgitea)
+			defer shutdownEtcd(tetcd)
+
+			gwClient := gwclient.NewClient(c.Gateway.APIExposedURL, "admintoken")
+			user, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01})
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			t.Logf("created agola user: %s", user.UserName)
+
+			token := createAgolaUserToken(ctx, t, c)
+
+			// From now use the user token
+			gwClient = gwclient.NewClient(c.Gateway.APIExposedURL, token)
+
+			directRun(t, dir, config, c.Gateway.APIExposedURL, token)
+
+			_ = testutil.Wait(30*time.Second, func() (bool, error) {
+				runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
+				if err != nil {
+					return false, nil
+				}
+
+				if len(runs) != 1 {
+					return false, nil
+				}
+
+				run := runs[0]
+				if run.Phase != rstypes.RunPhaseFinished {
+					return false, nil
+				}
+
+				return true, nil
+			})
+
+			runs, _, err := gwClient.GetRuns(ctx, nil, nil, []string{path.Join("/user", user.ID)}, nil, "", 0, false)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			t.Logf("runs: %s", util.Dump(runs))
+
+			if len(runs) != 1 {
+				t.Fatalf("expected 1 run got: %d", len(runs))
+			}
+
+			run, _, err := gwClient.GetRun(ctx, runs[0].ID)
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+
+			if run.Phase != rstypes.RunPhaseFinished {
+				t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase)
+			}
+			if run.Result != rstypes.RunResultSuccess {
+				t.Fatalf("expected run result %q, got %q", rstypes.RunResultSuccess, run.Result)
+			}
+
+			var task *gwapitypes.RunResponseTask
+			for _, t := range run.Tasks {
+				if t.Name == "task01" {
+					task = t
+					break
+				}
+			}
+
+			_ = testutil.Wait(30*time.Second, func() (bool, error) {
+				t, _, err := gwClient.GetRunTask(ctx, runs[0].ID, task.ID)
+				if err != nil {
+					return false, nil
+				}
+				if !t.Steps[tt.step].LogArchived {
+					return false, nil
+				}
+				return true, nil
+			})
+
+			if tt.delete {
+				_, err = gwClient.DeleteLogs(ctx, run.ID, task.ID, tt.setup, tt.step)
+			} else {
+				_, err = gwClient.GetLogs(ctx, run.ID, task.ID, tt.setup, tt.step)
+			}
+
+			if err != nil {
+				if tt.err == nil {
+					t.Fatalf("got error: %v, expected no error", err)
+				}
+				if !strings.HasPrefix(err.Error(), tt.err.Error()) {
+					t.Fatalf("got error: %v, want error: %v", err, tt.err)
+				}
+			} else {
+				if tt.err != nil {
+					t.Fatalf("got nil error, want error: %v", tt.err)
+				}
+			}
+		})
+	}
+}