runservice: delete executor task early

currently we are deleting the executor tasks only when all the run tasks
log/archives were fetched. But it'll better to remove a single executor task
when the task fetching is finished.

This could also fix possible issues on k8s since we are scheduling tasks but the
k8s scheduler may not schedule them if there aren't enough resources causing a
scheduling deadlock since we won't remove finished pods because their related
tasks are not removed and k8s cannot start new pods since it has no resources.

internal/services/runservice/scheduler.go

@@ -1177,6 +1177,15 @@ func (s *Runservice) fetcher(ctx context.Context) error {
 
 				s.fetchTaskLogs(ctx, r.ID, rt)
 				s.fetchTaskArchives(ctx, r.ID, rt)
+
+				// if the fetching is finished we can remove the executor tasks. We cannot
+				// remove it before since it contains the reference to the executor where we
+				// should fetch the data
+				if rt.LogsFetchFinished() && rt.ArchivesFetchFinished() {
+					if err := store.DeleteExecutorTask(ctx, s.e, rt.ID); err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -1299,17 +1308,6 @@ func (s *Runservice) finishedRunArchiver(ctx context.Context, r *types.Run) erro
 	}
 	log.Infof("run %q archiving completed", r.ID)
 
-	// if the fetching is finished we can remove the executor tasks. We cannot
-	// remove it before since it contains the reference to the executor where we
-	// should fetch the data
-
-	for _, rt := range r.Tasks {
-		log.Infof("deleting executor task %s", rt.ID)
-		if err := store.DeleteExecutorTask(ctx, s.e, rt.ID); err != nil {
-			return err
-		}
-	}
-
 	r.Archived = true
 	if _, err := store.AtomicPutRun(ctx, s.e, r, nil, nil); err != nil {
 		return err