fix: remove the job network even when container removal overruns (#1318)

Job teardown ran every step on one shared minute. When removing the job container took longer, the network removal ran on an expired context and the network stayed behind until the orphan reclaim, which only takes networks older than `workdir_cleanup_age`, 24h by default. All that time it held a subnet of the daemon's address pool.

The job and service containers are now removed in parallel, and the docker proxy sweep, the network and the volumes get a fresh budget after them. The idle reclaim takes unused networks and volumes once they are 10 minutes old, and network removal treats a network that is already gone as removed.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1318
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>

internal/app/run/runner.go

@@ -157,6 +157,8 @@ var removeOrphanJobVolumes = container.RemoveOrphanJobVolumes
 
 var removeOrphanKubernetesResources = container.RemoveOrphanKubernetesResources
 
+const orphanDockerResourceAge = 10 * time.Minute // spares the network and volumes of a job still starting its first container
+
 // OnIdle performs lightweight maintenance during polling idle windows.
 // It runs synchronously on the poller goroutine; shouldRunIdleCleanup
 // throttles invocations to runner.idle_cleanup_interval so the impact on
@@ -189,10 +191,12 @@ func (r *Runner) OnIdle(ctx context.Context) {
 }
 
 func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
+	ctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
 	if r.uuid == "" || (!r.requiresDocker() && !dockerReachable(ctx)) {
 		return
 	}
-	cutoff := r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)
+	cutoff := r.now().Add(-orphanDockerResourceAge)
 	if err := removeOrphanNetworks(ctx, r.uuid, r.isolatedCacheContainer(), cutoff); err != nil {
 		logging.Warnf("failed to clean up networks left behind by earlier jobs: %v", err)
 	}

internal/app/run/runner_idle_cleanup_test.go

@@ -317,7 +317,9 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	var sweptVolumes []string
 	var sweptCutoff time.Time
 	origRemoveOrphanNetworks := removeOrphanNetworks
-	removeOrphanNetworks = func(_ context.Context, runnerUUID, detach string, createdBefore time.Time) error {
+	removeOrphanNetworks = func(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
+		_, hasDeadline := ctx.Deadline()
+		assert.True(t, hasDeadline)
 		swept = append(swept, runnerUUID)
 		detached = append(detached, detach)
 		sweptCutoff = createdBefore
@@ -327,7 +329,7 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	origRemoveOrphanJobVolumes := removeOrphanJobVolumes
 	removeOrphanJobVolumes = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
 		sweptVolumes = append(sweptVolumes, runnerUUID)
-		assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
+		assert.Equal(t, now.Add(-orphanDockerResourceAge), createdBefore)
 		return nil
 	}
 	t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
@@ -350,7 +352,7 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	assert.Equal(t, []string{"runner-1"}, swept)
 	assert.Equal(t, swept, sweptVolumes)
 	// a network of a job starting during the pass is younger than this and so out of scope
-	assert.Equal(t, now.Add(-24*time.Hour), sweptCutoff)
+	assert.Equal(t, now.Add(-orphanDockerResourceAge), sweptCutoff)
 
 	// a host-only runner has no daemon to sweep
 	origDockerReachable := dockerReachable

internal/pkg/config/config.example.yaml

@@ -61,9 +61,9 @@ runner:
   #workdir_cleanup_age: 24h
   # Cadence for the idle cleanup pass. Besides the directories above, on runners that use
   # docker it removes the per-job networks and volumes of jobs this runner did not live to
-  # tear down, which would otherwise hold a subnet of the daemon address pool and the job's
-  # workspace contents until the host is rebuilt. On runners with kubernetes labels it
-  # removes such jobs' pods and Secrets.
+  # tear down once unused and 10 minutes old, which would otherwise hold a subnet of the
+  # daemon address pool and the job's workspace contents until the host is rebuilt. On
+  # runners with kubernetes labels it removes such jobs' pods and Secrets.
   #idle_cleanup_interval: 10m
   # The base interval for periodic log flush to the Gitea instance.
   # Logs may be sent earlier if the buffer reaches log_report_batch_size

internal/pkg/config/config.go

@@ -71,7 +71,7 @@ type Runner struct {
 	FetchTimeout          time.Duration     `yaml:"fetch_timeout"`            // FetchTimeout specifies the timeout duration for fetching resources.
 	FetchInterval         time.Duration     `yaml:"fetch_interval"`           // FetchInterval specifies the interval duration for fetching resources.
 	FetchIntervalMax      time.Duration     `yaml:"fetch_interval_max"`       // FetchIntervalMax specifies the maximum backoff interval when idle.
-	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker and Kubernetes job resources older than this duration during idle cleanup.
+	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned Kubernetes job resources older than this duration during idle cleanup.
 	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories, orphaned docker networks and volumes, orphaned job pods) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
 	LogReportInterval     time.Duration     `yaml:"log_report_interval"`      // LogReportInterval specifies the base interval for periodic log flush.
 	LogReportMaxLatency   time.Duration     `yaml:"log_report_max_latency"`   // LogReportMaxLatency specifies the max time a log row can wait before being sent.