fix: stop the docker proxy probe from leaking its container (#1357)

When the probe's deadline expired during `ContainerCreate`, dockerd still created the container but the probe never got its ID, leaving it in `Created` with its image's anonymous volumes. A slow daemon leaked one per job. The probe now runs detached with its own timeout, so the create is never cancelled and the container is always removed. Callers still fall back after their own deadline, and the runner waits for pending probes before exiting. Fixes: https://gitea.com/gitea/runner/issues/1353 Reviewed-on: https://gitea.com/gitea/runner/pulls/1357 Reviewed-by: Zettat123 <[email protected]> Co-authored-by: silverwind <[email protected]>
2026-11-06 15:20:29 +00:00 · 2026-10-29 15:53:32 +00:00
1 parent e3607b576e
commit c6567296a0
@@ -94,6 +94,8 @@ func TestDockerProxy(t *testing.T) {
 				return
 			}
 			_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
+		case strings.HasSuffix(r.URL.Path, "/containers/json"):
+			_, _ = w.Write([]byte(`[]`))
 		case strings.HasSuffix(r.URL.Path, "/json"):
 			http.NotFound(w, r)
 		case strings.HasSuffix(r.URL.Path, "/archive"):
@@ -235,7 +237,8 @@ func TestDockerProxy(t *testing.T) {
 			t.Skip("automatic proxy discovery requires Linux")
 		}
 		t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
-		t.Setenv("TMPDIR", shortTempDir(t))
+		tempDir := shortTempDir(t)
+		t.Setenv("TMPDIR", tempDir)
 		require.Nil(t, NewDockerProxy(t.Context(), "failed"))
 		for _, job := range []string{"first", "second"} {
 			proxy := NewDockerProxy(t.Context(), job)
@@ -243,12 +246,10 @@ func TestDockerProxy(t *testing.T) {
 			require.NoError(t, proxy.Close(context.Background()))
 		}
 		assert.EqualValues(t, 2, probes.Load())
-		dockerProxyDirectories.Range(func(key, value any) bool {
-			directory, _ := value.(dockerProxyDirectory)
-			directory.socket, _ = os.Stat(t.TempDir())
-			dockerProxyDirectories.Store(key, directory)
-			return true
-		})
+		workdir, err := os.Getwd()
+		require.NoError(t, err)
+		cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
+		cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
 		proxy := NewDockerProxy(t.Context(), "replaced daemon")
 		require.NotNil(t, proxy)
 		require.NoError(t, proxy.Close(context.Background()))
@@ -476,10 +477,23 @@ func BenchmarkDockerProxy(b *testing.B) {

 type probeClient struct {
 	mobyclient.APIClient
-	create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
-	remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
-	stat   func(string) error
-	mounts []container.MountPoint
+	create     func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
+	remove     func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
+	stat       func(string) error
+	mounts     []container.MountPoint
+	containers []container.Summary
+}
+
+func (c *probeClient) Close() error {
+	return nil
+}
+
+func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
+	return mobyclient.ContainerListResult{Items: c.containers}, nil
+}
+
+func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
+	return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
 }

 func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
@@ -490,8 +504,8 @@ func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (m
 	return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
 }

-func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
-	return c.create(opts)
+func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
+	return c.create(ctx, opts)
 }

 func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
@@ -518,7 +532,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
 				var calls, removals atomic.Int32
 				visible := false
 				cli := &probeClient{
-					create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
+					create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
 						calls.Add(1)
 						<-release
 						return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
@@ -538,23 +552,18 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
 				results := make(chan result, 8)
 				for range cap(results) {
 					go func() {
-						_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
-						results <- result{seen, err}
+						probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
+						results <- result{probe.seen, err}
 					}()
 				}
 				synctest.Wait()
 				assert.EqualValues(t, 1, calls.Load())
-				ctx, cancel := context.WithCancel(t.Context())
-				cancel()
-				_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
-				require.ErrorIs(t, err, context.Canceled)
-				assert.False(t, seen)
 				replacement, err := os.Stat(t.TempDir())
 				require.NoError(t, err)
 				replaced := make(chan result, 1)
 				go func() {
-					_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
-					replaced <- result{seen, err}
+					probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
+					replaced <- result{probe.seen, err}
 				}()
 				synctest.Wait()
 				assert.EqualValues(t, 2, calls.Load())
@@ -574,11 +583,11 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
 					assert.Zero(t, removals.Load())
 				}
 				createErr, visible = nil, true
-				directory, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
+				probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
 				require.NoError(t, err)
-				assert.True(t, seen)
-				assert.Equal(t, dir, directory.dir)
-				assert.Equal(t, dir, directory.daemonDir)
+				assert.True(t, probe.seen)
+				assert.Equal(t, dir, probe.directory.dir)
+				assert.Equal(t, dir, probe.directory.daemonDir)
 				assert.EqualValues(t, 3, calls.Load())
 				entries, err := os.ReadDir(dir)
 				require.NoError(t, err)
@@ -586,7 +595,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
 			})
 		})
 	}
-	t.Run("cancelled probe does not cancel another caller", func(t *testing.T) {
+	t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
 		synctest.Test(t, func(t *testing.T) {
 			dir := t.TempDir()
 			socket, err := os.Stat(t.TempDir())
@@ -595,49 +604,105 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
 			ctx, cancel := context.WithCancel(t.Context())
 			defer cancel()
 			release := make(chan struct{})
-			calls := 0
+			var calls, removals atomic.Int32
 			cli := &probeClient{
-				create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
-					calls++
-					if calls == 1 {
-						<-release
+				create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
+					calls.Add(1)
+					<-release
+					return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
+				},
+				stat: func(_path string) error { return nil },
+				remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
+					removals.Add(1)
+					return mobyclient.ContainerRemoveResult{}, nil
+				},
+			}
+			leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
+			go func() {
+				probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
+				leader <- result{probe.seen, err}
+			}()
+			synctest.Wait()
+			go func() {
+				probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
+				follower <- result{probe.seen, err}
+			}()
+			synctest.Wait()
+			cancel()
+			leaderResult := <-leader
+			require.ErrorIs(t, leaderResult.err, context.Canceled)
+			assert.False(t, leaderResult.seen)
+			go func() {
+				probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
+				late <- result{probe.seen, err}
+			}()
+			synctest.Wait()
+			close(release)
+			for _, results := range []chan result{follower, late} {
+				result := <-results
+				require.NoError(t, result.err)
+				assert.True(t, result.seen)
+			}
+			assert.EqualValues(t, 1, calls.Load())
+			assert.EqualValues(t, 1, removals.Load())
+			_, cached := dockerProxyProbes.Load(key)
+			assert.True(t, cached)
+		})
+	})
+	t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
+		synctest.Test(t, func(t *testing.T) {
+			dir := t.TempDir()
+			socket, err := os.Stat(t.TempDir())
+			require.NoError(t, err)
+			var calls atomic.Int32
+			cli := &probeClient{
+				create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
+					if calls.Add(1) == 1 {
+						<-ctx.Done()
 						return mobyclient.ContainerCreateResult{}, ctx.Err()
-					} else {
-						return mobyclient.ContainerCreateResult{ID: "probe"}, nil
 					}
+					return mobyclient.ContainerCreateResult{ID: "probe"}, nil
 				},
 				stat: func(_path string) error { return nil },
 				remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
 					return mobyclient.ContainerRemoveResult{}, nil
 				},
 			}
-			leader, follower := make(chan result, 1), make(chan result, 1)
-			go func() {
-				_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
-				leader <- result{seen, err}
-			}()
-			synctest.Wait()
-			go func() {
-				_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
-				follower <- result{seen, err}
-			}()
-			synctest.Wait()
-			assert.Equal(t, 1, calls)
-			cancel()
-			close(release)
-			leaderResult, followerResult := <-leader, <-follower
-			require.ErrorIs(t, leaderResult.err, context.Canceled)
-			assert.False(t, leaderResult.seen)
-			require.NoError(t, followerResult.err)
-			assert.True(t, followerResult.seen)
-			assert.Equal(t, 2, calls)
+			probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
+			require.NoError(t, err)
+			assert.True(t, probe.seen)
+			assert.EqualValues(t, 2, calls.Load())
 		})
 	})
 }

+func TestRemoveStaleProbeContainers(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
+		writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
+		var removed []string
+		cli := &probeClient{
+			containers: []container.Summary{
+				{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
+				{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
+				{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
+			},
+			remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
+				removed = append(removed, id)
+				return mobyclient.ContainerRemoveResult{}, nil
+			},
+		}
+		start := time.Now()
+		removeStaleProbeContainers(cli)
+		assert.Equal(t, []string{"abandoned"}, removed)
+		assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
+	})
+}
+
 func TestDaemonSeesDir(t *testing.T) {
 	dir := t.TempDir()
 	markers := make(map[string]bool)
+
 	for _, testCase := range []struct {
 		name      string
 		daemonDir string
@@ -656,7 +721,7 @@ func TestDaemonSeesDir(t *testing.T) {
 			removed := false
 			var attemptedImages []string
 			cli := &probeClient{
-				create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
+				create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
 					assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
 					attemptedImages = append(attemptedImages, opts.Config.Image)
 					if opts.Config.Image == "removed-image" {
@@ -692,6 +757,7 @@ func TestDaemonSeesDir(t *testing.T) {
 			assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
 			assert.Equal(t, testCase.seen, seen)
 			assert.Equal(t, testCase.removed, removed)
+
 			entries, err := os.ReadDir(dir)
 			require.NoError(t, err)
 			assert.Empty(t, entries)