perf: speed up job setup, action preparation and the cache server (#1327)

- Look up containers by name instead of listing all of them
- On Linux, cache the docker proxy discovery instead of starting a probe container per job
- Download a job's actions concurrently and share identical fetches between concurrent jobs
- Cache `runner.arch` per run and poll service health every 250ms
- Cache server: queue on an in-process lock instead of polling the database file lock, upload S3 objects up to 64MiB in one request and larger ones in 4 concurrent parts

| `exec`, median of 5 | Before | After |
|---|---:|---:|
| 1 step | 1.70s | 0.87s |
| 10 steps | 2.84s | 1.97s |
| Service + step | 3.78s | 2.18s |
| checkout + Node action | 3.64s | 2.52s |

Reviewed-on: https://gitea.com/gitea/runner/pulls/1327
Reviewed-by: Zettat123 <[email protected]>

act/container/docker_images.go

@@ -58,18 +58,14 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
 	}
 	defer cli.Close()
 
-	if _, err := cli.ImageInspect(ctx, imageName); cerrdefs.IsNotFound(err) {
+	if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
+		Force:         force,
+		PruneChildren: pruneChildren,
+	}); cerrdefs.IsNotFound(err) {
 		return false, nil
 	} else if err != nil {
 		return false, err
 	}
 
-	if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
-		Force:         force,
-		PruneChildren: pruneChildren,
-	}); err != nil {
-		return false, err
-	}
-
 	return true, nil
 }

act/container/docker_images_test.go

@@ -66,4 +66,11 @@ func TestImageExistsLocally(t *testing.T) {
 	wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
 	assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
 	assert.False(t, wrongPlatform)
+
+	removed, err := RemoveImage(ctx, amd64Ref, false, false)
+	require.NoError(t, err)
+	assert.True(t, removed)
+	removed, err = RemoveImage(ctx, amd64Ref, false, false)
+	require.NoError(t, err)
+	assert.False(t, removed)
 }

act/container/docker_proxy.go

@@ -48,6 +48,13 @@ var (
 	rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
 )
 
+var dockerProxyDirectories sync.Map
+
+type dockerProxyDirectory struct {
+	dir, daemonDir string
+	socket         os.FileInfo
+}
+
 func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
 		return nil
@@ -63,15 +70,24 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	if !ok {
 		return nil
 	}
-	if info, err := os.Stat(daemonSocket); err != nil || info.Mode()&os.ModeSocket == 0 {
+	socket, err := os.Stat(daemonSocket)
+	if err != nil || socket.Mode()&os.ModeSocket == 0 {
 		return nil
 	}
-	dir, daemonDir := runnerContainerWorkdir(probeCtx, cli)
-	seen := daemonDir != ""
+	workdir, _ := os.Getwd()
+	key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
+	cached, _ := dockerProxyDirectories.Load(key)
+	directory, seen := cached.(dockerProxyDirectory)
+	seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
+	dir, daemonDir := directory.dir, directory.daemonDir
 	if !seen {
-		if dir, err = filepath.Abs(os.TempDir()); err == nil {
-			daemonDir = dir
-			seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
+		dir, daemonDir = runnerContainerWorkdir(probeCtx, cli)
+		seen = daemonDir != ""
+		if !seen {
+			if dir, err = filepath.Abs(os.TempDir()); err == nil {
+				daemonDir = dir
+				seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
+			}
 		}
 	}
 	if err != nil {
@@ -87,9 +103,11 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	}
 	proxy, err := StartDockerProxy(daemonSocket, dir, job)
 	if err != nil {
+		dockerProxyDirectories.Delete(key)
 		common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
 		return nil
 	}
+	dockerProxyDirectories.Store(key, dockerProxyDirectory{dir, daemonDir, socket})
 	proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
 	return proxy
 }

act/container/docker_proxy_test.go

@@ -20,6 +20,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -68,6 +69,7 @@ func TestDockerProxy(t *testing.T) {
 	bodies := make(chan []byte, 8)
 	createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
 	streamsDone := make(chan struct{}, 2)
+	var probes atomic.Int32
 	daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
 	listener, err := net.Listen("unix", daemonSocket)
 	require.NoError(t, err)
@@ -75,12 +77,28 @@ func TestDockerProxy(t *testing.T) {
 		switch {
 		case strings.HasSuffix(r.URL.Path, "/create"):
 			body, _ := io.ReadAll(r.Body)
+			if bytes.Contains(body, []byte(`"Image":"probe-image"`)) {
+				_, _ = w.Write([]byte(`{"Id":"probe"}`))
+				return
+			}
 			if r.URL.RawQuery == "wait" {
 				close(createStarted)
 				<-releaseCreate
 			}
 			bodies <- body
 			w.WriteHeader(http.StatusCreated)
+		case strings.HasSuffix(r.URL.Path, "/images/json"):
+			if probes.Add(1) == 1 {
+				http.Error(w, "probe unavailable", http.StatusServiceUnavailable)
+				return
+			}
+			_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
+		case strings.HasSuffix(r.URL.Path, "/json"):
+			http.NotFound(w, r)
+		case strings.HasSuffix(r.URL.Path, "/archive"):
+			w.Header().Set("X-Docker-Container-Path-Stat", "e30=")
+		case r.Method == http.MethodDelete:
+			w.WriteHeader(http.StatusNoContent)
 		case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
 			if r.URL.Path == "/exec/error/start" {
 				w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
@@ -211,6 +229,31 @@ func TestDockerProxy(t *testing.T) {
 		assert.Equal(t, "OK GET /v1.47/_ping", string(body))
 	})
 
+	t.Run("discovery caches only successes", func(t *testing.T) {
+		if runtime.GOOS != "linux" {
+			t.Skip("automatic proxy discovery requires Linux")
+		}
+		t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
+		t.Setenv("TMPDIR", shortTempDir(t))
+		require.Nil(t, NewDockerProxy(t.Context(), "failed"))
+		for _, job := range []string{"first", "second"} {
+			proxy := NewDockerProxy(t.Context(), job)
+			require.NotNil(t, proxy)
+			require.NoError(t, proxy.Close(context.Background()))
+		}
+		assert.EqualValues(t, 2, probes.Load())
+		dockerProxyDirectories.Range(func(key, value any) bool {
+			directory, _ := value.(dockerProxyDirectory)
+			directory.socket, _ = os.Stat(t.TempDir())
+			dockerProxyDirectories.Store(key, directory)
+			return true
+		})
+		proxy := NewDockerProxy(t.Context(), "replaced daemon")
+		require.NotNil(t, proxy)
+		require.NoError(t, proxy.Close(context.Background()))
+		assert.EqualValues(t, 3, probes.Load())
+	})
+
 	t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
 		for _, stream := range []struct{ path, upgrade string }{
 			{path: "/v1.47/exec/abc/start"},

act/container/docker_run.go

@@ -476,22 +476,23 @@ func (cr *containerReference) find() common.Executor {
 			}
 			cr.id = ""
 		}
-		containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{
-			All: true,
-		})
+		result, err := cr.cli.ContainerInspect(ctx, cr.input.Name, client.ContainerInspectOptions{})
+		if err == nil && result.Container.Name == "/"+cr.input.Name {
+			cr.id = result.Container.ID
+			return nil
+		} else if cerrdefs.IsNotFound(err) {
+			return nil
+		}
+		containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{All: true}) // inspect also matches ids, and podman fails it with a 500 for a container removed mid-request
 		if err != nil {
 			return fmt.Errorf("failed to list containers: %w", err)
 		}
-
-		for _, c := range containers.Items {
-			for _, name := range c.Names {
-				if name[1:] == cr.input.Name {
-					cr.id = c.ID
-					return nil
-				}
+		for _, candidate := range containers.Items {
+			if slices.Contains(candidate.Names, "/"+cr.input.Name) {
+				cr.id = candidate.ID
+				break
 			}
 		}
-
 		return nil
 	}
 }

act/container/docker_run_test.go

@@ -13,9 +13,13 @@ import (
 	"errors"
 	"io"
 	"net"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"gitea.com/gitea/runner/act/common"
@@ -408,13 +412,12 @@ func TestFindRevalidatesStaleID(t *testing.T) {
 		client := &mockDockerClient{}
 		return &containerReference{id: id, cli: client, input: &NewContainerInput{Name: "job-1"}}, client
 	}
-	listOpts := mobyclient.ContainerListOptions{All: true}
 	inspectOpts := mobyclient.ContainerInspectOptions{}
 
 	t.Run("stale id cleared, name lookup empty", func(t *testing.T) {
 		cr, client := newCR("stale")
 		client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
-		client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, nil)
+		client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
 		require.NoError(t, cr.find()(ctx))
 		assert.Empty(t, cr.id)
 		client.AssertExpectations(t)
@@ -423,8 +426,18 @@ func TestFindRevalidatesStaleID(t *testing.T) {
 	t.Run("stale id cleared, name lookup repopulates", func(t *testing.T) {
 		cr, client := newCR("stale")
 		client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
-		client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{Items: []container.Summary{
-			{ID: "other", Names: []string{"/somebody-else"}},
+		client.On("ContainerInspect", ctx, "job-1", inspectOpts).
+			Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
+		require.NoError(t, cr.find()(ctx))
+		assert.Equal(t, "fresh", cr.id)
+		client.AssertExpectations(t)
+	})
+
+	t.Run("id match falls back to listing", func(t *testing.T) {
+		cr, client := newCR("")
+		client.On("ContainerInspect", ctx, "job-1", inspectOpts).
+			Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "job-1abc", Name: "/somebody-else"}}, nil)
+		client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: []container.Summary{
 			{ID: "fresh", Names: []string{"/job-1"}},
 		}}, nil)
 		require.NoError(t, cr.find()(ctx))
@@ -448,20 +461,52 @@ func TestFindRevalidatesStaleID(t *testing.T) {
 		client.AssertExpectations(t)
 	})
 
-	t.Run("list error propagates", func(t *testing.T) {
+	t.Run("name lookup error falls back to listing", func(t *testing.T) {
 		cr, client := newCR("")
-		client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, boom)
+		client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{}, boom)
+		client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: []container.Summary{
+			{ID: "other", Names: []string{"/somebody-else"}},
+			{ID: "fresh", Names: []string{"/job-1"}},
+		}}, nil).Once()
+		require.NoError(t, cr.find()(ctx))
+		assert.Equal(t, "fresh", cr.id)
+		client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, boom)
+		cr.id = ""
 		require.ErrorIs(t, cr.find()(ctx), boom)
 		client.AssertExpectations(t)
 	})
 }
 
+func TestRunnerArchCachedPerRun(t *testing.T) {
+	var infos atomic.Int32
+	daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		writer.Header().Set("Content-Type", "application/json")
+		writer.Header().Set("API-Version", "1.47")
+		if strings.HasSuffix(request.URL.Path, "/info") {
+			infos.Add(1)
+			_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
+		}
+	}))
+	t.Cleanup(daemon.Close)
+	t.Setenv("DOCKER_HOST", daemon.URL)
+	extensions := &LinuxContainerEnvironmentExtensions{}
+	run := WithRunnerArch(t.Context())
+	var lookups sync.WaitGroup
+	for range 4 {
+		lookups.Go(func() { assert.Equal(t, "ARM64", extensions.GetRunnerContext(run)["arch"]) })
+	}
+	lookups.Wait()
+	assert.Equal(t, "ARM64", extensions.GetRunnerContext(t.Context())["arch"])
+	assert.EqualValues(t, 2, infos.Load())
+}
+
 // Every daemon entry point fails fast with a clear, container-named
 // error when no live cr.id is known.
 func TestRejectsMissingContainer(t *testing.T) {
 	ctx := t.Context()
 	client := &mockDockerClient{}
-	client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
+	client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
+		Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
 	cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
 	check := func(op string, err error) {
 		t.Helper()
@@ -498,10 +543,8 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
 			client := &mockDockerClient{}
 			client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
 				Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
-			client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
-				Return(mobyclient.ContainerListResult{Items: []container.Summary{
-					{ID: "fresh", Names: []string{"/job-1"}},
-				}}, nil)
+			client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
+				Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
 			client.On("CopyToContainer", ctx, "fresh", mock.Anything).
 				Run(func(args mock.Arguments) {
 					opts := args.Get(2).(mobyclient.CopyToContainerOptions)

act/container/linux_container_environment_extensions.go

@@ -69,10 +69,41 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
 // DefaultToolCache is where the runner mounts the tool cache inside job containers.
 const DefaultToolCache = "/opt/hostedtoolcache"
 
+type runnerArchKey struct{}
+
+func WithRunnerArch(ctx context.Context) context.Context { // shares one architecture lookup per run
+	if ctx.Value(runnerArchKey{}) != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, runnerArchKey{}, &runnerArchCache{lock: make(chan struct{}, 1)})
+}
+
+type runnerArchCache struct {
+	lock chan struct{} // lets cancelled jobs stop waiting for the cache
+	arch string
+}
+
+func cachedRunnerArch(ctx context.Context) string {
+	cache, ok := ctx.Value(runnerArchKey{}).(*runnerArchCache)
+	if !ok {
+		return RunnerArch(ctx)
+	}
+	select {
+	case cache.lock <- struct{}{}:
+		defer func() { <-cache.lock }()
+	case <-ctx.Done():
+		return ""
+	}
+	if cache.arch == "" {
+		cache.arch = RunnerArch(ctx)
+	}
+	return cache.arch
+}
+
 func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
 	return map[string]any{
 		"os":         "Linux",
-		"arch":       RunnerArch(ctx),
+		"arch":       cachedRunnerArch(ctx),
 		"temp":       "/tmp",
 		"tool_cache": DefaultToolCache,
 	}