perf: speed up job setup, action preparation and the cache server (#1327)

- Look up containers by name instead of listing all of them
- On Linux, cache the docker proxy discovery instead of starting a probe container per job
- Download a job's actions concurrently and share identical fetches between concurrent jobs
- Cache `runner.arch` per run and poll service health every 250ms
- Cache server: queue on an in-process lock instead of polling the database file lock, upload S3 objects up to 64MiB in one request and larger ones in 4 concurrent parts

| `exec`, median of 5 | Before | After |
|---|---:|---:|
| 1 step | 1.70s | 0.87s |
| 10 steps | 2.84s | 1.97s |
| Service + step | 3.78s | 2.18s |
| checkout + Node action | 3.64s | 2.52s |

Reviewed-on: https://gitea.com/gitea/runner/pulls/1327
Reviewed-by: Zettat123 <[email protected]>

act/container/docker_proxy.go

@@ -48,6 +48,13 @@ var (
 	rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
 )
 
+var dockerProxyDirectories sync.Map
+
+type dockerProxyDirectory struct {
+	dir, daemonDir string
+	socket         os.FileInfo
+}
+
 func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
 		return nil
@@ -63,15 +70,24 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	if !ok {
 		return nil
 	}
-	if info, err := os.Stat(daemonSocket); err != nil || info.Mode()&os.ModeSocket == 0 {
+	socket, err := os.Stat(daemonSocket)
+	if err != nil || socket.Mode()&os.ModeSocket == 0 {
 		return nil
 	}
-	dir, daemonDir := runnerContainerWorkdir(probeCtx, cli)
-	seen := daemonDir != ""
+	workdir, _ := os.Getwd()
+	key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
+	cached, _ := dockerProxyDirectories.Load(key)
+	directory, seen := cached.(dockerProxyDirectory)
+	seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
+	dir, daemonDir := directory.dir, directory.daemonDir
 	if !seen {
-		if dir, err = filepath.Abs(os.TempDir()); err == nil {
-			daemonDir = dir
-			seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
+		dir, daemonDir = runnerContainerWorkdir(probeCtx, cli)
+		seen = daemonDir != ""
+		if !seen {
+			if dir, err = filepath.Abs(os.TempDir()); err == nil {
+				daemonDir = dir
+				seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
+			}
 		}
 	}
 	if err != nil {
@@ -87,9 +103,11 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
 	}
 	proxy, err := StartDockerProxy(daemonSocket, dir, job)
 	if err != nil {
+		dockerProxyDirectories.Delete(key)
 		common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
 		return nil
 	}
+	dockerProxyDirectories.Store(key, dockerProxyDirectory{dir, daemonDir, socket})
 	proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
 	return proxy
 }