perf: speed up job setup, action preparation and the cache server (#1327)

- Look up containers by name instead of listing all of them
- On Linux, cache the docker proxy discovery instead of starting a probe container per job
- Download a job's actions concurrently and share identical fetches between concurrent jobs
- Cache `runner.arch` per run and poll service health every 250ms
- Cache server: queue on an in-process lock instead of polling the database file lock, upload S3 objects up to 64MiB in one request and larger ones in 4 concurrent parts

| `exec`, median of 5 | Before | After |
|---|---:|---:|
| 1 step | 1.70s | 0.87s |
| 10 steps | 2.84s | 1.97s |
| Service + step | 3.78s | 2.18s |
| checkout + Node action | 3.64s | 2.52s |

Reviewed-on: https://gitea.com/gitea/runner/pulls/1327
Reviewed-by: Zettat123 <[email protected]>

act/artifactcache/handler_test.go

@@ -17,12 +17,14 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/timshannon/bolthold"
 	"go.etcd.io/bbolt"
+	berrors "go.etcd.io/bbolt/errors"
 )
 
 // testToken is registered with the cache server in every test that needs to
@@ -114,6 +116,26 @@ func TestHandler(t *testing.T) {
 
 	base := fmt.Sprintf("%s%s", handler.ExternalURL(), apiPath)
 
+	t.Run("database open recovers from failure, times out while held and close is idempotent", func(t *testing.T) {
+		dir := t.TempDir()
+		synctest.Test(t, func(t *testing.T) {
+			handler := &Handler{dir: dir, dbLock: make(chan struct{}, 1)}
+			require.NoError(t, os.Mkdir(filepath.Join(handler.dir, "bolt.db"), 0o755))
+			_, err := handler.openDB()
+			require.Error(t, err)
+			require.NoError(t, os.Remove(filepath.Join(handler.dir, "bolt.db")))
+			first, err := handler.openDB()
+			require.NoError(t, err)
+			_, err = handler.openDB()
+			require.ErrorIs(t, err, berrors.ErrTimeout)
+			require.NoError(t, first.Close())
+			require.NoError(t, first.Close())
+			second, err := handler.openDB()
+			require.NoError(t, err)
+			require.NoError(t, second.Close())
+		})
+	})
+
 	defer func() {
 		t.Run("inpect db", func(t *testing.T) {
 			db, err := handler.openDB()