feat(cache): add shared S3-compatible storage (#1244)

Store the cache used by `actions/cache` in an S3-compatible bucket such as AWS S3 or RustFS, with no `cache-server`. Runners with the same settings use the same cache.

A cache saved on one runner can be restored on the others within 30 seconds. The runner talks to the bucket over the S3 REST API with SigV4, without an SDK. Lifecycle rules on the bucket expire old caches.

*Description rewritten by Claude.*

---------

Co-authored-by: silverwind <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1244
Reviewed-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>

internal/pkg/config/config.example.yaml

@@ -166,6 +166,19 @@ cache:
   # Surrounding whitespace is trimmed, so a trailing newline in the file is fine.
   # Setting both external_secret and external_secret_file is an error.
   #external_secret_file: ""
+  # Store the cache in an S3-compatible bucket such as AWS S3, MinIO or RustFS. Runners with
+  # the same settings use the same cache. Cannot be combined with external_server. Add lifecycle
+  # rules to the bucket that expire old caches and abort incomplete multipart uploads.
+  # region defaults to us-east-1, and path_style to false for AWS and true otherwise.
+  # access_key and secret_key can also be set directly instead of read from files.
+  #s3:
+  #  endpoint: "http://minio:9000"
+  #  region: "us-east-1"
+  #  bucket: "runner-cache"
+  #  prefix: "gitea-runner-cache"
+  #  path_style: true
+  #  access_key_file: "/run/secrets/cache-s3-access-key"
+  #  secret_key_file: "/run/secrets/cache-s3-secret-key"
   # When true, reuse a cached action instead of fetching from the remote on every job.
   # A moved tag (e.g. a re-tagged "v6") or an updated branch stays at the cached commit
   # until its cache entry expires or is manually removed.
@@ -176,8 +189,8 @@ cache:
   # advertised. A bundle that does not match is left alone. With v2, artifact uploads go via the
   # cache server.
   #v2: true
-  # How the cache server discards entries, ignored when external_server is set since that
-  # server applies its own. Leave a setting out for its default; 0s or 0 turns the three
+  # How the cache server discards entries, ignored when external_server or s3 is set since
+  # those apply their own. Leave a setting out for its default; 0s or 0 turns the three
   # limits off. Sizes accept 10GB, 512mb, 1TiB or a plain byte count, binary either way.
   # Whatever these allow, the cache still sheds entries to keep free space on its volume
   # above health_check.min_free_disk_space_mb when health checks are enabled.

internal/pkg/config/config.go

@@ -101,13 +101,28 @@ type Cache struct {
 	OfflineMode        bool   `yaml:"offline_mode"`         // OfflineMode reuses a cached action without fetching from the remote; a moved tag or branch stays at the cached commit until the cache entry is removed.
 	V2                 *bool  `yaml:"v2"`                   // V2 advertises the actions cache service v2 API to jobs. The bundle edit that reaches it is made either way, the artifact actions need it too. Unset means enabled.
 
-	// Eviction settings, ignored when ExternalServer is set since that server applies its own.
+	S3 *CacheS3 `yaml:"s3"` // S3 stores the cache in an S3-compatible bucket, so runners with the same settings use the same cache.
+
+	// Eviction settings, ignored when ExternalServer is set since that server applies its own. With S3, bucket lifecycle rules replace all but SweepInterval.
 	Retention     time.Duration `yaml:"retention"`       // Retention removes entries nothing has read or written within this window. Default 168h, 0 keeps them regardless of age.
 	RepoSizeLimit Size          `yaml:"repo_size_limit"` // RepoSizeLimit caps one repository, evicting least recently accessed first. Default 10GB, 0 is no limit.
 	SizeLimit     Size          `yaml:"size_limit"`      // SizeLimit caps the whole cache the same way. No limit by default.
 	SweepInterval time.Duration `yaml:"sweep_interval"`  // SweepInterval is the minimum time between two eviction sweeps. Default 1h; a cadence has no "off".
 }
 
+// CacheS3 is an S3-compatible bucket that stores the cache.
+type CacheS3 struct {
+	Endpoint      string `yaml:"endpoint"`
+	Region        string `yaml:"region"`
+	Bucket        string `yaml:"bucket"`
+	Prefix        string `yaml:"prefix"`
+	PathStyle     *bool  `yaml:"path_style"`
+	AccessKey     string `yaml:"access_key"`
+	AccessKeyFile string `yaml:"access_key_file"`
+	SecretKey     string `yaml:"secret_key"`
+	SecretKeyFile string `yaml:"secret_key_file"`
+}
+
 // DefaultCache returns the cache eviction defaults, seeded before the file is read so a
 // written 0 can mean off. SizeLimit stays zero: the free space floor bounds the whole cache.
 func DefaultCache() Cache {
@@ -266,7 +281,7 @@ func LoadDefault(file string) (*Config, error) {
 		cfg.Cache.Enabled = &b
 	}
 	// Resolved regardless of cache.enabled, because the `cache-server` command reads the secret from the same key without checking cache.enabled.
-	if err := resolveCacheExternalSecret(cfg); err != nil {
+	if err := resolveSecretFile("cache.external_secret", &cfg.Cache.ExternalSecret, cfg.Cache.ExternalSecretFile); err != nil {
 		return nil, err
 	}
 	if *cfg.Cache.Enabled {
@@ -280,6 +295,9 @@ func LoadDefault(file string) (*Config, error) {
 		if cfg.Cache.ExternalServer != "" && cfg.Cache.ExternalSecret == "" {
 			return nil, errors.New("cache.external_server is set but no shared secret is configured; set cache.external_secret (or cache.external_secret_file) to the same value used by the gitea-runner cache-server")
 		}
+		if err := resolveCacheS3(&cfg.Cache); err != nil {
+			return nil, err
+		}
 	}
 	if cfg.Container.WorkdirParent == "" {
 		cfg.Container.WorkdirParent = "workspace"
@@ -426,23 +444,44 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
 	return defined, nil
 }
 
-// resolveCacheExternalSecret loads cache.external_secret from the file named by cache.external_secret_file,
-// so deployments can mount the secret instead of committing it to the config file.
-func resolveCacheExternalSecret(cfg *Config) error {
-	if cfg.Cache.ExternalSecretFile == "" {
+// resolveSecretFile reads key from key_file, so deployments can mount a secret instead of committing it.
+func resolveSecretFile(key string, secret *string, file string) error {
+	if file == "" {
 		return nil
 	}
-	if cfg.Cache.ExternalSecret != "" {
-		return errors.New("cache.external_secret and cache.external_secret_file are both set; configure only one of them")
+	if *secret != "" {
+		return fmt.Errorf("%s and %s_file are both set; configure only one of them", key, key)
 	}
-	content, err := os.ReadFile(cfg.Cache.ExternalSecretFile)
+	content, err := os.ReadFile(file)
 	if err != nil {
-		return fmt.Errorf("read cache.external_secret_file %q: %w", cfg.Cache.ExternalSecretFile, err)
+		return fmt.Errorf("read %s_file %q: %w", key, file, err)
 	}
-	secret := strings.TrimSpace(string(content))
-	if secret == "" {
-		return fmt.Errorf("cache.external_secret_file %q contains no secret", cfg.Cache.ExternalSecretFile)
+	*secret = strings.TrimSpace(string(content))
+	if *secret == "" {
+		return fmt.Errorf("%s_file %q contains no secret", key, file)
 	}
-	cfg.Cache.ExternalSecret = secret
+	return nil
+}
+
+func resolveCacheS3(cache *Cache) error {
+	s3 := cache.S3
+	switch {
+	case s3 == nil:
+		return nil
+	case cache.ExternalServer != "":
+		return errors.New("cache.s3 and cache.external_server cannot both be set")
+	case s3.Endpoint == "" || s3.Bucket == "":
+		return errors.New("cache.s3.endpoint and cache.s3.bucket must be set")
+	}
+	if err := resolveSecretFile("cache.s3.access_key", &s3.AccessKey, s3.AccessKeyFile); err != nil {
+		return err
+	}
+	if err := resolveSecretFile("cache.s3.secret_key", &s3.SecretKey, s3.SecretKeyFile); err != nil {
+		return err
+	}
+	if s3.AccessKey == "" || s3.SecretKey == "" {
+		return errors.New("cache.s3.access_key (or cache.s3.access_key_file) and cache.s3.secret_key (or cache.s3.secret_key_file) must be set")
+	}
+	s3.Prefix = strings.Trim(s3.Prefix, "/")
 	return nil
 }

internal/pkg/config/config_test.go

@@ -42,6 +42,36 @@ cache:
 	require.NoError(t, err)
 }
 
+func TestLoadDefault_CacheS3(t *testing.T) {
+	dir := t.TempDir()
+	accessPath := filepath.Join(dir, "access.key")
+	require.NoError(t, os.WriteFile(accessPath, []byte("access\n"), 0o600))
+	path := filepath.Join(dir, "config.yaml")
+	content := `
+cache:
+  s3:
+    endpoint: "http://minio:9000"
+    bucket: "caches"
+    prefix: "/runners/"
+    access_key_file: "` + accessPath + `"
+    secret_key: "secret"
+`
+	require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, CacheS3{
+		Endpoint: "http://minio:9000", Bucket: "caches", Prefix: "runners",
+		AccessKey: "access", AccessKeyFile: accessPath, SecretKey: "secret",
+	}, *cfg.Cache.S3)
+
+	require.NoError(t, os.WriteFile(path, []byte(content+`  external_server: "http://cache.invalid/"
+  external_secret: "shh"
+`), 0o600))
+	_, err = LoadDefault(path)
+	assert.ErrorContains(t, err, "cache.s3 and cache.external_server cannot both be set")
+}
+
 func TestLoadDefault_ToolCacheMode(t *testing.T) {
 	cfg, err := LoadDefault("")
 	require.NoError(t, err)