fix(cache): stop the cache server from running its volume out of space (#1377)

Committing a cache upload copied every part into a new archive before deleting the parts, so a 5 GB save needed 10 GB free. Failed writes and assemblies left their partial bytes until the next stale sweep, up to an hour later, and every other save on the runner failed meanwhile. Nothing made room for an upload, so once the cache had filled its volume down to the free space floor, larger saves kept failing.

- Assemble in place, deleting each part once appended.
- Drop partial parts and archives on failure.
- Before a part would cross the floor, evict the least recently used entries.
- Reject uploads to reservations the sweep already deleted, which left files nothing removed.

_Written by Claude (Opus 5.5)._

Reviewed-on: https://gitea.com/gitea/runner/pulls/1377
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>

act/artifactcache/handler.go

@@ -124,7 +124,7 @@ type Handler struct {
 
 	policy Policy
 
-	// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
+	// freeDisk is a field so tests can drive makeRoom without a full volume.
 	freeDisk func(string) (uint64, error)
 }
 
@@ -773,6 +773,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
 		h.responseJSON(w, r, 400, err)
 		return
 	}
+	h.makeRoom(r.ContentLength)
 	if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
 		h.responseJSON(w, r, 500, err)
 		return
@@ -875,11 +876,8 @@ func (h *Handler) commitCache(cache *Cache) error {
 	if h.s3 != nil {
 		return nil // lifecycle rules on the bucket bound its size
 	}
-	// A commit is the only thing that grows the store, so the only thing that can push the
-	// volume under the floor.
 	h.evictRepo(db, cache.Repo)
 	h.evictTotal(db)
-	h.evictForFreeSpace(db)
 	return nil
 }
 
@@ -1173,9 +1171,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
 	defer db.Close()
 
 	cache, err := db.get(id)
-	if errors.Is(err, errCacheNotFound) {
-		return nil
-	}
 	if err != nil {
 		return err
 	}
@@ -1260,23 +1255,29 @@ func (h *Handler) gcCache() {
 	h.evictExpired(db)
 	h.evictSuperseded(db)
 	h.evictOversized(db)
-	h.evictForFreeSpace(db)
+	_ = db.Close()
+	h.makeRoom(0)
 }
 
-// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
-// accounted for.
-func (h *Handler) evictForFreeSpace(db *cacheDB) {
+// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
+func (h *Handler) makeRoom(need int64) {
 	free, err := h.freeDisk(h.dir)
 	if err != nil {
 		h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
 		return
 	}
-	if free >= uint64(h.policy.MinFreeDisk) {
+	shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
+	if h.s3 != nil || shortfall <= 0 {
 		return
 	}
+	db, err := h.openDB()
+	if err != nil {
+		return
+	}
+	defer db.Close()
 
 	caches := h.completedByUse(db)
-	total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
+	total := totalSize(caches)
 	if total <= shortfall {
 		// Say so, or shedding everything and still being short reads as the backstop working.
 		h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)