fix(cache): stop the cache server from running its volume out of space (#1377)

Committing a cache upload copied every part into a new archive before deleting the parts, so a 5 GB save needed 10 GB free. Failed writes and assemblies left their partial bytes until the next stale sweep, up to an hour later, and every other save on the runner failed meanwhile. Nothing made room for an upload, so once the cache had filled its volume down to the free space floor, larger saves kept failing.

- Assemble in place, deleting each part once appended.
- Drop partial parts and archives on failure.
- Before a part would cross the floor, evict the least recently used entries.
- Reject uploads to reservations the sweep already deleted, which left files nothing removed.

_Written by Claude (Opus 5.5)._

Reviewed-on: https://gitea.com/gitea/runner/pulls/1377
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>
This commit is contained in:
silverwind authored and bircni committed 2026-11-04 16:38:21 +00:00
1 parent dcba546cb3
commit 3e1b4b42c2
6 files changed
+60 -51

No files matched your search

+15 -9
View File
@@ -18,6 +18,7 @@ import (
"strconv"
"strings"
"testing"
"testing/iotest"
"testing/synctest"
"time"
@@ -673,7 +674,7 @@ func TestHandler_evictPolicy(t *testing.T) {
}
}
func TestHandler_evictForFreeSpace(t *testing.T) {
func TestHandler_makeRoom(t *testing.T) {
t.Parallel()
free := func(n int64) func(string) (uint64, error) {
return func(string) (uint64, error) { return uint64(n), nil }
@@ -682,14 +683,16 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
for _, tc := range []struct {
name string
freeDisk func(string) (uint64, error)
upload int64
kept []string
}{
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
{"ample free space evicts nothing", free(defaultMinFreeDisk), 0, []string{"oldest", "middle", "newest"}},
{"room for an upload sheds the oldest entry", free(defaultMinFreeDisk), 4 * miB, []string{"middle", "newest"}},
{"a shortfall the cache cannot cover sheds all of it", free(0), 0, nil},
{
"an unreadable volume is treated as unavailable, not as full",
func(string) (uint64, error) { return 0, errors.New("unsupported") },
0,
[]string{"oldest", "middle", "newest"},
},
} {
@@ -702,11 +705,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
}
handler := newTestHandler(t, Policy{}, entries...)
handler.freeDisk = tc.freeDisk
db, err := handler.openDB()
require.NoError(t, err)
handler.evictForFreeSpace(db)
require.NoError(t, db.Close())
handler.makeRoom(tc.upload)
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
})
@@ -888,6 +887,13 @@ func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
return kept
}
func TestStorageDropsFailedWrites(t *testing.T) {
storage, err := NewStorage(t.TempDir())
require.NoError(t, err)
require.Error(t, storage.Write(1, 0, io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(io.ErrUnexpectedEOF))))
assert.NoFileExists(t, storage.tempName(1, 0))
}
func writeBlob(t *testing.T, handler *Handler, id uint64) {
t.Helper()
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))