fix: keep the shared action cache as extracted, re-fetch damaged trees (#1352)

The toolkit patch rewrote bundles in place in the shared action cache. Since https://gitea.com/gitea/runner/pulls/1316 trees are never refreshed, so a bundle left truncated failed every later job using it.

Patched bundles now only overwrite the job container's copy, so the runner no longer writes to an extracted tree.

Extraction records each file's size beside the tree, and a tree that no longer matches is fetched again, offline mode included, as a missing one always was. This also catches damage from outside the runner, such as storage losing a write, for a stat per file, a fraction of the copy each stage already makes. Trees from older runners have no record and download once more, except in offline mode, which keeps using them as before.

Generated with Claude Code.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1352
Reviewed-by: Lunny Xiao <[email protected]>
Reviewed-by: bircni <[email protected]>
Co-authored-by: Zettat123 <[email protected]>

act/common/git/fetch_action.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"encoding/json"
 	"errors"
 	"io"
 	"math/rand/v2"
@@ -28,6 +29,13 @@ import (
 
 var actionFetches sync.Map
 
+const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
+
+type treeFile struct {
+	Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
+	Size int64
+}
+
 type actionFetch struct {
 	done     chan struct{}
 	dir, sha string
@@ -86,9 +94,14 @@ func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, s
 	record := filepath.Join(input.Dir, "ref")
 	data, _ := os.ReadFile(record)
 	sha, etag, _ := strings.Cut(string(data), "\n")
-	if !isFullObjectID(sha) || !exists(tree(sha)) {
+	switch {
+	case !isFullObjectID(sha):
 		sha, etag = "", ""
-	} else if input.OfflineMode || isFullObjectID(input.Ref) {
+	case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
+		return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
+	case !treeIntact(input.Dir, sha):
+		sha, etag = "", ""
+	case input.OfflineMode || isFullObjectID(input.Ref):
 		return tree(sha), sha, nil
 	}
 
@@ -173,7 +186,7 @@ func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, toke
 		newEtag := response.Header.Get("ETag")
 		gz, err := gzip.NewReader(response.Body)
 		if err == nil {
-			sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
+			sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
 		}
 		var netErr net.Error
 		if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
@@ -191,18 +204,18 @@ func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, e
 	clone := input
 	clone.Dir = filepath.Join(input.Dir, "git")
 	sha, _, err := resolveClone(ctx, clone)
-	if err != nil || exists(filepath.Join(input.Dir, sha)) {
+	if err != nil || treeIntact(input.Dir, sha) {
 		return sha, err
 	}
 	archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
 	if err != nil {
 		return "", err
 	}
-	_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
+	_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
 	return sha, err
 }
 
-func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
+func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
 	temp, err := os.MkdirTemp(dir, "tmp-")
 	if err != nil {
 		return "", err
@@ -215,6 +228,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
 	defer root.Close()
 
 	sha := ""
+	sizes := map[string]int64{}
 	reader := tar.NewReader(archive)
 	for {
 		header, err := reader.Next()
@@ -225,7 +239,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
 			return "", err
 		}
 		if header.Typeflag == tar.TypeXGlobalHeader {
-			if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
+			if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
 				return sha, nil
 			}
 			continue
@@ -241,10 +255,10 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
 		case tar.TypeDir:
 			err = root.MkdirAll(name, 0o755)
 		case tar.TypeReg:
-			err = writeTreeFile(root, name, header.Mode, reader)
+			sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
 		case tar.TypeSymlink:
 			if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
-				err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
+				sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
 			}
 		}
 		if err != nil {
@@ -257,20 +271,76 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
 	if err := root.Close(); err != nil {
 		return "", err
 	}
-	return sha, os.Rename(temp, filepath.Join(dir, sha))
+	return sha, installTree(ctx, dir, sha, temp, sizes)
 }
 
-func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
+var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
+
+// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
+func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
+	files := make([]treeFile, 0, len(sizes))
+	for name, size := range sizes {
+		files = append(files, treeFile{[]byte(name), size})
+	}
+	data, err := json.Marshal(files)
+	if err != nil {
+		return err
+	}
+	// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
+	if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
+		return err
+	}
+	tree := filepath.Join(dir, sha)
+	if !exists(tree) {
+		return os.Rename(temp, tree)
+	}
+	stale, err := os.MkdirTemp(dir, "tmp-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(stale)          // after the unlock, as no job reads the old tree once it is moved
+	unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
+	if err != nil {
+		return err
+	}
+	defer unlock()
+	if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
+		return err
+	}
+	return os.Rename(temp, tree)
+}
+
+// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
+// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
+func treeIntact(dir, sha string) bool {
+	data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
+	if err != nil || !exists(filepath.Join(dir, sha)) {
+		return false
+	}
+	var files []treeFile
+	if json.Unmarshal(data, &files) != nil {
+		return false
+	}
+	for _, file := range files {
+		info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
+		if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
+			return false
+		}
+	}
+	return true
+}
+
+func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
 	perm := os.FileMode(0o644)
 	if mode&0o111 != 0 {
 		perm = 0o755
 	}
 	file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
 	if err != nil {
-		return err
+		return 0, err
 	}
-	_, err = io.Copy(file, content)
-	return errors.Join(err, file.Close())
+	n, err := io.Copy(file, content)
+	return n, errors.Join(err, file.Close())
 }
 
 func exists(name string) bool {

act/common/git/fetch_action_test.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -206,10 +207,17 @@ func TestFetchAction(t *testing.T) {
 		_, offlineSha, err := FetchAction(t.Context(), input)
 		require.NoError(t, err)
 		assert.Equal(t, movedSha, offlineSha)
+		require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
+		_, _, err = FetchAction(t.Context(), input)
+		require.NoError(t, err)
+		require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
+		reused, _, err := FetchAction(t.Context(), input)
+		require.NoError(t, err)
+		assert.Equal(t, moved, reused)
 
 		requests, headers := served()
-		assert.Equal(t, 3, requests)
-		require.Len(t, headers, 3)
+		assert.Equal(t, 4, requests)
+		require.Len(t, headers, 4)
 		assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
 		assert.Empty(t, headers[0].Get("If-None-Match"))
 		assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
@@ -233,6 +241,68 @@ func TestFetchAction(t *testing.T) {
 		assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
 	})
 
+	t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
+		sha := gitRevParse(t, remote, "main")
+		for _, mode := range []string{"prefixed", "none"} {
+			serve(mode, 0)
+			input := newInput(sha)
+			tree, _, err := FetchAction(t.Context(), input)
+			require.NoError(t, err, mode)
+			yml := filepath.Join(tree, "action.yml")
+			require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
+			_, _, err = FetchAction(t.Context(), input)
+			require.NoError(t, err, mode)
+			for _, damage := range []func() error{
+				func() error { return os.Truncate(yml, 4) },
+				func() error { return os.Remove(tree + treeSizes) },
+			} {
+				require.NoError(t, damage(), mode)
+				_, _, err := FetchAction(t.Context(), input)
+				require.NoError(t, err, mode)
+				body, err := os.ReadFile(yml)
+				require.NoError(t, err, mode)
+				assert.Equal(t, "name: action\n", string(body), mode)
+			}
+			_, headers := served()
+			assert.Len(t, headers, 3, mode)
+		}
+
+		serve("prefixed", 0)
+		input := newInput(sha)
+		tree, _, err := FetchAction(t.Context(), input)
+		require.NoError(t, err)
+		yml := filepath.Join(tree, "action.yml")
+		require.NoError(t, os.Truncate(yml, 4))
+		unlock, err := AcquireCloneLock(t.Context(), tree)
+		require.NoError(t, err)
+		release := sync.OnceFunc(unlock)
+		defer release()
+		waiting := make(chan struct{})
+		lockTree = func(ctx context.Context, dir string) (func(), error) {
+			close(waiting)
+			return AcquireCloneLock(ctx, dir)
+		}
+		t.Cleanup(func() { lockTree = AcquireCloneLock })
+		done := make(chan error, 1)
+		go func() {
+			_, _, err := FetchAction(t.Context(), input)
+			done <- err
+		}()
+		select {
+		case err := <-done:
+			t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
+		case <-waiting:
+		}
+		info, err := os.Stat(yml)
+		require.NoError(t, err)
+		assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
+		release()
+		require.NoError(t, <-done)
+		body, err := os.ReadFile(yml)
+		require.NoError(t, err)
+		assert.Equal(t, "name: action\n", string(body))
+	})
+
 	t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
 		serve("redirect", 2)
 		input := newInput("main")
@@ -306,10 +376,30 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
 			require.NoError(t, writer.Close())
 
 			dir := t.TempDir()
-			_, err := extractTree(&archive, dir, false)
+			_, err := extractTree(t.Context(), &archive, dir, false)
 			require.Error(t, err)
 			assert.NoFileExists(t, filepath.Join(dir, "escape"))
 			assert.NoFileExists(t, filepath.Join(outside, "escape"))
 		})
 	}
 }
+
+func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("macOS and Windows store no name that is not UTF-8")
+	}
+	sha := strings.Repeat("a", 40)
+	var archive bytes.Buffer
+	writer := tar.NewWriter(&archive)
+	require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
+	require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
+	_, err := writer.Write([]byte("x"))
+	require.NoError(t, err)
+	require.NoError(t, writer.Close())
+
+	dir := t.TempDir()
+	_, err = extractTree(t.Context(), &archive, dir, false)
+	require.NoError(t, err)
+	require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
+	assert.True(t, treeIntact(dir, sha))
+}