enhance: test on Windows and fix what it found (#1348)

Adds a weekly and on-demand workflow that runs the cross-compiled tests on Windows Server 2025 Core in a [dockur/windows](https://github.com/dockur/windows) QEMU VM, and fixes the Windows bugs it found in host and UNC paths, env handling, shell quoting, post-task scripts, health checks, checkout, action symlinks, step summaries, Git client certificates and docker mode from a Windows host.

Host-mode steps now emit UTF-8, `bash` falls back to Git's bash instead of the WSL launcher, the runner can run as a native Windows service, steps can keep background helpers with `RUNNER_TRACKING_ID`, and Windows arm64 binaries are released.

Fixes: https://gitea.com/gitea/runner/issues/1339
Fixes: https://gitea.com/gitea/runner/issues/637
Fixes: https://gitea.com/gitea/runner/issues/1172
Fixes: https://gitea.com/gitea/runner/issues/347
Reviewed-on: https://gitea.com/gitea/runner/pulls/1348
Reviewed-by: bircni <[email protected]>

act/common/git/fetch_action.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"math/rand/v2"
 	"net"
@@ -29,6 +30,10 @@ import (
 
 var actionFetches sync.Map
 
+var symlink = (*os.Root).Symlink
+
+var symlinkFallback = runtime.GOOS == "windows"
+
 const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
 
 type treeFile struct {
@@ -229,6 +234,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
 
 	sha := ""
 	sizes := map[string]int64{}
+	links := map[string]string{}
+	var deferred []string
 	reader := tar.NewReader(archive)
 	for {
 		header, err := reader.Next()
@@ -257,9 +264,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
 		case tar.TypeReg:
 			sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
 		case tar.TypeSymlink:
-			if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
-				sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
-			}
+			links[filepath.Clean(name)] = header.Linkname
+			deferred = append(deferred, filepath.Clean(name))
 		}
 		if err != nil {
 			return "", err
@@ -268,12 +274,166 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
 	if !isFullObjectID(sha) {
 		return "", errors.New("the archive names no commit")
 	}
+	linksByParent := map[string][]string{}
+	if symlinkFallback {
+		for name := range links {
+			linksByParent[filepath.Dir(name)] = append(linksByParent[filepath.Dir(name)], filepath.Base(name))
+		}
+	}
+	for requireTarget := symlinkFallback; len(deferred) > 0; {
+		unresolved := deferred[:0]
+		for _, name := range deferred {
+			if err := ctx.Err(); err != nil {
+				return "", err
+			}
+			if requireTarget {
+				if _, err := root.Stat(filepath.Dir(name) + string(filepath.Separator) + links[name]); err != nil {
+					unresolved = append(unresolved, name)
+					continue
+				}
+			}
+			if err := symlink(root, links[name], name); err == nil || !symlinkFallback {
+				if err != nil {
+					return "", err
+				}
+				continue
+			}
+			if err := copyTreeLink(ctx, root, name, links, linksByParent, sizes); err != nil {
+				return "", fmt.Errorf("materialize symlink %q: %w", filepath.ToSlash(name), err)
+			}
+		}
+		if len(unresolved) == len(deferred) {
+			requireTarget = false
+		}
+		deferred = unresolved
+	}
 	if err := root.Close(); err != nil {
 		return "", err
 	}
 	return sha, installTree(ctx, dir, sha, temp, sizes)
 }
 
+func resolveTreeLink(name string, links map[string]string) (string, bool) {
+	if !filepath.IsLocal(name) {
+		return "", false
+	}
+	parts := strings.Split(name, string(filepath.Separator))
+	var resolved []string
+	for count := 0; len(parts) > 0; {
+		part := parts[0]
+		parts = parts[1:]
+		switch part {
+		case "", ".":
+			continue
+		case "..":
+			if len(resolved) == 0 {
+				return "", false
+			}
+			resolved = resolved[:len(resolved)-1]
+			continue
+		}
+		target, ok := links[filepath.Join(append(resolved, part)...)]
+		if !ok {
+			resolved = append(resolved, part)
+			continue
+		}
+		count++
+		if target == "" || os.IsPathSeparator(target[0]) || filepath.VolumeName(target) != "" || count > 255 {
+			return "", false
+		}
+		parts = append(strings.Split(filepath.FromSlash(target), string(filepath.Separator)), parts...)
+	}
+	return filepath.Join(append([]string{"."}, resolved...)...), true
+}
+
+var (
+	errTreeLinkTarget = errors.New("unmaterializable symlink target")
+	errTreeLinkCycle  = fmt.Errorf("cyclic %w", errTreeLinkTarget)
+)
+
+func copyTreeLink(ctx context.Context, root *os.Root, name string, links map[string]string, linksByParent map[string][]string, sizes map[string]int64) error {
+	writeLinkText := func(destination, target string) error { // as git writes links it cannot create
+		if err := root.RemoveAll(destination); err != nil {
+			return err
+		}
+		for written := range sizes {
+			if written == destination || strings.HasPrefix(written, destination+string(filepath.Separator)) {
+				delete(sizes, written)
+			}
+		}
+		size, err := writeTreeFile(root, destination, 0o644, strings.NewReader(target))
+		sizes[destination] = size
+		return err
+	}
+	active := map[string]bool{}
+	var copyTarget func(string, string) error
+	copyTarget = func(source, destination string) error {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		source, ok := resolveTreeLink(source, links)
+		if !ok {
+			return errTreeLinkTarget
+		}
+		file, err := root.Open(source)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				return errTreeLinkTarget
+			}
+			return err
+		}
+		defer file.Close()
+		info, err := file.Stat()
+		if err != nil {
+			return err
+		}
+		if info.Mode().IsRegular() {
+			sizes[destination], err = writeTreeFile(root, destination, int64(info.Mode().Perm()), file)
+			return err
+		}
+		if !info.IsDir() {
+			return errTreeLinkTarget
+		}
+		if active[source] {
+			return errTreeLinkCycle
+		}
+		active[source] = true
+		defer delete(active, source)
+		entries, err := file.ReadDir(-1)
+		if err != nil {
+			return err
+		}
+		children := map[string]bool{}
+		for _, entry := range entries {
+			children[entry.Name()] = true
+		}
+		for _, child := range linksByParent[source] {
+			children[child] = true
+		}
+		if err := root.MkdirAll(destination, info.Mode().Perm()); err != nil {
+			return err
+		}
+		for child := range children {
+			childSource, childDestination := filepath.Join(source, child), filepath.Join(destination, child)
+			err := copyTarget(childSource, childDestination)
+			if target, isLink := links[childSource]; isLink && errors.Is(err, errTreeLinkTarget) && !errors.Is(err, errTreeLinkCycle) {
+				err = writeLinkText(childDestination, target)
+			}
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	if err := copyTarget(name, name); !errors.Is(err, errTreeLinkTarget) {
+		return err
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return writeLinkText(name, links[name])
+}
+
 var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
 
 // installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.

act/common/git/fetch_action_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/cgi"
@@ -40,6 +41,7 @@ func TestFetchAction(t *testing.T) {
 	require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
 	require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
 	require.NoError(t, gitCmd("-C", remote, "add", "."))
+	require.NoError(t, gitCmd("-C", remote, "update-index", "--chmod=+x", "run.sh"))
 	require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
 	require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
 	expectedSha := gitRevParse(t, remote, "main")
@@ -192,7 +194,9 @@ func TestFetchAction(t *testing.T) {
 		assert.NoDirExists(t, filepath.Join(tree, ".git"))
 		info, err := os.Stat(filepath.Join(tree, "run.sh"))
 		require.NoError(t, err)
-		assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
+		if runtime.GOOS != "windows" {
+			assert.EqualValues(t, 0o755, info.Mode().Perm())
+		}
 		link, err := os.Readlink(filepath.Join(tree, "link"))
 		require.NoError(t, err)
 		assert.Equal(t, "action.yml", link)
@@ -391,6 +395,79 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
 	}
 }
 
+func TestExtractTreeCopiesDeferredLinks(t *testing.T) {
+	originalSymlink, originalFallback := symlink, symlinkFallback
+	symlink = func(_root *os.Root, _target, _name string) error { return errors.New("symlink privilege unavailable") }
+	symlinkFallback = true
+	t.Cleanup(func() { symlink, symlinkFallback = originalSymlink, originalFallback })
+
+	for _, test := range []struct {
+		name  string
+		links [][2]string
+		files []string
+		text  bool
+	}{
+		{name: "file chain", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "dist/index.js"}}, files: []string{"index.js", "entry.js"}},
+		{name: "directory chain", links: [][2]string{{"lib", "bundle"}, {"bundle", "dist"}, {"dist/entry.js", "index.js"}}, files: []string{"lib/index.js", "lib/entry.js", "lib/nested/index.js", "bundle/index.js", "bundle/entry.js", "bundle/nested/index.js"}},
+		{name: "directory in target path", links: [][2]string{{"index.js", "bundle/nested/index.js"}, {"bundle", "dist"}}, files: []string{"index.js", "bundle/nested/index.js"}},
+		{name: "parent after directory link", links: [][2]string{{"index.js", "bundle/../index.js"}, {"bundle", "dist/nested"}}, files: []string{"index.js", "bundle/index.js"}},
+		{name: "outside", links: [][2]string{{"index.js", "../outside.js"}}, text: true},
+		{name: "absolute", links: [][2]string{{"index.js", t.TempDir()}}, text: true},
+		{name: "dangling", links: [][2]string{{"index.js", "missing.js"}}, text: true},
+		{name: "dangling child of directory link", links: [][2]string{{"lib", "dist"}, {"dist/optional", "missing.js"}}, files: []string{"lib/index.js", "lib/nested/index.js"}},
+		{name: "cycle", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "index.js"}}, text: true},
+		{name: "directory cycle", links: [][2]string{{"dist/loop", "."}}, text: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			sha := strings.Repeat("a", 40)
+			var archive bytes.Buffer
+			writer := tar.NewWriter(&archive)
+			require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
+			require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755}))
+			require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist/nested", Typeflag: tar.TypeDir, Mode: 0o755}))
+			for _, link := range test.links {
+				require.NoError(t, writer.WriteHeader(&tar.Header{Name: link[0], Typeflag: tar.TypeSymlink, Linkname: link[1]}))
+			}
+			content := "console.log('action');\n"
+			for _, name := range []string{"dist/index.js", "dist/nested/index.js"} {
+				require.NoError(t, writer.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(content))}))
+				_, err := writer.Write([]byte(content))
+				require.NoError(t, err)
+			}
+			require.NoError(t, writer.Close())
+
+			dir := t.TempDir()
+			_, err := extractTree(t.Context(), &archive, dir, false)
+			require.NoError(t, err)
+			assert.True(t, treeIntact(dir, sha))
+			if test.text {
+				for _, link := range test.links {
+					data, err := os.ReadFile(filepath.Join(dir, sha, link[0]))
+					require.NoError(t, err)
+					assert.Equal(t, link[1], string(data))
+					info, err := os.Lstat(filepath.Join(dir, sha, link[0]))
+					require.NoError(t, err)
+					assert.True(t, info.Mode().IsRegular())
+				}
+			}
+			for _, name := range test.files {
+				data, err := os.ReadFile(filepath.Join(dir, sha, name))
+				require.NoError(t, err)
+				assert.Equal(t, content, string(data))
+				info, err := os.Lstat(filepath.Join(dir, sha, name))
+				require.NoError(t, err)
+				assert.True(t, info.Mode().IsRegular())
+				if runtime.GOOS != "windows" {
+					assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
+				}
+				require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), nil, 0o644))
+				assert.False(t, treeIntact(dir, sha), name)
+				require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), []byte(content), 0o755))
+			}
+		})
+	}
+}
+
 func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
 	if runtime.GOOS != "linux" {
 		t.Skip("macOS and Windows store no name that is not UTF-8")
@@ -410,3 +487,76 @@ func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
 	require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
 	assert.True(t, treeIntact(dir, sha))
 }
+
+func TestExtractTreeCreatesLinksInPlatformOrder(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("requires symlink privileges")
+	}
+	original, originalFallback := symlink, symlinkFallback
+	var created []string
+	symlink = func(root *os.Root, target, name string) error {
+		created = append(created, name)
+		return root.Symlink(target, name)
+	}
+	t.Cleanup(func() { symlink, symlinkFallback = original, originalFallback })
+
+	var archive bytes.Buffer
+	writer := tar.NewWriter(&archive)
+	require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
+	for _, entry := range []*tar.Header{
+		{Name: "a", Typeflag: tar.TypeSymlink, Linkname: "b"},
+		{Name: "b", Typeflag: tar.TypeSymlink, Linkname: "dist"},
+		{Name: "dangling", Typeflag: tar.TypeSymlink, Linkname: "missing"},
+		{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755},
+	} {
+		require.NoError(t, writer.WriteHeader(entry))
+	}
+	require.NoError(t, writer.Close())
+	for _, test := range []struct {
+		name     string
+		fallback bool
+		want     []string
+	}{
+		{name: "native", want: []string{"a", "b", "dangling"}},
+		{name: "fallback", fallback: true, want: []string{"b", "a", "dangling"}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			symlinkFallback = test.fallback
+			created = nil
+			_, err := extractTree(t.Context(), bytes.NewReader(archive.Bytes()), t.TempDir(), false)
+			require.NoError(t, err)
+			assert.Equal(t, test.want, created)
+		})
+	}
+}
+
+func TestCopyTreeLinkCancellationBetweenEntries(t *testing.T) {
+	root, err := os.OpenRoot(t.TempDir())
+	require.NoError(t, err)
+	defer root.Close()
+	require.NoError(t, root.Mkdir("source", 0o755))
+	for _, name := range []string{"first", "second"} {
+		_, err := writeTreeFile(root, filepath.Join("source", name), 0o644, strings.NewReader("content"))
+		require.NoError(t, err)
+	}
+	sizes := map[string]int64{}
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	observed := &treeCopyContext{Context: ctx, check: func() {
+		if len(sizes) == 1 {
+			cancel()
+		}
+	}}
+	require.ErrorIs(t, copyTreeLink(observed, root, "copy", map[string]string{"copy": "source"}, nil, sizes), context.Canceled)
+	assert.Len(t, sizes, 1)
+}
+
+type treeCopyContext struct {
+	context.Context
+	check func()
+}
+
+func (ctx *treeCopyContext) Err() error {
+	ctx.check()
+	return ctx.Context.Err()
+}

act/common/git/git.go

@@ -5,6 +5,7 @@
 package git
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
@@ -27,6 +28,7 @@ import (
 	"gitea.com/gitea/runner/act/common"
 	"gitea.com/gitea/runner/act/common/logging"
 	"gitea.com/gitea/runner/internal/pkg/lock"
+	"gitea.com/gitea/runner/internal/pkg/process"
 )
 
 var (
@@ -387,6 +389,17 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
 	return config
 }
 
+func ClientCertConfig(prefix, certFile, keyFile, executionOS string) []string {
+	if certFile == "" {
+		return nil
+	}
+	config := []string{prefix + "sslCert=" + certFile, prefix + "sslKey=" + keyFile}
+	if strings.EqualFold(executionOS, "windows") {
+		config = append(config, prefix+"sslBackend=openssl")
+	}
+	return config
+}
+
 // ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
 func ConfigEnv(config ...string) map[string]string {
 	env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
@@ -444,9 +457,7 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
 	config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
 	if input != nil {
 		config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
-		if input.ClientCertFile != "" {
-			config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
-		}
+		config = append(config, ClientCertConfig("http.", input.ClientCertFile, input.ClientKeyFile, runtime.GOOS)...)
 	}
 	if dir != "" {
 		args = append([]string{"-C", dir}, args...)
@@ -461,15 +472,28 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
 		cmd.Env = append(cmd.Env, key+"="+value)
 	}
 	cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
-	if runtime.GOOS != "windows" {
+	var treeKill *process.TreeKill
+	if runtime.GOOS == "windows" {
+		treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
+	} else {
 		cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
 	}
 	cmd.WaitDelay = time.Second
+	var stdout bytes.Buffer
 	var stderr strings.Builder
+	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
-	output, err := cmd.Output()
-	if err != nil {
-		return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
+	err := cmd.Start()
+	if err == nil {
+		if treeKill != nil {
+			if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
+				defer killer.Close()
+			}
+		}
+		err = cmd.Wait()
 	}
-	return output, nil
+	if err != nil {
+		return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
+	}
+	return stdout.Bytes(), nil
 }

act/common/git/git_test.go

@@ -54,6 +54,22 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
+func TestClientCertConfig(t *testing.T) {
+	for _, testcase := range []struct {
+		prefix, certFile, executionOS string
+		expected                      []string
+	}{
+		{"http.", "cert.pem", "windows", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslBackend=openssl"}},
+		{"http.https://example.com/.", "cert.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslBackend=openssl"}},
+		{"http.", "cert.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem"}},
+		{"http.", "", "Windows", nil},
+	} {
+		t.Run(testcase.prefix+testcase.certFile+testcase.executionOS, func(t *testing.T) {
+			assert.Equal(t, testcase.expected, ClientCertConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.executionOS))
+		})
+	}
+}
+
 func TestFindGitSlug(t *testing.T) {
 	for _, testcase := range []struct {
 		remoteURL string
@@ -185,7 +201,8 @@ func TestGitFindRef(t *testing.T) {
 func TestGitCloneExecutor(t *testing.T) {
 	remoteDir, workDir := newGitRemote(t)
 	require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
-	require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
+	require.NoError(t, os.WriteFile(filepath.Join(workDir, ".gitattributes"), []byte("*.sh text\n"), 0o644))
+	require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
 	require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
 	require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
 	require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
@@ -200,7 +217,7 @@ func TestGitCloneExecutor(t *testing.T) {
 	require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
 
 	hostConfig := filepath.Join(t.TempDir(), "gitconfig")
-	require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
+	require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
 	t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
 	t.Setenv("GIT_DIR", t.TempDir())
 
@@ -676,8 +693,8 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
 
 	select {
 	case <-reached:
-	case <-time.After(10 * time.Second):
-		t.Fatal("the executor never reached the remote")
+	case err := <-done:
+		t.Fatalf("the executor finished before reaching the remote: %v", err)
 	}
 	assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
 	cancel()