From 27e9b4de87767a9177e7c75e1f0f5c34bba0dc30 Mon Sep 17 00:00:00 2001 From: bircni Date: Thu, 8 Oct 2026 17:08:29 +0000 Subject: [PATCH] fix!: use the git CLI for action clones and make builtin:checkout match actions/checkout (#1277) The runner now uses the `git` command line instead of go-git to download actions and reusable workflows. go-git had recurring bugs and does not support SHA-256 repositories, and full clones of actions are several times faster. `builtin:checkout` now runs `git` where the job runs and behaves like `actions/checkout`: 1. A branch is checked out as a local branch tracking `origin`, tag and pull request runs keep their ref, and a triggering tag that moved fails the step. 1. Later steps can fetch and push with the persisted token or SSH key. 1. It supports the `ssh-*`, `lfs`, `submodules`, `sparse-checkout`, `filter`, `fetch-tags`, `clean`, `persist-credentials`, `show-progress` and `set-safe-directory` inputs and the `ref` and `commit` outputs. 1. SHA-256 repositories work for actions, reusable workflows and `builtin:checkout`. Git keeps the token off its command line, ignores `GIT_DIR` and similar variables inherited by the runner, and keeps action files LF regardless of `core.autocrlf`. A cancelled job lets git remove its lock files. This is a breaking change: the runner needs Git 2.34.1 or newer on its `PATH`, and so do job images that use `builtin:checkout`. The official runner images include it. Fixes: https://gitea.com/gitea/runner/issues/1077 Fixes: https://gitea.com/gitea/runner/issues/1267 Fixes: https://gitea.com/gitea/runner/issues/1279 Fixes: https://gitea.com/gitea/runner/issues/1284 Fixes: https://gitea.com/gitea/runner/issues/1285 Co-authored-by: silverwind Co-authored-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: Lunny Xiao Reviewed-on: https://gitea.com/gitea/runner/pulls/1277 Reviewed-by: Lunny Xiao Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: bircni --- README.md | 14 +- act/common/git/git.go | 580 +++++++----------- act/common/git/git_test.go | 276 ++++----- act/runner/step_action_builtin.go | 2 + act/runner/step_action_remote.go | 17 +- docs/builtin-actions.md | 26 +- e2e/checkout_test.go | 22 + e2e/compatibility_test.go | 1 + e2e/gitea_api.go | 6 + e2e/gitea_fixture.go | 1 + e2e/main_test.go | 7 +- e2e/testdata/workflows/checkout.yml | 61 ++ internal/pkg/action/action.go | 1 + internal/pkg/action/checkout/checkout.go | 416 ++++++++++++- internal/pkg/action/checkout/checkout_test.go | 333 ++++++++-- 15 files changed, 1111 insertions(+), 652 deletions(-) create mode 100644 e2e/checkout_test.go create mode 100644 e2e/testdata/workflows/checkout.yml diff --git a/README.md b/README.md index 676d47f9..5bd1a72d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,19 @@ ### Prerequisites -Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/). +On the runner: + +- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows. +- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/). + +Where steps run, in the job image or on the runner in host mode: + +- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host. +- `node` for JavaScript actions and `hashFiles()`. +- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md). +- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used. + +Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images. ### Download pre-built binary diff --git a/act/common/git/git.go b/act/common/git/git.go index 1f7e6294..2ad07ff9 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -6,28 +6,23 @@ package git import ( "context" + "crypto/sha256" + "encoding/base64" "errors" "fmt" - "io" - nethttp "net/http" "os" - "path" + "os/exec" + "path/filepath" "regexp" + "runtime" + "slices" + "strconv" "strings" - "sync" "time" "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/pkg/lock" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/storer" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/mattn/go-isatty" log "github.com/sirupsen/logrus" ) @@ -36,26 +31,19 @@ var ( codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`) githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`) githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`) + hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`) + + LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY", + "GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS", + "GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR", + } cloneLocks lock.Keyed[string] // key: clone target directory ErrShortRef = errors.New("short SHA references are not supported") ) -func init() { - httpClient := newHTTPClient(&nethttp.HTTP2Config{SendPingTimeout: 30 * time.Second}) - client.InstallProtocol("http", httpClient) - client.InstallProtocol("https", httpClient) -} - -// newHTTPClient pings idle HTTP/2 connections to drop stalled ones. -func newHTTPClient(h2 *nethttp.HTTP2Config) transport.Transport { - defaultTransport, _ := nethttp.DefaultTransport.(*nethttp.Transport) - httpTransport := defaultTransport.Clone() - httpTransport.HTTP2 = h2 - return http.NewClient(&nethttp.Client{Transport: httpTransport}) -} - // AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir. // Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel. // Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too, @@ -69,146 +57,60 @@ type Error struct { commit string } -func (e *Error) Error() string { - return e.err.Error() -} - -func (e *Error) Unwrap() error { - return e.err -} - -func (e *Error) Commit() string { - return e.commit -} - -// goGitMu serializes go-git repository access across the process. go-git is not safe for -// concurrent use of the same repository (even read access decodes packfiles into shared -// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded -// operations are fast local reads; gitea runs one job per process, so the lock is effectively -// uncontended in production. -var goGitMu sync.Mutex +func (e *Error) Error() string { return e.err.Error() } +func (e *Error) Unwrap() error { return e.err } +func (e *Error) Commit() string { return e.commit } // FindGitRevision get the current git revision -func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) { - goGitMu.Lock() - defer goGitMu.Unlock() - return findGitRevision(ctx, file) -} - -func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) { +func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) { logger := common.Logger(ctx) - - gitDir, err := git.PlainOpenWithOptions( - file, - &git.PlainOpenOptions{ - DetectDotGit: true, - EnableDotGitCommonDir: true, - }, - ) + sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}") if err != nil { logger.WithError(err).Error("path", file, "not located inside a git repository") return "", "", err } - - head, err := gitDir.Reference(plumbing.HEAD, true) - if err != nil { - return "", "", err + if len(sha) < 7 { + return "", "", errors.New("HEAD could not be resolved") } - - if head.Hash().IsZero() { - return "", "", errors.New("HEAD sha1 could not be resolved") - } - - hash := head.Hash().String() - - logger.Debugf("Found revision: %s", hash) - return hash[:7], strings.TrimSpace(hash), nil + logger.Debugf("Found revision: %s", sha) + return sha[:7], sha, nil } // FindGitRef get the current git ref func FindGitRef(ctx context.Context, file string) (string, error) { - goGitMu.Lock() - defer goGitMu.Unlock() - logger := common.Logger(ctx) - logger.Debugf("Loading revision from git directory") - _, ref, err := findGitRevision(ctx, file) + _, revision, err := FindGitRevision(ctx, file) if err != nil { return "", err } + logger.Debugf("HEAD points to '%s'", revision) - logger.Debugf("HEAD points to '%s'", ref) - - // Prefer the git library to iterate over the references and find a matching tag or branch. - refTag := "" - refBranch := "" - repo, err := git.PlainOpenWithOptions( - file, - &git.PlainOpenOptions{ - DetectDotGit: true, - EnableDotGitCommonDir: true, - }, - ) + refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads") if err != nil { return "", err } - - iter, err := repo.References() - if err != nil { - return "", err - } - - // find the reference that matches the revision's has - err = iter.ForEach(func(r *plumbing.Reference) error { - /* tags and branches will have the same hash - * when a user checks out a tag, it is not mentioned explicitly - * in the go-git package, we must identify the revision - * then check if any tag matches that revision, - * if so then we checked out a tag - * else we look for branches and if matches, - * it means we checked out a branch - * - * If a branches matches first we must continue and check all tags (all references) - * in case we match with a tag later in the interation - */ - if r.Hash().String() == ref { - if r.Name().IsTag() { - refTag = r.Name().String() - } - if r.Name().IsBranch() { - refBranch = r.Name().String() - } + var tag, branch string + for ref := range strings.FieldsSeq(refs) { + switch { + case strings.HasPrefix(ref, "refs/tags/"): + tag = ref + case strings.HasPrefix(ref, "refs/heads/"): + branch = ref } - - // we found what we where looking for - if refTag != "" && refBranch != "" { - return storer.ErrStop - } - - return nil - }) - if err != nil { - return "", err } - - // order matters here see above comment. - if refTag != "" { - return refTag, nil + if tag != "" { + return tag, nil } - if refBranch != "" { - return refBranch, nil + if branch != "" { + return branch, nil } - - return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref) + return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision) } // FindGithubRepo get the repo func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) { - goGitMu.Lock() - defer goGitMu.Unlock() - - url, err := findGitRemoteURL(ctx, file, "origin") + url, err := findGitRemoteURL(ctx, file) if err != nil { return "", err } @@ -216,28 +118,16 @@ func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, e return slug, nil } -func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) { - repo, err := git.PlainOpenWithOptions( - file, - &git.PlainOpenOptions{ - DetectDotGit: true, - EnableDotGitCommonDir: true, - }, - ) +func findGitRemoteURL(ctx context.Context, file string) (string, error) { + urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf if err != nil { return "", err } - - remote, err := repo.Remote(remoteName) - if err != nil { - return "", err + if urls == "" { + return "", errors.New("remote 'origin' exists but has no URL") } - - if len(remote.Config().URLs) < 1 { - return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName) - } - - return remote.Config().URLs[0], nil + url, _, _ := strings.Cut(urls, "\n") + return url, nil } func findGitSlug(url, githubInstance string) (string, string) { @@ -281,66 +171,31 @@ type NewGitCloneExecutorInput struct { InsecureSkipTLS bool } -// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused. -func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) { - r, err := git.PlainOpen(input.Dir) +// CloneIfRequired reports whether an existing local clone was reused. +func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) { + var origin string + _, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir if err == nil { - // Verify the cached clone still points to the resolved URL before reusing it. - remote, err := r.Remote("origin") - if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL { - // Reuse existing clone - return r, true, nil - } - - switch { - case err != nil: - logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err) - case len(remote.Config().URLs) == 0: - logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir) - default: - logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL) - } - if err := os.RemoveAll(input.Dir); err != nil { - return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err) - } + origin, err = findGitRemoteURL(ctx, input.Dir) } - - var progressWriter io.Writer - if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) { - if entry, ok := logger.(*log.Entry); ok { - progressWriter = entry.WriterLevel(log.DebugLevel) - } else if lgr, ok := logger.(*log.Logger); ok { - progressWriter = lgr.WriterLevel(log.DebugLevel) - } else { - log.Errorf("Unable to get writer from logger (type=%T)", logger) - progressWriter = os.Stdout - } + if err == nil && origin == input.URL { + return true, nil } - - cloneOptions := git.CloneOptions{ - URL: input.URL, - Progress: progressWriter, - - InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea + if err == nil { + logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL) + } else if _, statErr := os.Stat(input.Dir); statErr == nil { + logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err) } - if input.Token != "" { - cloneOptions.Auth = &http.BasicAuth{ - Username: "token", - Password: input.Token, - } + if err := os.RemoveAll(input.Dir); err != nil { + return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err) } - - r, err = cloneAtDepth(ctx, input, cloneOptions, logger) - if err != nil { - logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err) - return nil, false, err + if err := clone(ctx, input); err != nil { + return false, err } - - if err = os.Chmod(input.Dir, 0o755); err != nil { - return nil, false, err + if err := os.Chmod(input.Dir, 0o755); err != nil { + return false, err } - - return r, false, nil + return false, nil } // NewGitCloneExecutor creates an executor to clone git repos @@ -353,134 +208,125 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref) } logger.Debugf(" cloning %s to %s", input.URL, input.Dir) - defer AcquireCloneLock(input.Dir)() - refName := plumbing.ReferenceName("refs/heads/" + input.Ref) - r, reused, err := CloneIfRequired(ctx, refName, input, logger) + reused, err := CloneIfRequired(ctx, input, logger) if err != nil { return err } - - isOfflineMode := input.OfflineMode - - fetchOptions := git.FetchOptions{ - RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD", "refs/heads/*:refs/remotes/origin/*"}, - Force: true, - InsecureSkipTLS: input.InsecureSkipTLS, + resolved, err := resolveRef(ctx, input.Dir, input.Ref) + if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) { + return err } - if input.Token != "" { - fetchOptions.Auth = &http.BasicAuth{ - Username: "token", - Password: input.Token, - } - } - - // Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth - // Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone. - if isShallow(r) { - fetchOptions.Depth = 1 - if spec, ok := shallowFetchRefSpec(r, input.Ref); ok { - fetchOptions.RefSpecs = []config.RefSpec{spec} - } - } - - // A just-cloned ref is as current as a fetch would make it, and a commit hash never moves. - // TODO: revalidate a mutable ref with a conditional archive request instead, once every - // supported Gitea sends an ETag for them: https://github.com/go-gitea/gitea/pull/39289 - _, present := refRevision(r, input.Ref) - refresh := !isOfflineMode && (!present || (reused && !plumbing.IsHash(input.Ref))) - - if refresh { - err = r.FetchContext(ctx, &fetchOptions) - if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) { + if err := fetch(ctx, input); err != nil { return err } + resolved, err = resolveRef(ctx, input.Dir, input.Ref) } - - var hash *plumbing.Hash - rev := plumbing.Revision(input.Ref) - if hash, err = r.ResolveRevision(rev); err != nil { - // ResolveRevision returns a nil hash on error, and a branch ref legitimately fails - // here (no local refs/heads/); the duck-typing below resolves it. - logger.Errorf("Unable to resolve %s: %v", input.Ref, err) - } else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) { - return &Error{ - err: ErrShortRef, - commit: hash.String(), - } - } - - rev, _ = refRevision(r, input.Ref) - - if hash, err = r.ResolveRevision(rev); err != nil { + if err != nil { logger.Errorf("Unable to resolve %s: %v", input.Ref, err) return err } - - var w *git.Worktree - if w, err = r.Worktree(); err != nil { - return err + if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil { + return fmt.Errorf("checkout %s: %w", resolved, err) } - reusedMsg := "" - if isOfflineMode && reused { + if input.OfflineMode && reused { reusedMsg = " (reused in offline mode)" } - logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg) - - if err = w.Checkout(&git.CheckoutOptions{ - Hash: *hash, - Force: true, - }); err != nil { - logger.Errorf("Unable to checkout %s: %v", *hash, err) - return err - } - logger.Debugf("Checked out %s", input.Ref) return nil } } -// cloneAtDepth falls back to a full clone when the shallow attempt fails. -func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) { +func clone(ctx context.Context, input NewGitCloneExecutorInput) error { + if input.Depth > 0 && isFullObjectID(input.Ref) { + if err := shallowPinnedClone(ctx, input); err == nil { + return nil + } else if err := removePartialClone(input.Dir); err != nil { + return err + } + } + args := []string{"clone", "--no-checkout"} if input.Depth > 0 { - if plumbing.IsHash(input.Ref) { - r, err := fetchPinnedSHA(ctx, input, opts) - if err == nil { - return r, nil - } - logger.Debugf("Shallow fetch of %s at %s failed: %v", input.URL, input.Ref, err) - if err := removePartialClone(input.Dir); err != nil { - return nil, err - } - } else { - refNames := []plumbing.ReferenceName{plumbing.NewBranchReferenceName(input.Ref), plumbing.NewTagReferenceName(input.Ref)} - if input.Ref == plumbing.HEAD.String() || strings.HasPrefix(input.Ref, "refs/") { - refNames = []plumbing.ReferenceName{plumbing.ReferenceName(input.Ref)} // HEAD is the remote's default branch - } - for _, refName := range refNames { - shallowOpts := opts - shallowOpts.Depth = input.Depth - shallowOpts.SingleBranch = true - shallowOpts.ReferenceName = refName - shallowOpts.Tags = git.NoTags + args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags") + if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") { + args = append(args, "--branch", input.Ref) + } + } + args = append(args, input.URL, input.Dir) + if _, err := runGit(ctx, "", &input, args...); err == nil { + return nil + } else if input.Depth == 0 { + return err + } + if err := removePartialClone(input.Dir); err != nil { + return err + } + _, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir) + return err +} - r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts) - if err == nil { - return r, nil - } - logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err) - if err := removePartialClone(input.Dir); err != nil { - return nil, err - } +func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error { + if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil { + return err + } + if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil { + return err + } + _, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref) + return err +} + +func fetch(ctx context.Context, input NewGitCloneExecutorInput) error { + args := []string{"fetch", "--force", "--prune"} + if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil { + args = append(args, "--depth", "1", "--no-tags") + } + _, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch + return err +} + +func cachedRef(ref string) string { + return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref))) +} + +func resolveRef(ctx context.Context, dir, ref string) (string, error) { + if !isFullObjectID(ref) { + for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} { + if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil { + return sha, nil } } - logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref) } + sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}") + if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) { + return "", &Error{err: ErrShortRef, commit: sha} + } + return sha, err +} - return git.PlainCloneContext(ctx, input.Dir, false, &opts) +func isFullObjectID(ref string) bool { + return ObjectFormat(ref) != "" +} + +func isFullHash(ref, resolved string) bool { + return isFullObjectID(ref) && len(ref) == len(resolved) +} + +// ObjectFormat returns the object format of a full commit hash, empty for anything else. +func ObjectFormat(sha string) string { + switch { + case !hexRefRegex.MatchString(sha): + return "" + case len(sha) == 40: + return "sha1" + case len(sha) == 64: + return "sha256" + } + return "" } func removePartialClone(dir string) error { @@ -490,71 +336,59 @@ func removePartialClone(dir string) error { return nil } -func fetchPinnedSHA(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions) (*git.Repository, error) { - r, err := git.PlainInit(input.Dir, false) +func gitOutput(ctx context.Context, dir string, args ...string) (string, error) { + output, err := runGit(ctx, dir, nil, args...) + return strings.TrimSpace(string(output)), err +} + +func RemoteConfig(token string, insecureSkipTLS bool) []string { + var config []string + if insecureSkipTLS { + config = append(config, "http.sslVerify=false") + } + if token != "" { + config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token))) + } + return config +} + +// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv. +func ConfigEnv(config ...string) map[string]string { + env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))} + for i, entry := range config { + key, value, _ := strings.Cut(entry, "=") + env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key + env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value + } + return env +} + +func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) { + config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF + if input != nil { + config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...) + } + if dir != "" { + args = append([]string{"-C", dir}, args...) + } + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool { + name, _, _ := strings.Cut(entry, "=") + return slices.Contains(LocalEnvVars, name) + }) + for key, value := range ConfigEnv(config...) { + cmd.Env = append(cmd.Env, key+"="+value) + } + cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") + if runtime.GOOS != "windows" { + cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files + } + cmd.WaitDelay = time.Second + var stderr strings.Builder + cmd.Stderr = &stderr + output, err := cmd.Output() if err != nil { - return nil, err + return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) } - if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{input.URL}}); err != nil { - return nil, err - } - if err := r.FetchContext(ctx, &git.FetchOptions{ - RefSpecs: []config.RefSpec{pinnedRefSpec(input.Ref)}, - Depth: input.Depth, - Tags: git.NoTags, - Auth: opts.Auth, - Progress: opts.Progress, - InsecureSkipTLS: opts.InsecureSkipTLS, - }); err != nil { - return nil, err - } - return r, nil -} - -// pinnedRefSpec keeps a hash-shaped name out of refs/heads and refs/tags. -func pinnedRefSpec(sha string) config.RefSpec { - return config.RefSpec(fmt.Sprintf("+%s:refs/pinned/%s", sha, sha)) -} - -// refRevision picks the revision to check out and reports whether it resolves locally. -func refRevision(r *git.Repository, ref string) (plumbing.Revision, bool) { - if plumbing.IsHash(ref) { - // git ignores a ref named as 40 hex digits, so a full hash always denotes the commit itself. - _, err := r.CommitObject(plumbing.NewHash(ref)) - return plumbing.Revision(ref), err == nil - } - if _, err := r.Tag(ref); err == nil { - return plumbing.Revision(path.Join("refs", "tags", ref)), true - } - remoteRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", ref)) - if _, err := r.Reference(remoteRef, false); err == nil { - return plumbing.Revision(remoteRef), true - } - rev := plumbing.Revision(ref) - _, err := r.ResolveRevision(rev) - return rev, err == nil -} - -// isShallow reports whether the local repository was cloned with a limited depth. -func isShallow(r *git.Repository) bool { - shallows, err := r.Storer.Shallow() - return err == nil && len(shallows) > 0 -} - -// shallowFetchRefSpec limits a shallow update to the requested ref, falling back to the broad refspec when it is not present locally. -func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) { - tagRef := plumbing.NewTagReferenceName(ref) - if _, err := r.Reference(tagRef, false); err == nil { - return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true - } - remoteRef := plumbing.NewRemoteReferenceName("origin", ref) - if _, err := r.Reference(remoteRef, false); err == nil { - branchRef := plumbing.NewBranchReferenceName(ref) - return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true - } - if plumbing.IsHash(ref) { - // The broad refspec carries only advertised tips, never a pinned commit. - return pinnedRefSpec(ref), true - } - return "", false + return output, nil } diff --git a/act/common/git/git_test.go b/act/common/git/git_test.go index 4355c3a4..79ced67a 100644 --- a/act/common/git/git_test.go +++ b/act/common/git/git_test.go @@ -6,9 +6,8 @@ package git import ( "context" - "crypto/tls" + "encoding/base64" "fmt" - "io" "net/http" "net/http/httptest" "os" @@ -17,18 +16,12 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "syscall" "testing" "time" "gitea.com/gitea/runner/act/common" - gogit "github.com/go-git/go-git/v5" - gogitconfig "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing/transport" - gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client" - gogitfile "github.com/go-git/go-git/v5/plumbing/transport/file" log "github.com/sirupsen/logrus" logrustest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" @@ -90,14 +83,27 @@ func cleanGitHooks(dir string) error { return nil } -func TestFindGithubRepoUsesOrigin(t *testing.T) { +func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) { + t.Parallel() basedir := t.TempDir() const remoteURL = "https://github.com/owner/repo.git" - require.NoError(t, gitCmd("init", basedir)) + require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir)) require.NoError(t, cleanGitHooks(basedir)) + require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init")) require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL)) + require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git")) - url, err := findGitRemoteURL(context.Background(), basedir, "origin") + shortSHA, sha, err := FindGitRevision(context.Background(), basedir) + require.NoError(t, err) + require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha) + require.Len(t, sha, 64) + require.Equal(t, sha[:7], shortSHA) + + ref, err := FindGitRef(context.Background(), basedir) + require.NoError(t, err) + require.Equal(t, "refs/heads/main", ref) + + url, err := findGitRemoteURL(context.Background(), basedir) require.NoError(t, err) require.Equal(t, remoteURL, url) @@ -107,6 +113,7 @@ func TestFindGithubRepoUsesOrigin(t *testing.T) { } func TestGitFindRef(t *testing.T) { + t.Parallel() basedir := t.TempDir() for name, tt := range map[string]struct { @@ -193,7 +200,9 @@ func TestGitCloneExecutor(t *testing.T) { workDir := t.TempDir() require.NoError(t, gitCmd("clone", remoteDir, workDir)) require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main")) - require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial")) + require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755)) + require.NoError(t, gitCmd("-C", workDir, "add", "action.sh")) + require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial")) require.NoError(t, gitCmd("-C", workDir, "tag", "v2")) require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main")) require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2")) @@ -207,6 +216,13 @@ func TestGitCloneExecutor(t *testing.T) { require.NoError(t, err) fullSha := strings.TrimSpace(string(out)) + require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4])) + + hostConfig := filepath.Join(t.TempDir(), "gitconfig") + require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644)) + t.Setenv("GIT_CONFIG_GLOBAL", hostConfig) + t.Setenv("GIT_DIR", t.TempDir()) + for name, tt := range map[string]struct { Err error Ref string @@ -227,26 +243,42 @@ func TestGitCloneExecutor(t *testing.T) { Err: &Error{ErrShortRef, fullSha}, Ref: fullSha[:7], }, + "HEAD": { + Ref: "HEAD", + }, + "hex branch named like its own commit": { + Ref: fullSha[:4], + }, } { t.Run(name, func(t *testing.T) { + dir := t.TempDir() clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: remoteDir, + URL: "https://example.invalid/action", Ref: tt.Ref, - Dir: t.TempDir(), + Dir: dir, }) err := clone(context.Background()) if tt.Err != nil { assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act assert.Equal(t, tt.Err, err) - } else { - assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act + assert.Equal(t, tt.Err, clone(context.Background()), "a retry on the cache keeps the error") + return } + assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act + marker := filepath.Join(dir, "marker") + require.NoError(t, os.WriteFile(marker, nil, 0o644)) + require.NoError(t, clone(context.Background())) + assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache") + script, err := os.ReadFile(filepath.Join(dir, "action.sh")) + require.NoError(t, err) + assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files") }) } } func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) { + t.Parallel() createRemote := func(message string) string { remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) @@ -270,6 +302,11 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) { Dir: cacheDir, })(t.Context())) + nested := filepath.Join(cacheDir, "nested") + require.NoError(t, os.Mkdir(nested, 0o755)) + require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context())) + assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository") + markerPath := filepath.Join(cacheDir, "stale-marker") require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644)) @@ -279,7 +316,7 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) { Dir: cacheDir, })(t.Context())) - originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin") + originURL, err := findGitRemoteURL(t.Context(), cacheDir) require.NoError(t, err) assert.Equal(t, newRemoteDir, originURL) @@ -291,61 +328,36 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) { require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning") } -func TestGitCloneExecutorNonFastForwardRef(t *testing.T) { - // Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes - // non-fast-forward between two fetches. Before the fix, the fetch used Force=false, - // causing go-git to return ErrForceNeeded and short-circuit the checkout. - - // Create a bare "remote" repo with an initial commit on main and a feature branch. +func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) { + t.Parallel() remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) - - // We need a working clone to push commits from. workDir := t.TempDir() require.NoError(t, gitCmd("clone", remoteDir, workDir)) require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main")) require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial")) + require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1")) require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main")) - // Create a feature branch (simulates refs/pull/N/head). - require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature")) - require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1")) - require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature")) - - // First clone via the executor — should succeed and cache the repo. - cloneDir := t.TempDir() - clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: remoteDir, - Ref: "main", - Dir: cloneDir, - }) + dir := t.TempDir() + clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir}) require.NoError(t, clone(context.Background())) - // Now force-push the feature branch to a non-fast-forward commit (simulates - // a PR rebase). This makes refs/heads/feature non-fast-forward. - require.NoError(t, gitCmd("-C", workDir, "checkout", "main")) - require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature")) - require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature")) + require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1")) require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten")) - require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature")) + require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main")) + require.NoError(t, clone(context.Background()), "a non-fast-forward ref must update") + assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir)) - // Also advance main so we can verify the clone picks up the new commit. - require.NoError(t, gitCmd("-C", workDir, "checkout", "main")) - require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second")) - require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main")) - - // Second clone to the same directory — before the fix this returned ErrForceNeeded - // and left the working tree at the old commit. - err := clone(context.Background()) - require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true") - - // Verify the working tree was actually updated to the latest main commit. - out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output() - require.NoError(t, err) - assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit") + tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: t.TempDir()}) + require.Error(t, tagClone(context.Background())) + require.NoError(t, gitCmd("-C", workDir, "tag", "v9")) + require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9")) + require.NoError(t, tagClone(context.Background()), "a tag published after a failed lookup must be fetched") } func TestGitCloneExecutorOfflineMode(t *testing.T) { + t.Parallel() // Build a local "remote" with a single commit on main. remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) @@ -392,6 +404,7 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) { } func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) { + t.Parallel() remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) workDir := t.TempDir() @@ -430,6 +443,7 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) { } func TestGitCloneExecutorShallow(t *testing.T) { + t.Parallel() // Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one. remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) @@ -473,12 +487,11 @@ func TestGitCloneExecutorShallow(t *testing.T) { return remote } - t.Run("commit hash falls back to a full clone when the remote refuses unadvertised objects", func(t *testing.T) { + t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) { dir := t.TempDir() require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{ URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1, })(t.Context())) - assert.NoFileExists(t, shallowMarker(dir)) assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD")) }) @@ -505,76 +518,46 @@ func TestGitCloneExecutorShallow(t *testing.T) { assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD")) }) - t.Run("moving branch updates while staying shallow", func(t *testing.T) { + t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) { + require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef")) dir := t.TempDir() - require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: remoteDir, Ref: "main", Dir: dir, Depth: 1, - })(t.Context())) + clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ + URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1, + }) + require.NoError(t, clone(t.Context())) require.Equal(t, "c3", gitHeadSubject(t, dir)) - // Advance main on the remote, then reuse the existing shallow clone. require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4")) - require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main")) + require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef")) - require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: remoteDir, Ref: "main", Dir: dir, Depth: 1, - })(t.Context())) + require.NoError(t, clone(t.Context())) assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip") assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update") assert.Equal(t, 1, gitRevCount(t, dir)) }) -} -func TestGitCloneExecutorTransportSessions(t *testing.T) { - workDir := t.TempDir() - require.NoError(t, gitCmd("init", "--initial-branch=main", workDir)) - require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c1")) - require.NoError(t, gitCmd("-C", workDir, "tag", "v1")) - require.NoError(t, gitCmd("-C", workDir, "tag", "-a", "v2", "-m", "v2")) - - for name, tt := range map[string]struct { - Ref string - Depth int - }{ - "shallow branch": {"main", 1}, - "full clone branch": {"main", 0}, - "full lightweight tag": {"v1", 0}, - "full annotated tag": {"v2", 0}, - } { - t.Run(name, func(t *testing.T) { - counter := installCountingTransport(t) - dir := t.TempDir() - clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: workDir, Ref: tt.Ref, Dir: dir, Depth: tt.Depth, - }) - require.NoError(t, clone(t.Context())) - assert.Equal(t, int64(1), counter.sessions.Swap(0), "cold clone") - assert.Equal(t, gitRevParse(t, workDir, tt.Ref+"^{commit}"), gitRevParse(t, dir, "HEAD")) - - require.NoError(t, clone(t.Context())) - assert.Equal(t, int64(1), counter.sessions.Swap(0), "unchanged warm cache") - - require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.yml"), []byte(name), 0o644)) - require.NoError(t, gitCmd("-C", workDir, "add", "action.yml")) - require.NoError(t, gitCmd("-C", workDir, "commit", "-m", name)) - require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v1")) - require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "-a", "v2", "-m", "v2")) - - require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("staged"), 0o644)) - require.NoError(t, gitCmd("-C", dir, "add", "action.yml")) - require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("unstaged"), 0o644)) - - require.NoError(t, clone(t.Context())) - assert.Equal(t, int64(1), counter.sessions.Load(), "updated warm cache") - assert.Equal(t, gitRevParse(t, workDir, "HEAD"), gitRevParse(t, dir, "HEAD")) - status, err := exec.Command("git", "-C", dir, "status", "--porcelain").Output() - require.NoError(t, err) - assert.Empty(t, string(status)) - }) - } + t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) { + remote := filepath.Join(t.TempDir(), "remote.git") + require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote)) + work := t.TempDir() + require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work)) + require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote)) + require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned")) + require.NoError(t, gitCmd("-C", work, "push", "origin", "main")) + sha := gitRevParse(t, work, "HEAD") + dir := t.TempDir() + clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1}) + require.NoError(t, clone(t.Context())) + assert.FileExists(t, shallowMarker(dir)) + require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef) + require.NoError(t, os.RemoveAll(remote)) + require.NoError(t, clone(t.Context())) + assert.Equal(t, sha, gitRevParse(t, dir, "HEAD")) + }) } func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) { + t.Parallel() remoteDir := t.TempDir() require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir)) workDir := t.TempDir() @@ -598,25 +581,6 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) { } } -type countingTransport struct { - transport.Transport - sessions atomic.Int64 -} - -func (c *countingTransport) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { - c.sessions.Add(1) - return c.Transport.NewUploadPackSession(ep, auth) -} - -func installCountingTransport(t *testing.T) *countingTransport { - t.Helper() - counter := &countingTransport{Transport: gogitfile.DefaultClient} - gogitclient.InstallProtocol("file", counter) - t.Cleanup(func() { gogitclient.InstallProtocol("file", gogitfile.DefaultClient) }) - - return counter -} - func gitRevParse(t *testing.T, dir, rev string) string { t.Helper() out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output() @@ -717,12 +681,17 @@ func TestAcquireCloneLock(t *testing.T) { } // An unresponsive remote must not pin a job: the refresh has to be interruptible. -func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) { +func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) { + t.Parallel() block := make(chan struct{}) reached := make(chan struct{}) var once sync.Once - server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { - once.Do(func() { close(reached) }) + var authorization string + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + once.Do(func() { + authorization = request.Header.Get("Authorization") + close(reached) + }) <-block })) t.Cleanup(func() { @@ -731,17 +700,15 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) { }) dir := filepath.Join(t.TempDir(), "cached-action") - repo, err := gogit.PlainInit(dir, false) - require.NoError(t, err) - _, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}}) - require.NoError(t, err) + require.NoError(t, gitCmd("init", dir)) + require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL)) ctx, cancel := context.WithCancel(t.Context()) defer cancel() done := make(chan error, 1) go func() { - done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx) + done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx) }() select { @@ -749,6 +716,7 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) { case <-time.After(10 * time.Second): t.Fatal("the executor never reached the remote") } + assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization) cancel() select { @@ -758,25 +726,3 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) { t.Fatal("fetch ignored context cancellation") } } - -func TestNewGitCloneExecutorFailsOnSilentHTTP2Connection(t *testing.T) { - server := httptest.NewUnstartedServer(nil) - server.EnableHTTP2 = true - server.Config.TLSNextProto = map[string]func(*http.Server, *tls.Conn, http.Handler){ - "h2": func(_ *http.Server, conn *tls.Conn, _ http.Handler) { _, _ = io.Copy(io.Discard, conn) }, - } - server.StartTLS() - t.Cleanup(server.Close) - - previous := gogitclient.Protocols["https"] - gogitclient.InstallProtocol("https", newHTTPClient(&http.HTTP2Config{SendPingTimeout: 50 * time.Millisecond, PingTimeout: 50 * time.Millisecond})) - t.Cleanup(func() { gogitclient.InstallProtocol("https", previous) }) - - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - err := NewGitCloneExecutor(NewGitCloneExecutorInput{ - URL: server.URL + "/action", Ref: "main", Dir: filepath.Join(t.TempDir(), "action"), InsecureSkipTLS: true, - })(ctx) - require.Error(t, err) - require.NoError(t, ctx.Err()) -} diff --git a/act/runner/step_action_builtin.go b/act/runner/step_action_builtin.go index 233a133e..aeaa19e3 100644 --- a/act/runner/step_action_builtin.go +++ b/act/runner/step_action_builtin.go @@ -38,6 +38,7 @@ func (sab *stepActionBuiltin) main() common.Executor { defer rawLogger.Infof("::endgroup::") rc := sab.RunContext + rc.ApplyExtraPath(ctx, &sab.env) return sab.run(ctx, &action.Context{ Container: rc.JobContainer, Github: rc.getGithubContext(ctx), @@ -48,6 +49,7 @@ func (sab *stepActionBuiltin) main() common.Executor { NoSkipCheckout: rc.Config.NoSkipCheckout, UseGitIgnore: rc.Config.UseGitIgnore, InsecureSkipTLS: rc.Config.InsecureSkipTLS, + Env: sab.env, }) }) } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index ebd86aa9..04807615 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -18,7 +18,6 @@ import ( "gitea.com/gitea/runner/act/common/git" "gitea.dev/actionslib/pkg/model" - gogit "github.com/go-git/go-git/v5" ) type stepActionRemote struct { @@ -89,15 +88,12 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { InsecureSkipTLS: sar.cloneSkipTLS(), // For Gitea }) - var ntErr common.Executor if err := gitClone(ctx); err != nil { var refErr *git.Error switch { case errors.As(err, &refErr) && errors.Is(err, git.ErrShortRef): return fmt.Errorf("unable to resolve action `%s`, the provided ref `%s` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `%s` instead", sar.Step.Uses, sar.remoteAction.Ref, refErr.Commit()) - case errors.Is(err, gogit.ErrForceNeeded): // TODO: figure out if it will be easy to shadow/alias go-git err's - ntErr = common.NewInfoExecutor("Non-terminating error while running 'git clone': %v", err) default: return err } @@ -115,15 +111,10 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { return f, f, err } - return common.NewPipelineExecutor( - ntErr, - func(ctx context.Context) error { - defer git.AcquireCloneLock(actionDir)() - actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile) - sar.action = actionModel - return err - }, - )(ctx) + defer git.AcquireCloneLock(actionDir)() + actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile) + sar.action = actionModel + return err } } diff --git a/docs/builtin-actions.md b/docs/builtin-actions.md index 6ebb3265..feffc8ed 100644 --- a/docs/builtin-actions.md +++ b/docs/builtin-actions.md @@ -8,14 +8,32 @@ Built-in actions ship with the runner, so a job needs no action download and no ## checkout -Works like `actions/checkout` with these inputs. Any other input fails the step. +Works like `actions/checkout` with these inputs. Any other input fails the step. It runs `git` where the job runs, see the [prerequisites](../README.md#prerequisites). | Input | Default | | --- | --- | | `repository` | `github.repository` | -| `ref` | the triggering commit, or the default branch of another repository | +| `ref` | the triggering ref and commit, or the default branch of another repository | | `token` | `github.token` | +| `ssh-key` | empty, a private key fetches over SSH instead | +| `ssh-known-hosts` | empty, host keys trusted besides `~/.ssh/known_hosts` | +| `ssh-strict` | `true`, rejects unknown host keys | +| `ssh-user` | the user of the repository's SSH URL, else `git` | +| `persist-credentials` | `true`, later steps of the job can fetch and push with `token` or `ssh-key` | | `path` | workspace root | -| `fetch-depth` | `1`, `0` for full history | +| `clean` | `true`, removes untracked files, also in submodules | +| `filter` | empty, a partial clone filter such as `blob:none` | +| `sparse-checkout` | empty, the paths to check out, one per line | +| `sparse-checkout-cone-mode` | `true`, `false` reads `sparse-checkout` as patterns | +| `fetch-depth` | `1`, `0` for the full history of every branch and tag | +| `fetch-tags` | `false`, `true` fetches every tag | +| `show-progress` | `true`, shows the fetch progress | +| `lfs` | `false`, `true` downloads Git LFS files | +| `submodules` | `false`, `true` checks out submodules, `recursive` also theirs | +| `set-safe-directory` | `true`, lets checkout's `git` work in a directory another user owns | -With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` is set. +A branch is checked out as a local branch tracking `origin`, a tag or commit as a detached `HEAD`. A triggering tag that moved since the event fails the step. The step outputs `ref` and `commit` like `actions/checkout`. + +Persisted credentials end with the job. `token` is only sent to `github.server_url`, while `ssh-key` is offered to any SSH host, as with `actions/checkout`. Without `ssh-key`, submodule URLs of the form `git@:` are fetched from `github.server_url` with `token`. With `ssh-key`, the repository is fetched from the SSH URL in the event's `repository.ssh_url`, or from the server's host on port 22. + +With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` or `ssh-key` is set. diff --git a/e2e/checkout_test.go b/e2e/checkout_test.go new file mode 100644 index 00000000..70f98933 --- /dev/null +++ b/e2e/checkout_test.go @@ -0,0 +1,22 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build e2e + +package e2e + +import "testing" + +func testBuiltinCheckout(t *testing.T) { + t.Parallel() + + api, repo := newScenario(t) + pushWorkflow(t, api, repo, "checkout.yml") + branchRun := waitForRun(t, api, repo) + if err := api.CreateTag(t.Context(), repo, "v1"); err != nil { + t.Fatalf("create tag: %v", err) + } + tagRun := waitForRunAfter(t, api, repo, branchRun.ID) + requireSuccess(t, api, repo, branchRun.ID) + requireSuccess(t, api, repo, tagRun.ID) +} diff --git a/e2e/compatibility_test.go b/e2e/compatibility_test.go index 2dee668e..b63ee06d 100644 --- a/e2e/compatibility_test.go +++ b/e2e/compatibility_test.go @@ -30,6 +30,7 @@ func TestCompatibility(t *testing.T) { t.Run("artifact", testArtifactRoundTrip) t.Run("cancellation_and_log_streaming", testRunCancellation) t.Run("dispatch", testWorkflowDispatch) + t.Run("builtin_checkout", testBuiltinCheckout) t.Run("ephemeral", testEphemeralRunner) t.Run("kubernetes", testKubernetes) } diff --git a/e2e/gitea_api.go b/e2e/gitea_api.go index 4987cec6..1568887a 100644 --- a/e2e/gitea_api.go +++ b/e2e/gitea_api.go @@ -53,6 +53,12 @@ func (a *GiteaAPI) CreateFile(ctx context.Context, repo, path, content, message return a.doJSON(ctx, http.MethodPost, url, body, nil) } +func (a *GiteaAPI) CreateTag(ctx context.Context, repo, name string) error { + body := map[string]any{"tag_name": name} + url := fmt.Sprintf("/api/v1/repos/%s/%s/tags", giteaAdminUser, repo) + return a.doJSON(ctx, http.MethodPost, url, body, nil) +} + func (a *GiteaAPI) CreateSecret(ctx context.Context, repo, name, value string) error { body := map[string]any{"data": value} url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/secrets/%s", giteaAdminUser, repo, name) diff --git a/e2e/gitea_fixture.go b/e2e/gitea_fixture.go index 37447274..bc0e3ba0 100644 --- a/e2e/gitea_fixture.go +++ b/e2e/gitea_fixture.go @@ -188,6 +188,7 @@ func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, e "GITEA__security__INSTALL_LOCK=true", "GITEA__database__DB_TYPE=sqlite3", "GITEA__actions__ENABLED=true", + "GITEA__server__LFS_START_SERVER=true", "GITEA__server__ROOT_URL=" + baseURL + "/", }, ExposedPorts: network.PortSet{containerPort: struct{}{}}, diff --git a/e2e/main_test.go b/e2e/main_test.go index bde84ee6..3abe2956 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -141,6 +141,11 @@ func dumpRunLogs(t *testing.T, api *GiteaAPI, repo string, runID int64) { } func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun { + t.Helper() + return waitForRunAfter(t, api, repo, 0) +} + +func waitForRunAfter(t *testing.T, api *GiteaAPI, repo string, previousID int64) *ActionRun { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() @@ -150,7 +155,7 @@ func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun { if err != nil { t.Fatalf("get latest run: %v", err) } - if len(runs) > 0 { + if len(runs) > 0 && runs[0].ID > previousID { return &runs[0] } select { diff --git a/e2e/testdata/workflows/checkout.yml b/e2e/testdata/workflows/checkout.yml new file mode 100644 index 00000000..c9c67ada --- /dev/null +++ b/e2e/testdata/workflows/checkout.yml @@ -0,0 +1,61 @@ +name: checkout +on: push +jobs: + checkout: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - id: checkout + uses: builtin:checkout + - uses: builtin:checkout + with: + ref: ${{ github.ref_name }} + path: short + - run: test "$(git -C short rev-parse HEAD)" = "$GITHUB_SHA" + - if: github.ref_type == 'branch' + run: | + test "$(git symbolic-ref HEAD)" = "$GITHUB_REF" + test "${{ steps.checkout.outputs.ref }}" = "$GITHUB_REF" + test "${{ steps.checkout.outputs.commit }}" = "$GITHUB_SHA" + git -c user.name=e2e -c user.email=e2e@example.com commit --allow-empty -m pushed + git push + git tag moved + git push --force origin moved + git switch --orphan child + echo child > child.txt + git add child.txt + git -c user.name=e2e -c user.email=e2e@example.com commit -m child + git push origin child + git switch -c content main + git submodule add -b child "../${{ github.event.repository.name }}.git" sub + git lfs install --local + git lfs track '*.bin' + mkdir keep drop + echo big > keep/big.bin + echo big > drop/big.bin + git add .gitattributes keep drop + git -c user.name=e2e -c user.email=e2e@example.com commit -m content + git push origin content + - if: github.ref_type == 'branch' + uses: builtin:checkout + with: + ref: content + path: full + submodules: true + lfs: true + - if: github.ref_type == 'branch' + uses: builtin:checkout + with: + ref: content + path: sparse + sparse-checkout: keep + lfs: true + - if: github.ref_type == 'branch' + run: | + test "$(cat full/sub/child.txt)" = child + test "$(cat full/keep/big.bin)" = big + test "$(cat sparse/keep/big.bin)" = big + test ! -e sparse/drop + - if: github.ref_type == 'tag' + run: test "$(git describe --tags)" = "$GITHUB_REF_NAME" diff --git a/internal/pkg/action/action.go b/internal/pkg/action/action.go index e9c4f3c3..51e52099 100644 --- a/internal/pkg/action/action.go +++ b/internal/pkg/action/action.go @@ -20,4 +20,5 @@ type Context struct { NoSkipCheckout bool UseGitIgnore bool InsecureSkipTLS bool + Env map[string]string } diff --git a/internal/pkg/action/checkout/checkout.go b/internal/pkg/action/checkout/checkout.go index 4220374b..9fed9527 100644 --- a/internal/pkg/action/checkout/checkout.go +++ b/internal/pkg/action/checkout/checkout.go @@ -6,21 +6,36 @@ package checkout import ( "cmp" "context" + "crypto/sha256" + "errors" "fmt" - "os" + "maps" + "net/url" "path" "path/filepath" + "runtime" "slices" "strconv" "strings" + "time" "gitea.com/gitea/runner/act/common/git" + "gitea.com/gitea/runner/act/container" "gitea.com/gitea/runner/internal/pkg/action" + + "gitea.dev/actionslib/pkg/model" + "github.com/kballard/go-shellquote" ) -var inputNames = []string{"repository", "ref", "token", "path", "fetch-depth"} +var inputNames = []string{ + "repository", "ref", "token", "ssh-key", "ssh-known-hosts", "ssh-strict", "ssh-user", "persist-credentials", "path", "clean", + "filter", "sparse-checkout", "sparse-checkout-cone-mode", "fetch-depth", "fetch-tags", "show-progress", "lfs", "submodules", + "set-safe-directory", +} -func Main(ctx context.Context, c *action.Context) error { +const credentialsInclude = "includeIf.gitdir:**.path" // the form actions/checkout removes from a repository it checks out again + +func Main(ctx context.Context, c *action.Context) (err error) { for name := range c.Inputs { if !slices.Contains(inputNames, name) { return fmt.Errorf("checkout: unsupported input %q", name) @@ -37,38 +52,382 @@ func Main(ctx context.Context, c *action.Context) error { ghc := c.Github repository := cmp.Or(c.Inputs["repository"], ghc.Repository) - ref := shortBranch(c.Inputs["ref"]) - isWorkflowRepo := repository == ghc.Repository - local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, ref) && !c.NoSkipCheckout - if local && !c.BindWorkdir { - return c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx) - } - if local && dst == c.Workspace { + ref := c.Inputs["ref"] + sshKey := strings.TrimSpace(c.Inputs["ssh-key"]) + isWorkflowRepo := strings.EqualFold(repository, ghc.Repository) + local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, shortBranch(ref)) && !c.NoSkipCheckout && sshKey == "" + if local && (!c.BindWorkdir || dst == c.Workspace) { + if !c.BindWorkdir { + if err := c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx); err != nil { + return err + } + } + if output := c.Env["GITHUB_OUTPUT"]; output != "" { + return c.Container.Copy(path.Dir(output), &container.FileEntry{Name: path.Base(output), Mode: 0o666, Body: fmt.Sprintf("ref=%s\ncommit=%s\n", ghc.Ref, ghc.Sha)})(ctx) + } return nil } - if ref == "" { - ref = "HEAD" - if isWorkflowRepo { - ref = ghc.Sha + co := &checkout{ + url: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository, + dst: dst, + ref: ref, + depth: depth, + fetchTags: boolInput(c.Inputs, "fetch-tags", false), + clean: boolInput(c.Inputs, "clean", true), + lfs: boolInput(c.Inputs, "lfs", false), + progress: boolInput(c.Inputs, "show-progress", true), + cone: boolInput(c.Inputs, "sparse-checkout-cone-mode", true), + filter: strings.TrimSpace(c.Inputs["filter"]), + output: c.Env["GITHUB_OUTPUT"], + } + for pattern := range strings.Lines(c.Inputs["sparse-checkout"]) { + if pattern = strings.TrimSpace(pattern); pattern != "" { + co.sparse = append(co.sparse, pattern) } } - dir, err := os.MkdirTemp("", "checkout-") - if err != nil { + if len(co.sparse) > 0 { + co.filter = cmp.Or(co.filter, "blob:none") + } + if submodules := strings.ToLower(strings.TrimSpace(c.Inputs["submodules"])); submodules == "true" || submodules == "recursive" { + co.submodules = submodules + } + if sshKey != "" { + if co.url, err = sshURL(ghc, repository, strings.TrimSpace(c.Inputs["ssh-user"])); err != nil { + return err + } + } + co.objectFormat = git.ObjectFormat(ref) + switch { + case co.objectFormat != "": + co.sha, co.ref = ref, "" + case ref == "" && isWorkflowRepo && strings.HasPrefix(ghc.Ref, "refs/"): + co.sha, co.ref = ghc.Sha, ghc.Ref + case ref == "" && isWorkflowRepo: // a merged pull request's ref is its merge commit + co.sha = ghc.Sha + case ref == "HEAD": + co.ref = "" + } + if isWorkflowRepo { + co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(ghc.Sha)) + } + + // named like actions/checkout's, which create-pull-request knows to hide, and gone with the job + runnerTemp, _ := c.Container.GetRunnerContext(ctx)["temp"].(string) + digest := sha256.Sum256([]byte(dst)) + credentials := fmt.Sprintf("git-credentials-%x.config", digest[:8]) + co.credentials = path.Join(runnerTemp, credentials) + persist := boolInput(c.Inputs, "persist-credentials", true) + config, remoteConfig := serverConfig(ghc.ServerURL, cmp.Or(c.Inputs["token"], ghc.Token), c.InsecureSkipTLS, co.submodules != "" && sshKey == "") + if boolInput(c.Inputs, "set-safe-directory", true) { + remoteConfig = append(remoteConfig, "safe.directory=*") // a bound workdir belongs to the host user + } + env := gitEnv(c, remoteConfig) + env["GIT_LFS_SKIP_SMUDGE"] = strconv.FormatBool(!co.lfs || len(co.sparse) == 0) // lfs pull fetches the objects of a full checkout in one go + if sshKey != "" { + key := fmt.Sprintf("git-ssh-key-%x", digest[:8]) + files := []*container.FileEntry{{Name: key, Mode: 0o600, Body: sshKey + "\n"}} + ssh := []string{"ssh", "-i", path.Join(runnerTemp, key)} + if boolInput(c.Inputs, "ssh-strict", true) { + ssh = append(ssh, "-o", "StrictHostKeyChecking=yes", "-o", "CheckHostIP=no") + } + if knownHosts := strings.TrimSpace(c.Inputs["ssh-known-hosts"]); knownHosts != "" { + name := fmt.Sprintf("git-ssh-known-hosts-%x", digest[:8]) + files = append(files, &container.FileEntry{Name: name, Mode: 0o600, Body: knownHosts + "\n"}) + ssh = append(ssh, "-o", "UserKnownHostsFile=~/.ssh/known_hosts "+strconv.Quote(path.Join(runnerTemp, name))) + } + if !persist { + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancel() + err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx)) + }() + } + if err := c.Container.Copy(runnerTemp, files...)(ctx); err != nil { + return err + } + env["GIT_SSH_COMMAND"] = shellquote.Join(ssh...) + config += "[core]\n\tsshCommand = " + strconv.Quote(env["GIT_SSH_COMMAND"]) + "\n" + } + if !persist { + config = "" + } + + if co.objectFormat == "" || co.sha == "" && !strings.HasPrefix(co.ref, "refs/") { + advertised, err := lsRemote(ctx, c, env, co.url, co.ref) + if err != nil { + return fmt.Errorf("checkout: %s@%s: %w", repository, ref, err) + } + if co.resolve(advertised); co.ref == "" && co.sha == "" { + return fmt.Errorf("checkout: %s has no ref %q", repository, ref) + } + } + co.outputRef = co.ref + if ref != co.sha { + co.outputRef = cmp.Or(ref, co.ref) // as given, like actions/checkout + } + if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: credentials, Mode: 0o600, Body: config})(ctx); err != nil { return err } - defer os.RemoveAll(dir) - if err := git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ - URL: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository, - Ref: ref, - Dir: dir, - Token: cmp.Or(c.Inputs["token"], ghc.Token), - Depth: depth, - InsecureSkipTLS: c.InsecureSkipTLS, - })(ctx); err != nil { - return fmt.Errorf("checkout: clone %s@%s: %w", repository, ref, err) + + clone := func(commands [][]string) error { + if err := run(ctx, c, env, commands); err != nil { + return fmt.Errorf("checkout: clone %s@%s: %w", repository, cmp.Or(co.ref, co.sha), err) + } + return nil } - return c.Container.CopyDir(dst, dir+string(filepath.Separator)+".", false, false)(ctx) + commands, fetched := co.commands() + if co.sha != "" && strings.HasPrefix(co.ref, "refs/tags/") { + if err := clone(commands[:fetched]); err != nil { + return err + } + tagged, err := output(ctx, c, env, "git", "-C", dst, "rev-parse", "--verify", "--quiet", co.ref+"^{commit}") + if err != nil { + return fmt.Errorf("checkout: %s: %w", co.ref, err) + } + if !strings.EqualFold(strings.TrimSpace(tagged), co.sha) { + return fmt.Errorf("checkout: %s does not point to the expected commit %s, it may have been moved after the workflow was triggered", co.ref, co.sha) + } + commands = commands[fetched:] + } + return clone(commands) +} + +func gitEnv(c *action.Context, config []string) map[string]string { + env := map[string]string{"GIT_TERMINAL_PROMPT": "0"} + maps.Copy(env, c.Env) + maps.DeleteFunc(env, func(name, _ string) bool { return slices.Contains(git.LocalEnvVars, name) }) + maps.Copy(env, git.ConfigEnv(append([]string{"protocol.version=2", "init.defaultBranch=main", "advice.detachedHead=false", "maintenance.auto=false"}, config...)...)) + return env +} + +func run(ctx context.Context, c *action.Context, env map[string]string, commands [][]string) error { + if _, host := c.Container.(*container.HostEnvironment); !host || runtime.GOOS != "windows" { + script := []string{"set -e", `command -v git >/dev/null || { echo "builtin:checkout needs git on PATH" >&2; exit 127; }`} + for _, command := range commands { + script = append(script, shellquote.Join(command...)) + } + return c.Container.Exec([]string{"sh", "-c", strings.Join(script, "\n")}, env, "", "")(ctx) + } + for _, command := range commands { + if err := c.Container.Exec(command, env, "", "")(ctx); err != nil { + return err + } + } + return nil +} + +// output runs where the job runs, so the job's network and git config apply as for the fetch. +func output(ctx context.Context, c *action.Context, env map[string]string, command ...string) (string, error) { + var out strings.Builder + stdout, stderr := c.Container.ReplaceLogWriter(&out, &out) + defer c.Container.ReplaceLogWriter(stdout, stderr) + if err := c.Container.Exec(command, env, "", "")(ctx); err != nil { + return "", fmt.Errorf("%w: %s", err, strings.TrimSpace(out.String())) + } + return out.String(), nil +} + +func lsRemote(ctx context.Context, c *action.Context, env map[string]string, url, ref string) (string, error) { + patterns := []string{ref} + switch { + case ref == "": + patterns = []string{"HEAD"} + case !strings.HasPrefix(ref, "refs/"): + patterns = []string{"refs/heads/" + ref, "refs/tags/" + ref} + } + return output(ctx, c, env, append([]string{"git", "ls-remote", "--symref", url}, patterns...)...) +} + +// serverConfig scopes the token to the server, so no other host like a repository's LFS endpoint gets it. +func serverConfig(serverURL, token string, insecureSkipTLS, rewriteSSH bool) (file string, env []string) { + scope := strings.TrimSuffix(serverURL, "/") + "/" + file = fmt.Sprintf("[http %q]\n", scope) + for _, entry := range git.RemoteConfig(token, insecureSkipTLS) { + setting := strings.TrimPrefix(entry, "http.") + key, value, _ := strings.Cut(setting, "=") + file += "\t" + key + " = " + value + "\n" + env = append(env, "http."+scope+"."+setting) + } + if server, err := url.Parse(serverURL); rewriteSSH && err == nil && server.Hostname() != "" { // submodules on the server fetch over HTTP with the token + file += fmt.Sprintf("[url %q]\n\tinsteadOf = git@%s:\n", scope, server.Hostname()) + env = append(env, "url."+scope+".insteadOf=git@"+server.Hostname()+":") + } + return file, env +} + +func sshURL(ghc *model.GithubContext, repository, user string) (string, error) { + endpoint := &url.URL{Scheme: "ssh"} + if advertised, _ := model.NestedMapLookup(ghc.Event, "repository", "ssh_url").(string); advertised != "" { + if colon := strings.LastIndex(advertised, ":"); !strings.HasPrefix(advertised, "ssh://") && colon >= 0 { + advertised = "ssh://" + advertised[:colon] + "/" + advertised[colon+1:] // scp-like git@host:owner/repo.git + } + var err error + if endpoint, err = url.Parse(advertised); err != nil { + return "", fmt.Errorf("checkout: parse the event's ssh_url: %w", err) + } + } else if server, err := url.Parse(ghc.ServerURL); err == nil { + endpoint.Host = server.Hostname() + } + endpoint.User = url.User(cmp.Or(user, endpoint.User.Username(), "git")) + endpoint.Path = "/" + repository + ".git" + return endpoint.String(), nil +} + +type checkout struct { + url, dst, objectFormat, credentials, output, outputRef string + ref, sha string // ref is fully qualified or empty, sha pins it when known + filter, submodules string + sparse []string + depth int + fetchTags, clean, lfs, progress, cone bool +} + +const fullDepth = "2147483647" // git's INFINITE_DEPTH, also deepens a shallow dst + +// commands checks out co.dst, the first `fetched` of them fetch so a fetched tag can be verified before the rest. +func (co *checkout) commands() (commands [][]string, fetched int) { + repo := func(args ...string) []string { return append([]string{"git", "-C", co.dst}, args...) } + commands = [][]string{ + {"git", "init", "--quiet", "--object-format=" + co.objectFormat, co.dst}, + repo("config", "remote.origin.url", co.url), // `remote add` fails on a repeated checkout + repo("config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"), + repo("config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials), + } + if co.lfs { + commands = append(commands, repo("lfs", "install", "--local")) + } + + depth := strconv.Itoa(co.depth) + var refSpecs []string + if co.depth == 0 { + depth = fullDepth + refSpecs = append(refSpecs, "+refs/heads/*:refs/remotes/origin/*") + } + fetch := []string{"fetch", "--force", "--no-tags", "--no-recurse-submodules", "--depth", depth} + if co.progress { + fetch = append(fetch, "--progress") + } else { + fetch = append(fetch, "--no-progress") + } + if co.filter != "" { + fetch = append(fetch, "--filter="+co.filter) + } + if co.fetchTags || co.depth == 0 { + refSpecs = append(refSpecs, "+refs/tags/*:refs/tags/*") + } + tag := strings.HasPrefix(co.ref, "refs/tags/") // fetched itself, which keeps an annotated tag's object + pinned := co.sha != "" && co.ref != "" && !tag + if co.ref != "" && !pinned { + refSpecs = append(refSpecs, "+"+co.ref+":"+trackingRef(co.ref)) + } + if co.sha != "" && !tag { + refSpecs = append(refSpecs, co.sha) + } + commands = append(commands, repo(append(append(fetch, "origin"), refSpecs...)...)) + if pinned { + commands = append(commands, repo("update-ref", trackingRef(co.ref), co.sha)) + } + fetched = len(commands) + + if len(co.sparse) > 0 { + mode := "--cone" + if !co.cone { + mode = "--no-cone" + } + commands = append(commands, repo("sparse-checkout", "init", mode), repo(append([]string{"sparse-checkout", "set", "--"}, co.sparse...)...)) + } + if branch, ok := strings.CutPrefix(co.ref, "refs/heads/"); ok { + commands = append(commands, repo("checkout", "--force", "--ignore-other-worktrees", "-B", branch, "--track", trackingRef(co.ref))) + } else { + commands = append(commands, repo("checkout", "--force", cmp.Or(co.sha, trackingRef(co.ref)))) + } + if len(co.sparse) == 0 { // after the checkout, so a sparse dst of another repository needs nothing of it + commands = append(commands, repo("sparse-checkout", "disable"), repo("config", "--local", "--unset-all", "extensions.worktreeConfig")) + } + if co.lfs && len(co.sparse) == 0 { + commands = append(commands, repo("lfs", "pull", "origin")) + } + if co.clean { + commands = append(commands, repo("clean", "-ffdx")) + } + if co.submodules != "" { + commands = append(commands, co.submoduleCommands(repo)...) + } + if co.output != "" { + commands = append(commands, repo("log", "-1", "--format=commit=%H%nref="+strings.ReplaceAll(co.outputRef, "%", "%%"), "--output="+co.output)) + } + return commands, fetched +} + +func (co *checkout) submoduleCommands(repo func(args ...string) []string) [][]string { + var recursive []string + if co.submodules == "recursive" { + recursive = []string{"--recursive"} + } + foreach := func(commands ...[]string) []string { + joined := make([]string, len(commands)) + for i, command := range commands { + joined[i] = shellquote.Join(command...) + } + return repo(append(append([]string{"submodule", "foreach"}, recursive...), strings.Join(joined, " && "))...) + } + commands := [][]string{repo(append([]string{"submodule", "sync"}, recursive...)...)} + if co.clean { + commands = append(commands, foreach([]string{"git", "clean", "-ffdx"})) + } + update := append([]string{"submodule", "update", "--init", "--force"}, recursive...) + if co.depth > 0 { + update = append(update, "--depth="+strconv.Itoa(co.depth)) + } + include := []string{"git", "config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials} + if co.lfs { + return append(commands, repo(update...), foreach([]string{"git", "lfs", "install", "--local"}, []string{"git", "lfs", "pull", "origin"}, include)) + } + return append(commands, repo(update...), foreach(include)) +} + +func (co *checkout) resolve(advertised string) { + ref := co.ref + if !strings.HasPrefix(ref, "refs/") { + co.ref = "" + } + for line := range strings.Lines(advertised) { + value, name, _ := strings.Cut(strings.TrimSpace(line), "\t") + if target, ok := strings.CutPrefix(value, "ref: "); ok { + if name == "HEAD" { + co.ref = target + } + continue + } + co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(value)) + switch name { + case "HEAD": + if co.ref == "" { + co.sha = value + } + case "refs/heads/" + ref: + co.ref = name + case "refs/tags/" + ref: + co.ref = cmp.Or(co.ref, name) + } + } + co.objectFormat = cmp.Or(co.objectFormat, "sha1") +} + +func trackingRef(ref string) string { + if branch, ok := strings.CutPrefix(ref, "refs/heads/"); ok { + return "refs/remotes/origin/" + branch + } + if pull, ok := strings.CutPrefix(ref, "refs/pull/"); ok { + return "refs/remotes/pull/" + pull + } + return ref +} + +func boolInput(inputs map[string]string, name string, fallback bool) bool { + value := strings.TrimSpace(inputs[name]) + return strings.EqualFold(value, "true") || value == "" && fallback } func workspaceDestination(workspace, checkoutPath string) (string, error) { @@ -91,7 +450,6 @@ func fetchDepth(input string) (int, error) { return depth, nil } -// NewGitCloneExecutor resolves branches by short name, a full tag ref stays exact. -func shortBranch(ref string) string { +func shortBranch(ref string) string { // full tag refs stay exact return strings.TrimPrefix(ref, "refs/heads/") } diff --git a/internal/pkg/action/checkout/checkout_test.go b/internal/pkg/action/checkout/checkout_test.go index b11ef419..cc41a7da 100644 --- a/internal/pkg/action/checkout/checkout_test.go +++ b/internal/pkg/action/checkout/checkout_test.go @@ -4,14 +4,16 @@ package checkout import ( - "context" + "cmp" + "crypto/sha256" + "fmt" + "io" "os" "os/exec" "path/filepath" "strings" "testing" - "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/container" "gitea.com/gitea/runner/internal/pkg/action" @@ -20,22 +22,6 @@ import ( "github.com/stretchr/testify/require" ) -type fakeContainer struct { - container.ExecutionsEnvironment - dst, src string - log []string -} - -func (f *fakeContainer) CopyDir(dst, src string, _, _ bool) common.Executor { - return func(context.Context) error { - f.dst, f.src = dst, src - if out, err := exec.Command("git", "--git-dir", filepath.Join(src, ".git"), "log", "--format=%s").Output(); err == nil { - f.log = strings.Fields(string(out)) - } - return nil - } -} - func runGit(t *testing.T, args ...string) string { t.Helper() cmd := exec.Command("git", args...) @@ -49,67 +35,282 @@ func runGit(t *testing.T, args ...string) string { return strings.TrimSpace(string(out)) } +func gitLog(dir string) []string { + out, err := exec.Command("git", "--git-dir", filepath.Join(dir, ".git"), "log", "--format=%s").Output() + if err != nil { + return nil + } + return strings.Fields(string(out)) +} + func TestCheckout(t *testing.T) { - work := t.TempDir() - runGit(t, "init", "--initial-branch=main", work) - runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1") - sha := runGit(t, "-C", work, "rev-parse", "HEAD") - runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2") - runGit(t, "-C", work, "checkout", "-b", "feature") - runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1") - runGit(t, "-C", work, "checkout", "--orphan", "trunk") - runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1") - runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2") - runGit(t, "-C", work, "tag", "trunk", "main") server := t.TempDir() - runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo")) - runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo")) + var sha string + _, lfsErr := exec.LookPath("git-lfs") + require.True(t, t.Run("fixtures", func(t *testing.T) { + t.Run("history", func(t *testing.T) { + t.Parallel() + work := t.TempDir() + runGit(t, "init", "--initial-branch=main", work) + runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1") + sha = runGit(t, "-C", work, "rev-parse", "HEAD") + runGit(t, "-C", work, "tag", "v1") + runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2") + runGit(t, "-C", work, "checkout", "-b", "feature") + runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1") + runGit(t, "-C", work, "checkout", "--orphan", "trunk") + runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1") + runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2") + runGit(t, "-C", work, "tag", "trunk", "main") + runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo")) + runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo")) + runGit(t, "-C", filepath.Join(server, "other", "repo"), "symbolic-ref", "refs/remotes/origin/HEAD", "refs/heads/main") + }) + t.Run("sha256", func(t *testing.T) { + t.Parallel() + sha256Repo := filepath.Join(server, "org", "sha256") + runGit(t, "init", "--object-format=sha256", "--initial-branch=main", sha256Repo) + runGit(t, "-C", sha256Repo, "commit", "--allow-empty", "-m", "s1") + runGit(t, "-C", sha256Repo, "checkout", "--detach") + }) + t.Run("submodules", func(t *testing.T) { + t.Parallel() + publish(t, server, "org/nested", map[string]string{"nested.txt": "nested"}, nil) + publish(t, server, "org/child", map[string]string{"child.txt": "child"}, map[string]string{"nested": "../nested"}) + publish(t, server, "org/content", map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b"}, map[string]string{"sub": "../child"}) + runGit(t, "-C", filepath.Join(server, "org", "content"), "config", "uploadpack.allowFilter", "true") + }) + t.Run("lfs", func(t *testing.T) { + t.Parallel() + oid := fmt.Sprintf("%x", sha256.Sum256([]byte("big"))) + pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize 3\n" + publish(t, server, "org/lfs", map[string]string{".gitattributes": "*.bin filter=lfs diff=lfs merge=lfs -text\n", "keep/big.bin": pointer, "drop/big.bin": pointer}, nil) + object := filepath.Join(server, "org", "lfs", "lfs", "objects", oid[:2], oid[2:4], oid) + require.NoError(t, os.MkdirAll(filepath.Dir(object), 0o755)) + require.NoError(t, os.WriteFile(object, []byte("big"), 0o644)) + }) + })) host := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(host, "marker"), []byte("marker"), 0o644)) + tools := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tools, "ssh"), []byte(`#!/bin/sh +test -s "$2" || exit 1 +printf '%s\n' "$@" > "$SSH_ARGS" +for last; do :; done +exec sh -c "$(printf '%s' "$last" | sed "s#git-upload-pack '/\(.*\)\.git'#git upload-pack '$SSH_SERVER/\1'#")" +`), 0o755)) + globalConfig := filepath.Join(t.TempDir(), "gitconfig") + require.NoError(t, os.WriteFile(globalConfig, []byte("[protocol \"file\"]\n\tallow = always\n"), 0o644)) + sshEvent := func(url string) map[string]any { return map[string]any{"repository": map[string]any{"ssh_url": url}} } for _, tc := range []struct { - name string - bind, noSkip bool - inputs map[string]string - dst string - log []string - hostCopy, err bool + name string + bind, noSkip bool + eventRef string + event map[string]any + before, inputs map[string]string + dst, head, err string + log, refs []string + files, config map[string]string }{ {name: "bound workspace is skipped", bind: true}, - {name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "/workspace/src", hostCopy: true}, - {name: "bound path clones event sha", bind: true, inputs: map[string]string{"path": "src"}, dst: "/workspace/src", log: []string{"c1"}}, - {name: "no-skip clones event sha", noSkip: true, dst: "/workspace", log: []string{"c1"}}, - {name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, dst: "/workspace", log: []string{"f1"}}, - {name: "tag ref wins over same-named branch", inputs: map[string]string{"ref": "refs/tags/trunk"}, dst: "/workspace", log: []string{"c2"}}, - {name: "fetch-depth 0 clones full history", inputs: map[string]string{"ref": "feature", "fetch-depth": "0"}, dst: "/workspace", log: []string{"f1", "c2", "c1"}}, - {name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, dst: "/workspace", log: []string{"t2"}}, - {name: "unsupported input", inputs: map[string]string{"submodules": "true"}, err: true}, - {name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: true}, - {name: "absolute path", inputs: map[string]string{"path": "/x"}, err: true}, - {name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: true}, + {name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "src", files: map[string]string{"marker": "marker"}}, + {name: "bound path checks out the event branch", bind: true, inputs: map[string]string{"path": "src"}, dst: "src", head: "main", log: []string{"c1"}}, + {name: "no-skip checks out the event branch", noSkip: true, head: "main", log: []string{"c1"}, refs: []string{"refs/remotes/origin/main"}}, + {name: "event tag is kept", noSkip: true, eventRef: "refs/tags/v1", log: []string{"c1"}, refs: []string{"refs/tags/v1"}}, + {name: "moved event tag fails", noSkip: true, eventRef: "refs/tags/trunk", err: "does not point to the expected commit " + sha}, + {name: "deleted event tag fails", noSkip: true, eventRef: "refs/tags/deleted", err: "checkout: clone"}, + { + name: "sha input removes previously persisted credentials without clean", noSkip: true, before: map[string]string{"ref": sha}, + inputs: map[string]string{"ref": sha, "persist-credentials": "false", "clean": "false"}, + log: []string{"c1"}, + }, + {name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, head: "feature", log: []string{"f1"}}, + {name: "tag ref wins over same-named branch and is kept", inputs: map[string]string{"ref": "refs/tags/trunk"}, log: []string{"c2"}, refs: []string{"refs/tags/trunk"}}, + {name: "branch ref wins over same-named tag", inputs: map[string]string{"ref": "refs/heads/trunk"}, head: "trunk", log: []string{"t2"}}, + { + name: "short branch wins over same-named tag and fetch-tags fetches every tag", inputs: map[string]string{"ref": "trunk", "fetch-tags": "true"}, + head: "trunk", log: []string{"t2"}, refs: []string{"refs/tags/trunk", "refs/tags/v1"}, + }, + {name: "short tag", inputs: map[string]string{"ref": "v1"}, log: []string{"c1"}, refs: []string{"refs/tags/v1"}}, + { + name: "fetch-depth 0 deepens a shallow checkout to every branch and tag", before: map[string]string{"ref": "feature"}, + inputs: map[string]string{"ref": "refs/heads/feature", "fetch-depth": "0"}, head: "feature", log: []string{"f1", "c2", "c1"}, + refs: []string{"refs/remotes/origin/feature", "refs/remotes/origin/main", "refs/remotes/origin/trunk", "refs/tags/trunk", "refs/tags/v1"}, + }, + {name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, head: "trunk", log: []string{"t2"}}, + {name: "explicit HEAD checks out the default branch", inputs: map[string]string{"repository": "other/repo", "ref": "HEAD"}, head: "trunk", log: []string{"t2"}}, + {name: "detached sha256 HEAD is checked out", inputs: map[string]string{"repository": "org/sha256"}, log: []string{"s1"}}, + { + name: "ssh key with the advertised endpoint persists", event: sshEvent("ssh://svc@ssh.example:2222/org/repo.git"), + inputs: map[string]string{"ssh-key": "KEY", "ssh-user": "me", "ssh-known-hosts": "ssh.example ssh-ed25519 AAAA"}, + head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://me@ssh.example:2222/org/repo.git"}, + }, + { + name: "ssh key without persisting is emptied", event: sshEvent("svc@ssh.example:org/repo.git"), + inputs: map[string]string{"ssh-key": "KEY", "persist-credentials": "false"}, + head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://svc@ssh.example/org/repo.git"}, + }, + { + name: "sparse cone checkout filters blobs", inputs: map[string]string{"repository": "org/content", "sparse-checkout": "a\n", "show-progress": "false"}, + head: "main", log: []string{"content"}, files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": ""}, + config: map[string]string{"remote.origin.partialclonefilter": "blob:none"}, + }, + { + name: "non-cone sparse checkout with an explicit filter", head: "main", log: []string{"content"}, + inputs: map[string]string{"repository": "org/content", "sparse-checkout": "b/b.txt", "sparse-checkout-cone-mode": "false", "filter": "blob:limit=1024"}, + files: map[string]string{"root.txt": "", "a/a.txt": "", "b/b.txt": "b"}, config: map[string]string{"remote.origin.partialclonefilter": "blob:limit=1024"}, + }, + { + name: "full checkout after a sparse one restores every file", before: map[string]string{"repository": "org/content", "sparse-checkout": "a"}, + inputs: map[string]string{"repository": "org/content", "set-safe-directory": "false"}, head: "main", log: []string{"content"}, + files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b", "sub/child.txt": ""}, + }, + { + name: "recursive submodules after plain ones", before: map[string]string{"repository": "org/content", "submodules": "true"}, + inputs: map[string]string{"repository": "org/content", "submodules": "Recursive"}, head: "main", log: []string{"content"}, + files: map[string]string{"sub/child.txt": "child", "sub/nested/nested.txt": "nested"}, + }, + { + name: "lfs content stays when narrowing a full checkout to a sparse one", before: map[string]string{"repository": "org/lfs", "lfs": "true"}, + inputs: map[string]string{"repository": "org/lfs", "lfs": "true", "sparse-checkout": "keep"}, head: "main", log: []string{"content"}, + files: map[string]string{"keep/big.bin": "big", "drop/big.bin": ""}, + }, + {name: "unknown ref", inputs: map[string]string{"ref": "missing"}, err: "has no ref"}, + {name: "unsupported input", inputs: map[string]string{"github-server-url": "https://example.com"}, err: "unsupported input"}, + {name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: "must be relative"}, + {name: "absolute path", inputs: map[string]string{"path": "/x"}, err: "must be relative"}, + {name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: "non-negative"}, } { t.Run(tc.name, func(t *testing.T) { - fc := &fakeContainer{} - err := Main(t.Context(), &action.Context{ - Container: fc, - Github: &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", Sha: sha, ServerURL: server}, - Inputs: tc.inputs, - HostWorkdir: host, - Workspace: "/workspace", - BindWorkdir: tc.bind, - NoSkipCheckout: tc.noSkip, - }) - if tc.err { - require.Error(t, err) - assert.Empty(t, fc.dst) + t.Parallel() + if tc.inputs["lfs"] == "true" && lfsErr != nil { + t.Skip("needs git-lfs") + } + workspace, runnerTemp := t.TempDir(), t.TempDir() + output := filepath.Join(runnerTemp, "output") + checkout := func(inputs map[string]string) error { + return Main(t.Context(), &action.Context{ + Container: &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard}, + Github: &model.GithubContext{ + Repository: "org/repo", Ref: cmp.Or(tc.eventRef, "refs/heads/main"), Sha: sha, ServerURL: "file://" + server, Token: "tok", Event: tc.event, + }, + Inputs: inputs, + HostWorkdir: host, + Workspace: workspace, + BindWorkdir: tc.bind, + NoSkipCheckout: tc.noSkip, + Env: map[string]string{ + "PATH": tools + string(os.PathListSeparator) + os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": globalConfig, "GIT_CONFIG_NOSYSTEM": "1", + "GIT_DIR": host, "GITHUB_OUTPUT": output, "SSH_SERVER": server, "SSH_ARGS": filepath.Join(runnerTemp, "ssh-args"), + }, + }) + } + dst := filepath.Join(workspace, tc.dst) + if tc.err != "" { + require.ErrorContains(t, checkout(tc.inputs), tc.err) + if tc.eventRef == "" { + entries, _ := os.ReadDir(workspace) + assert.Empty(t, entries) + } else { + assert.Nil(t, gitLog(dst)) + } return } + first := tc.before + if first == nil { + first = tc.inputs + } + require.NoError(t, checkout(first)) + repos := []string{dst} + if fileExists(filepath.Join(dst, "sub", ".git")) { + repos = append(repos, filepath.Join(dst, "sub")) + } + for _, repo := range repos { + require.NoError(t, os.WriteFile(filepath.Join(repo, "untracked"), nil, 0o644)) + } + require.NoError(t, checkout(tc.inputs)) + outputs, err := os.ReadFile(output) require.NoError(t, err) - assert.Equal(t, tc.dst, fc.dst) - assert.Equal(t, tc.log, fc.log) - assert.Equal(t, tc.hostCopy, strings.HasPrefix(fc.src, host)) - if tc.log != nil { - assert.NoDirExists(t, fc.src) + assert.Equal(t, tc.log, gitLog(dst)) + for name, content := range tc.files { + body, err := os.ReadFile(filepath.Join(dst, name)) + assert.Equal(t, content, strings.TrimSpace(string(body)), name) + assert.Equal(t, content == "", os.IsNotExist(err), name) + } + assert.Equal(t, tc.files["marker"] != "", fileExists(filepath.Join(dst, "marker"))) + for key, value := range tc.config { + assert.Equal(t, value, gitOutput(dst, "config", key), key) + } + if tc.refs != nil { + assert.Subset(t, strings.Fields(runGit(t, "-C", dst, "for-each-ref", "--format=%(refname)")), tc.refs) + } + if tc.log == nil { + assert.Contains(t, string(outputs), "commit="+sha+"\n") + assert.Contains(t, string(outputs), "ref="+cmp.Or(tc.eventRef, "refs/heads/main")+"\n") + return + } + if tc.head == "" { + assert.Empty(t, gitOutput(dst, "symbolic-ref", "-q", "HEAD")) + } else { + assert.Equal(t, "refs/heads/"+tc.head, gitOutput(dst, "symbolic-ref", "-q", "HEAD")) + assert.Equal(t, "origin/"+tc.head, gitOutput(dst, "rev-parse", "--abbrev-ref", "@{u}")) + } + assert.Contains(t, string(outputs), "commit="+gitOutput(dst, "rev-parse", "HEAD")+"\n") + if inputRef := tc.inputs["ref"]; inputRef != "" && inputRef != sha { + assert.Contains(t, string(outputs), "ref="+inputRef+"\n") + } + assert.Equal(t, tc.inputs["sparse-checkout"] != "", gitOutput(dst, "config", "--local", "extensions.worktreeConfig") == "true") + persisted := tc.inputs["persist-credentials"] != "false" + for _, repo := range repos { + assert.Equal(t, tc.inputs["clean"] == "false", fileExists(filepath.Join(repo, "untracked")), repo) + assert.Len(t, strings.Fields(gitOutput(repo, "config", "--local", "--get-all", credentialsInclude)), 1, repo) + headers, _ := exec.Command("git", "-C", repo, "config", "--get-all", "http.file://"+server+"/.extraheader").Output() + assert.Equal(t, persisted, string(headers) == "\nAuthorization: Basic dG9rZW46dG9r\n", repo) + } + if tc.inputs["ssh-key"] != "" { + keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-ssh-key-*")) + require.Len(t, keys, 1) + key, _ := os.ReadFile(keys[0]) + if persisted { + assert.Equal(t, "KEY\n", string(key)) + } else { + assert.Empty(t, key) + } + args, _ := os.ReadFile(filepath.Join(runnerTemp, "ssh-args")) + assert.Contains(t, string(args), "StrictHostKeyChecking=yes") + assert.Equal(t, tc.inputs["ssh-known-hosts"] != "", strings.Contains(string(args), "UserKnownHostsFile=")) + assert.Equal(t, persisted, gitOutput(dst, "config", "core.sshCommand") != "") } }) } } + +func publish(t *testing.T, server, name string, files, submodules map[string]string) { + work := t.TempDir() + runGit(t, "init", "--initial-branch=main", work) + for file, content := range files { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(work, file)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(work, file), []byte(content), 0o644)) + } + runGit(t, "-C", work, "add", ".") + for path, url := range submodules { + runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".path", path) + runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".url", url) + commit := runGit(t, "--git-dir", filepath.Join(server, name, url), "rev-parse", "main") + runGit(t, "-C", work, "update-index", "--add", "--cacheinfo", "160000,"+commit+","+path) + runGit(t, "-C", work, "add", ".gitmodules") + } + runGit(t, "-C", work, "commit", "-m", "content") + runGit(t, "clone", "--bare", "--quiet", work, filepath.Join(server, name)) +} + +func gitOutput(dir string, args ...string) string { + out, _ := exec.Command("git", append([]string{"-C", dir}, args...)...).Output() + return strings.TrimSpace(string(out)) +} + +func fileExists(name string) bool { + _, err := os.Stat(name) + return err == nil +}