runner/act/common/git/git.go

// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package git

import (
	"context"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"slices"
	"strconv"
	"strings"
	"time"

	"gitea.com/gitea/runner/act/common"
	"gitea.com/gitea/runner/act/common/logging"
	"gitea.com/gitea/runner/internal/pkg/lock"
)

var (
	codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
	codeCommitSSHRegex  = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
	hexRefRegex         = regexp.MustCompile(`^[0-9a-fA-F]+$`)

	LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
		"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
		"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
		"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
	}

	cloneLocks lock.Keyed[string] // key: clone target directory

	ErrShortRef = errors.New("short SHA references are not supported")
)

// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
	dir = filepath.Clean(dir)
	if runtime.GOOS == "plan9" {
		return cloneLocks.Lock(dir), nil
	}
	if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
		return nil, err
	}
	for {
		release, err := lock.TryLock(dir)
		if err == nil {
			return func() { _ = release() }, nil
		}
		if !errors.Is(err, lock.ErrLocked) {
			return cloneLocks.Lock(dir), nil
		}
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-time.After(50 * time.Millisecond):
		}
	}
}

type Error struct {
	err    error
	commit string
}

func (e *Error) Error() string  { return e.err.Error() }
func (e *Error) Unwrap() error  { return e.err }
func (e *Error) Commit() string { return e.commit }

// FindGitRevision get the current git revision
func FindGitRevision(ctx context.Context, file string) (string, error) {
	logger := common.Logger(ctx)
	sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
	if err != nil {
		logger.WithError(err).Error("path", file, "not located inside a git repository")
		return "", err
	}
	if len(sha) < 7 {
		return "", errors.New("HEAD could not be resolved")
	}
	logger.Debugf("Found revision: %s", sha)
	return sha, nil
}

// FindGitRef get the current git ref
func FindGitRef(ctx context.Context, file string) (string, error) {
	logger := common.Logger(ctx)
	logger.Debugf("Loading revision from git directory")
	revision, err := FindGitRevision(ctx, file)
	if err != nil {
		return "", err
	}
	logger.Debugf("HEAD points to '%s'", revision)

	refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
	if err != nil {
		return "", err
	}
	var tag, branch string
	for ref := range strings.FieldsSeq(refs) {
		switch {
		case strings.HasPrefix(ref, "refs/tags/"):
			tag = ref
		case strings.HasPrefix(ref, "refs/heads/"):
			branch = ref
		}
	}
	if tag != "" {
		return tag, nil
	}
	if branch != "" {
		return branch, nil
	}
	return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
}

// FindGithubRepo get the repo
func FindGithubRepo(ctx context.Context, file string) (string, error) {
	url, err := findGitRemoteURL(ctx, file)
	if err != nil {
		return "", err
	}
	return findGitSlug(url)
}

func findGitRemoteURL(ctx context.Context, file string) (string, error) {
	urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
	if err != nil {
		return "", err
	}
	if urls == "" {
		return "", errors.New("remote 'origin' exists but has no URL")
	}
	url, _, _ := strings.Cut(urls, "\n")
	return url, nil
}

func findGitSlug(remoteURL string) (string, error) {
	if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
		return matches[2], nil
	} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
		return matches[2], nil
	}
	if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
		remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
	}
	parsed, err := url.Parse(remoteURL)
	if err != nil {
		return "", errors.New("cannot parse the origin remote URL")
	}
	owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
	owner = path.Base(owner)
	if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
		return "", errors.New("cannot determine owner and repository from the origin remote URL")
	}
	return owner + "/" + repo, nil
}

// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
type NewGitCloneExecutorInput struct {
	URL         string
	Ref         string
	Dir         string
	Token       string
	OfflineMode bool

	// Depth limits the clone/fetch to the given number of commits from the tip of the requested ref.
	// 0 for full clone.
	Depth int

	// For Gitea
	InsecureSkipTLS               bool
	ClientCertFile, ClientKeyFile string            // PEM, absolute as git runs in other directories
	Transport                     http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
}

// CloneIfRequired reports whether an existing local clone was reused.
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
	var origin string
	_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
	if err == nil {
		origin, err = findGitRemoteURL(ctx, input.Dir)
	}
	if err == nil && origin == input.URL {
		return true, nil
	}
	if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
		return false, ctxErr
	}
	if err == nil {
		logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
	} else if _, statErr := os.Stat(input.Dir); statErr == nil {
		logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
	}
	if err := os.RemoveAll(input.Dir); err != nil {
		return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
	}
	if err := clone(ctx, input); err != nil {
		return false, err
	}
	if err := os.Chmod(input.Dir, 0o755); err != nil {
		return false, err
	}
	return false, nil
}

// NewGitCloneExecutor creates an executor to clone git repos
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
	return func(ctx context.Context) error {
		logger := common.Logger(ctx)
		logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
		logger.Debugf("  cloning %s to %s", input.URL, input.Dir)
		unlock, err := AcquireCloneLock(ctx, input.Dir)
		if err != nil {
			return err
		}
		defer unlock()

		resolved, reused, err := resolveClone(ctx, input)
		if err != nil {
			return err
		}
		if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
			return fmt.Errorf("checkout %s: %w", resolved, err)
		}
		reusedMsg := ""
		if input.OfflineMode && reused {
			reusedMsg = " (reused in offline mode)"
		}
		logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
		logger.Debugf("Checked out %s", input.Ref)
		return nil
	}
}

func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
	logger := common.Logger(ctx)
	reused, err := CloneIfRequired(ctx, input, logger)
	if err != nil {
		return "", false, err
	}
	resolved, err := resolveRef(ctx, input.Dir, input.Ref)
	if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
		return "", false, err
	}
	if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
		if err := fetch(ctx, input); err != nil {
			return "", false, err
		}
		resolved, err = resolveRef(ctx, input.Dir, input.Ref)
	}
	if err != nil {
		logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
	}
	return resolved, reused, err
}

func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
	if input.Depth > 0 && isFullObjectID(input.Ref) {
		if err := shallowPinnedClone(ctx, input); err == nil {
			return nil
		} else if err := removePartialClone(input.Dir); err != nil {
			return err
		}
	}
	args := []string{"clone", "--no-checkout"}
	if input.Depth > 0 {
		args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
		if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
			args = append(args, "--branch", input.Ref)
		}
	}
	args = append(args, input.URL, input.Dir)
	if _, err := runGit(ctx, "", &input, args...); err == nil {
		return nil
	} else if input.Depth == 0 {
		return err
	}
	if err := removePartialClone(input.Dir); err != nil {
		return err
	}
	_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
	return err
}

func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
	if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
		return err
	}
	if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
		return err
	}
	_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
	return err
}

func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
	args := []string{"fetch", "--force", "--no-prune"}
	if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
		args = append(args, "--depth", "1", "--no-tags")
	}
	_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
	return err
}

func cachedRef(ref string) string {
	return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
}

func resolveRef(ctx context.Context, dir, ref string) (string, error) {
	if !isFullObjectID(ref) {
		revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
		if strings.ContainsAny(ref, "\x00\n") {
			for _, revision := range revisions {
				if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
					return sha, nil
				}
			}
		} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
			for sha := range strings.SplitSeq(string(output), "\n") {
				if isFullObjectID(sha) {
					return sha, nil
				}
			}
		}
	}
	sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
	if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
		return "", &Error{err: ErrShortRef, commit: sha}
	}
	return sha, err
}

func isFullObjectID(ref string) bool {
	return ObjectFormat(ref) != ""
}

func isFullHash(ref, resolved string) bool {
	return isFullObjectID(ref) && len(ref) == len(resolved)
}

// ObjectFormat returns the object format of a full commit hash, empty for anything else.
func ObjectFormat(sha string) string {
	switch {
	case !hexRefRegex.MatchString(sha):
		return ""
	case len(sha) == 40:
		return "sha1"
	case len(sha) == 64:
		return "sha256"
	}
	return ""
}

func removePartialClone(dir string) error {
	if err := os.RemoveAll(dir); err != nil {
		return fmt.Errorf("remove partial clone %s: %w", dir, err)
	}
	return nil
}

func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
	output, err := runGit(ctx, dir, nil, args...)
	return strings.TrimSpace(string(output)), err
}

func RemoteConfig(token string, insecureSkipTLS bool) []string {
	var config []string
	if insecureSkipTLS {
		config = append(config, "http.sslVerify=false")
	}
	if token != "" {
		config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
	}
	return config
}

// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
func ConfigEnv(config ...string) map[string]string {
	env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
	for i, entry := range config {
		key, value, _ := strings.Cut(entry, "=")
		env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
		env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
	}
	return env
}

func ListFiles(ctx context.Context, dir string) ([]string, error) {
	listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
	if err != nil {
		return nil, err
	}
	var files, repos []string
	for entry := range strings.SplitSeq(string(listed), "\x00") {
		tag, file, ok := strings.Cut(entry, " ")
		if tag != "?" {
			var meta string
			meta, file, ok = strings.Cut(file, "\t")
			if strings.HasPrefix(meta, "160000 ") {
				file += "/"
			}
		}
		if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
			repos = append(repos, repo)
		} else if ok {
			files = append(files, file)
		}
	}
	slices.Sort(repos)
	for _, repo := range slices.Compact(repos) {
		files = append(files, repo)
		if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
			continue // git lists an uninitialized submodule as "./" from inside it
		}
		nested, err := ListFiles(ctx, filepath.Join(dir, repo))
		if err != nil {
			return nil, err
		}
		for _, file := range nested {
			files = append(files, repo+"/"+file)
		}
	}
	return files, nil
}

func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
	return runGitWithStdin(ctx, dir, input, nil, args...)
}

func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
	config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
	if input != nil {
		config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
		if input.ClientCertFile != "" {
			config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
		}
	}
	if dir != "" {
		args = append([]string{"-C", dir}, args...)
	}
	cmd := exec.CommandContext(ctx, "git", args...)
	cmd.Stdin = stdin
	cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
		name, _, _ := strings.Cut(entry, "=")
		return slices.Contains(LocalEnvVars, name)
	})
	for key, value := range ConfigEnv(config...) {
		cmd.Env = append(cmd.Env, key+"="+value)
	}
	cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
	if runtime.GOOS != "windows" {
		cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
	}
	cmd.WaitDelay = time.Second
	var stderr strings.Builder
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
	}
	return output, nil
}