mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 06:45:44 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b14a38d4a | ||
|
|
179e435cd0 | ||
|
|
f963e9cfe8 | ||
|
|
e5aa9add02 | ||
|
|
329115b148 | ||
|
|
ff30f39b1b | ||
|
|
3802feb3fe | ||
|
|
27e9b4de87 | ||
|
|
4bedadef9d | ||
|
|
83b63250a3 | ||
|
|
2e59141ae9 | ||
|
|
020f0d2861 | ||
|
|
711b72735b | ||
|
|
436d0d2942 | ||
|
|
a284a62d96 | ||
|
|
2b1e64e414 |
No files matched your search
@@ -1,9 +1,7 @@
|
||||
name: checks
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
# The runner image ships a stale docker.io login; point docker at an empty config so
|
||||
|
||||
@@ -9,7 +9,7 @@ XGO_VERSION := go-1.27.x
|
||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
|
||||
@@ -24,7 +24,7 @@ DOCKER_TAG ?= nightly
|
||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
||||
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]3.2 # renovate: datasource=go
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/[email protected] # renovate: datasource=go
|
||||
|
||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||
|
||||
@@ -4,7 +4,19 @@
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
On the runner:
|
||||
|
||||
- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows.
|
||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
|
||||
Where steps run, in the job image or on the runner in host mode:
|
||||
|
||||
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
|
||||
- `node` for JavaScript actions and `hashFiles()`.
|
||||
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
|
||||
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
|
||||
|
||||
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
|
||||
|
||||
### Download pre-built binary
|
||||
|
||||
@@ -157,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
|
||||
|
||||
#### Environment variables
|
||||
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
|
||||
### Labels
|
||||
|
||||
@@ -205,7 +217,7 @@ a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubunt
|
||||
|
||||
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
|
||||
|
||||
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs in `runner.default_image`: as a container when the runner has `docker` labels, as a pod when it has `kubernetes` labels, else as a container where docker is available and on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
|
||||
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
|
||||
|
||||
@@ -252,7 +264,7 @@ volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. The proxy removes containers, networks and volumes after post steps and the completed hook. Named volumes created through it are job-scoped. To retain resources, mount the daemon socket explicitly in `container.options`. Host jobs use their existing Docker access, where `.` works.
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
#### Proxy
|
||||
|
||||
|
||||
+212
-374
@@ -6,28 +6,23 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nethttp "net/http"
|
||||
"os"
|
||||
"path"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/storer"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/mattn/go-isatty"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -36,26 +31,19 @@ var (
|
||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||
|
||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
|
||||
"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
|
||||
"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
|
||||
}
|
||||
|
||||
cloneLocks lock.Keyed[string] // key: clone target directory
|
||||
|
||||
ErrShortRef = errors.New("short SHA references are not supported")
|
||||
)
|
||||
|
||||
func init() {
|
||||
httpClient := newHTTPClient(&nethttp.HTTP2Config{SendPingTimeout: 30 * time.Second})
|
||||
client.InstallProtocol("http", httpClient)
|
||||
client.InstallProtocol("https", httpClient)
|
||||
}
|
||||
|
||||
// newHTTPClient pings idle HTTP/2 connections to drop stalled ones.
|
||||
func newHTTPClient(h2 *nethttp.HTTP2Config) transport.Transport {
|
||||
defaultTransport, _ := nethttp.DefaultTransport.(*nethttp.Transport)
|
||||
httpTransport := defaultTransport.Clone()
|
||||
httpTransport.HTTP2 = h2
|
||||
return http.NewClient(&nethttp.Client{Transport: httpTransport})
|
||||
}
|
||||
|
||||
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
|
||||
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
|
||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
||||
@@ -69,146 +57,60 @@ type Error struct {
|
||||
commit string
|
||||
}
|
||||
|
||||
func (e *Error) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e *Error) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func (e *Error) Commit() string {
|
||||
return e.commit
|
||||
}
|
||||
|
||||
// goGitMu serializes go-git repository access across the process. go-git is not safe for
|
||||
// concurrent use of the same repository (even read access decodes packfiles into shared
|
||||
// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded
|
||||
// operations are fast local reads; gitea runs one job per process, so the lock is effectively
|
||||
// uncontended in production.
|
||||
var goGitMu sync.Mutex
|
||||
func (e *Error) Error() string { return e.err.Error() }
|
||||
func (e *Error) Unwrap() error { return e.err }
|
||||
func (e *Error) Commit() string { return e.commit }
|
||||
|
||||
// FindGitRevision get the current git revision
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
return findGitRevision(ctx, file)
|
||||
}
|
||||
|
||||
func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
gitDir, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
head, err := gitDir.Reference(plumbing.HEAD, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
if len(sha) < 7 {
|
||||
return "", "", errors.New("HEAD could not be resolved")
|
||||
}
|
||||
|
||||
if head.Hash().IsZero() {
|
||||
return "", "", errors.New("HEAD sha1 could not be resolved")
|
||||
}
|
||||
|
||||
hash := head.Hash().String()
|
||||
|
||||
logger.Debugf("Found revision: %s", hash)
|
||||
return hash[:7], strings.TrimSpace(hash), nil
|
||||
logger.Debugf("Found revision: %s", sha)
|
||||
return sha[:7], sha, nil
|
||||
}
|
||||
|
||||
// FindGitRef get the current git ref
|
||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
logger.Debugf("Loading revision from git directory")
|
||||
_, ref, err := findGitRevision(ctx, file)
|
||||
_, revision, err := FindGitRevision(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
logger.Debugf("HEAD points to '%s'", revision)
|
||||
|
||||
logger.Debugf("HEAD points to '%s'", ref)
|
||||
|
||||
// Prefer the git library to iterate over the references and find a matching tag or branch.
|
||||
refTag := ""
|
||||
refBranch := ""
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
iter, err := repo.References()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// find the reference that matches the revision's has
|
||||
err = iter.ForEach(func(r *plumbing.Reference) error {
|
||||
/* tags and branches will have the same hash
|
||||
* when a user checks out a tag, it is not mentioned explicitly
|
||||
* in the go-git package, we must identify the revision
|
||||
* then check if any tag matches that revision,
|
||||
* if so then we checked out a tag
|
||||
* else we look for branches and if matches,
|
||||
* it means we checked out a branch
|
||||
*
|
||||
* If a branches matches first we must continue and check all tags (all references)
|
||||
* in case we match with a tag later in the interation
|
||||
*/
|
||||
if r.Hash().String() == ref {
|
||||
if r.Name().IsTag() {
|
||||
refTag = r.Name().String()
|
||||
}
|
||||
if r.Name().IsBranch() {
|
||||
refBranch = r.Name().String()
|
||||
}
|
||||
var tag, branch string
|
||||
for ref := range strings.FieldsSeq(refs) {
|
||||
switch {
|
||||
case strings.HasPrefix(ref, "refs/tags/"):
|
||||
tag = ref
|
||||
case strings.HasPrefix(ref, "refs/heads/"):
|
||||
branch = ref
|
||||
}
|
||||
|
||||
// we found what we where looking for
|
||||
if refTag != "" && refBranch != "" {
|
||||
return storer.ErrStop
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// order matters here see above comment.
|
||||
if refTag != "" {
|
||||
return refTag, nil
|
||||
if tag != "" {
|
||||
return tag, nil
|
||||
}
|
||||
if refBranch != "" {
|
||||
return refBranch, nil
|
||||
if branch != "" {
|
||||
return branch, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
|
||||
}
|
||||
|
||||
// FindGithubRepo get the repo
|
||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
url, err := findGitRemoteURL(ctx, file, "origin")
|
||||
url, err := findGitRemoteURL(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -216,28 +118,16 @@ func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, e
|
||||
return slug, nil
|
||||
}
|
||||
|
||||
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||
urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
remote, err := repo.Remote(remoteName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
if urls == "" {
|
||||
return "", errors.New("remote 'origin' exists but has no URL")
|
||||
}
|
||||
|
||||
if len(remote.Config().URLs) < 1 {
|
||||
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)
|
||||
}
|
||||
|
||||
return remote.Config().URLs[0], nil
|
||||
url, _, _ := strings.Cut(urls, "\n")
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func findGitSlug(url, githubInstance string) (string, string) {
|
||||
@@ -278,69 +168,35 @@ type NewGitCloneExecutorInput struct {
|
||||
Quiet bool
|
||||
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
}
|
||||
|
||||
// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) {
|
||||
r, err := git.PlainOpen(input.Dir)
|
||||
// CloneIfRequired reports whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) {
|
||||
var origin string
|
||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||
if err == nil {
|
||||
// Verify the cached clone still points to the resolved URL before reusing it.
|
||||
remote, err := r.Remote("origin")
|
||||
if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL {
|
||||
// Reuse existing clone
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
case len(remote.Config().URLs) == 0:
|
||||
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
|
||||
default:
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
|
||||
}
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
origin, err = findGitRemoteURL(ctx, input.Dir)
|
||||
}
|
||||
|
||||
var progressWriter io.Writer
|
||||
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
|
||||
if entry, ok := logger.(*log.Entry); ok {
|
||||
progressWriter = entry.WriterLevel(log.DebugLevel)
|
||||
} else if lgr, ok := logger.(*log.Logger); ok {
|
||||
progressWriter = lgr.WriterLevel(log.DebugLevel)
|
||||
} else {
|
||||
log.Errorf("Unable to get writer from logger (type=%T)", logger)
|
||||
progressWriter = os.Stdout
|
||||
}
|
||||
if err == nil && origin == input.URL {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
cloneOptions := git.CloneOptions{
|
||||
URL: input.URL,
|
||||
Progress: progressWriter,
|
||||
|
||||
InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea
|
||||
if err == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
}
|
||||
if input.Token != "" {
|
||||
cloneOptions.Auth = &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: input.Token,
|
||||
}
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
|
||||
r, err = cloneAtDepth(ctx, input, cloneOptions, logger)
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err)
|
||||
return nil, false, err
|
||||
if err := clone(ctx, input); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err = os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return nil, false, err
|
||||
if err := os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return r, false, nil
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutor creates an executor to clone git repos
|
||||
@@ -353,134 +209,125 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||
}
|
||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||
|
||||
defer AcquireCloneLock(input.Dir)()
|
||||
|
||||
refName := plumbing.ReferenceName("refs/heads/" + input.Ref)
|
||||
r, reused, err := CloneIfRequired(ctx, refName, input, logger)
|
||||
reused, err := CloneIfRequired(ctx, input, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isOfflineMode := input.OfflineMode
|
||||
|
||||
fetchOptions := git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD", "refs/heads/*:refs/remotes/origin/*"},
|
||||
Force: true,
|
||||
InsecureSkipTLS: input.InsecureSkipTLS,
|
||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||
return err
|
||||
}
|
||||
if input.Token != "" {
|
||||
fetchOptions.Auth = &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: input.Token,
|
||||
}
|
||||
}
|
||||
|
||||
// Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth
|
||||
// Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone.
|
||||
if isShallow(r) {
|
||||
fetchOptions.Depth = 1
|
||||
if spec, ok := shallowFetchRefSpec(r, input.Ref); ok {
|
||||
fetchOptions.RefSpecs = []config.RefSpec{spec}
|
||||
}
|
||||
}
|
||||
|
||||
// A just-cloned ref is as current as a fetch would make it, and a commit hash never moves.
|
||||
// TODO: revalidate a mutable ref with a conditional archive request instead, once every
|
||||
// supported Gitea sends an ETag for them: https://github.com/go-gitea/gitea/pull/39289
|
||||
_, present := refRevision(r, input.Ref)
|
||||
refresh := !isOfflineMode && (!present || (reused && !plumbing.IsHash(input.Ref)))
|
||||
|
||||
if refresh {
|
||||
err = r.FetchContext(ctx, &fetchOptions)
|
||||
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||
if err := fetch(ctx, input); err != nil {
|
||||
return err
|
||||
}
|
||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||
}
|
||||
|
||||
var hash *plumbing.Hash
|
||||
rev := plumbing.Revision(input.Ref)
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
// ResolveRevision returns a nil hash on error, and a branch ref legitimately fails
|
||||
// here (no local refs/heads/<ref>); the duck-typing below resolves it.
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
} else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) {
|
||||
return &Error{
|
||||
err: ErrShortRef,
|
||||
commit: hash.String(),
|
||||
}
|
||||
}
|
||||
|
||||
rev, _ = refRevision(r, input.Ref)
|
||||
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
return err
|
||||
}
|
||||
|
||||
var w *git.Worktree
|
||||
if w, err = r.Worktree(); err != nil {
|
||||
return err
|
||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||
}
|
||||
|
||||
reusedMsg := ""
|
||||
if isOfflineMode && reused {
|
||||
if input.OfflineMode && reused {
|
||||
reusedMsg = " (reused in offline mode)"
|
||||
}
|
||||
|
||||
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
|
||||
|
||||
if err = w.Checkout(&git.CheckoutOptions{
|
||||
Hash: *hash,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
logger.Errorf("Unable to checkout %s: %v", *hash, err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Checked out %s", input.Ref)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// cloneAtDepth falls back to a full clone when the shallow attempt fails.
|
||||
func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
|
||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||
return nil
|
||||
} else if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
args := []string{"clone", "--no-checkout"}
|
||||
if input.Depth > 0 {
|
||||
if plumbing.IsHash(input.Ref) {
|
||||
r, err := fetchPinnedSHA(ctx, input, opts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow fetch of %s at %s failed: %v", input.URL, input.Ref, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
refNames := []plumbing.ReferenceName{plumbing.NewBranchReferenceName(input.Ref), plumbing.NewTagReferenceName(input.Ref)}
|
||||
if input.Ref == plumbing.HEAD.String() || strings.HasPrefix(input.Ref, "refs/") {
|
||||
refNames = []plumbing.ReferenceName{plumbing.ReferenceName(input.Ref)} // HEAD is the remote's default branch
|
||||
}
|
||||
for _, refName := range refNames {
|
||||
shallowOpts := opts
|
||||
shallowOpts.Depth = input.Depth
|
||||
shallowOpts.SingleBranch = true
|
||||
shallowOpts.ReferenceName = refName
|
||||
shallowOpts.Tags = git.NoTags
|
||||
args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
|
||||
if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
|
||||
args = append(args, "--branch", input.Ref)
|
||||
}
|
||||
}
|
||||
args = append(args, input.URL, input.Dir)
|
||||
if _, err := runGit(ctx, "", &input, args...); err == nil {
|
||||
return nil
|
||||
} else if input.Depth == 0 {
|
||||
return err
|
||||
}
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
|
||||
return err
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
args := []string{"fetch", "--force", "--prune"}
|
||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||
args = append(args, "--depth", "1", "--no-tags")
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
|
||||
return err
|
||||
}
|
||||
|
||||
func cachedRef(ref string) string {
|
||||
return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
|
||||
}
|
||||
|
||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||
if !isFullObjectID(ref) {
|
||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
||||
return sha, nil
|
||||
}
|
||||
}
|
||||
logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
|
||||
}
|
||||
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
|
||||
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
|
||||
return "", &Error{err: ErrShortRef, commit: sha}
|
||||
}
|
||||
return sha, err
|
||||
}
|
||||
|
||||
return git.PlainCloneContext(ctx, input.Dir, false, &opts)
|
||||
func isFullObjectID(ref string) bool {
|
||||
return ObjectFormat(ref) != ""
|
||||
}
|
||||
|
||||
func isFullHash(ref, resolved string) bool {
|
||||
return isFullObjectID(ref) && len(ref) == len(resolved)
|
||||
}
|
||||
|
||||
// ObjectFormat returns the object format of a full commit hash, empty for anything else.
|
||||
func ObjectFormat(sha string) string {
|
||||
switch {
|
||||
case !hexRefRegex.MatchString(sha):
|
||||
return ""
|
||||
case len(sha) == 40:
|
||||
return "sha1"
|
||||
case len(sha) == 64:
|
||||
return "sha256"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func removePartialClone(dir string) error {
|
||||
@@ -490,71 +337,62 @@ func removePartialClone(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchPinnedSHA(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions) (*git.Repository, error) {
|
||||
r, err := git.PlainInit(input.Dir, false)
|
||||
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
|
||||
output, err := runGit(ctx, dir, nil, args...)
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
var config []string
|
||||
if insecureSkipTLS {
|
||||
config = append(config, "http.sslVerify=false")
|
||||
}
|
||||
if token != "" {
|
||||
config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
for i, entry := range config {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
|
||||
env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||
name, _, _ := strings.Cut(entry, "=")
|
||||
return slices.Contains(LocalEnvVars, name)
|
||||
})
|
||||
for key, value := range ConfigEnv(config...) {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{input.URL}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.FetchContext(ctx, &git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{pinnedRefSpec(input.Ref)},
|
||||
Depth: input.Depth,
|
||||
Tags: git.NoTags,
|
||||
Auth: opts.Auth,
|
||||
Progress: opts.Progress,
|
||||
InsecureSkipTLS: opts.InsecureSkipTLS,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// pinnedRefSpec keeps a hash-shaped name out of refs/heads and refs/tags.
|
||||
func pinnedRefSpec(sha string) config.RefSpec {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:refs/pinned/%s", sha, sha))
|
||||
}
|
||||
|
||||
// refRevision picks the revision to check out and reports whether it resolves locally.
|
||||
func refRevision(r *git.Repository, ref string) (plumbing.Revision, bool) {
|
||||
if plumbing.IsHash(ref) {
|
||||
// git ignores a ref named as 40 hex digits, so a full hash always denotes the commit itself.
|
||||
_, err := r.CommitObject(plumbing.NewHash(ref))
|
||||
return plumbing.Revision(ref), err == nil
|
||||
}
|
||||
if _, err := r.Tag(ref); err == nil {
|
||||
return plumbing.Revision(path.Join("refs", "tags", ref)), true
|
||||
}
|
||||
remoteRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", ref))
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
return plumbing.Revision(remoteRef), true
|
||||
}
|
||||
rev := plumbing.Revision(ref)
|
||||
_, err := r.ResolveRevision(rev)
|
||||
return rev, err == nil
|
||||
}
|
||||
|
||||
// isShallow reports whether the local repository was cloned with a limited depth.
|
||||
func isShallow(r *git.Repository) bool {
|
||||
shallows, err := r.Storer.Shallow()
|
||||
return err == nil && len(shallows) > 0
|
||||
}
|
||||
|
||||
// shallowFetchRefSpec limits a shallow update to the requested ref, falling back to the broad refspec when it is not present locally.
|
||||
func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
|
||||
tagRef := plumbing.NewTagReferenceName(ref)
|
||||
if _, err := r.Reference(tagRef, false); err == nil {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true
|
||||
}
|
||||
remoteRef := plumbing.NewRemoteReferenceName("origin", ref)
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
branchRef := plumbing.NewBranchReferenceName(ref)
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
|
||||
}
|
||||
if plumbing.IsHash(ref) {
|
||||
// The broad refspec carries only advertised tips, never a pinned commit.
|
||||
return pinnedRefSpec(ref), true
|
||||
}
|
||||
return "", false
|
||||
return output, nil
|
||||
}
|
||||
+111
-165
@@ -6,9 +6,8 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -17,18 +16,12 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
gogitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||
gogitfile "github.com/go-git/go-git/v5/plumbing/transport/file"
|
||||
log "github.com/sirupsen/logrus"
|
||||
logrustest "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -90,14 +83,27 @@ func cleanGitHooks(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFindGithubRepoUsesOrigin(t *testing.T) {
|
||||
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
const remoteURL = "https://github.com/owner/repo.git"
|
||||
require.NoError(t, gitCmd("init", basedir))
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, cleanGitHooks(basedir))
|
||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||
|
||||
url, err := findGitRemoteURL(context.Background(), basedir, "origin")
|
||||
shortSHA, sha, err := FindGitRevision(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
||||
require.Len(t, sha, 64)
|
||||
require.Equal(t, sha[:7], shortSHA)
|
||||
|
||||
ref, err := FindGitRef(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/main", ref)
|
||||
|
||||
url, err := findGitRemoteURL(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, remoteURL, url)
|
||||
|
||||
@@ -107,6 +113,7 @@ func TestFindGithubRepoUsesOrigin(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitFindRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
@@ -193,7 +200,9 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||
@@ -207,6 +216,13 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
fullSha := strings.TrimSpace(string(out))
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Err error
|
||||
Ref string
|
||||
@@ -227,26 +243,42 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
Err: &Error{ErrShortRef, fullSha},
|
||||
Ref: fullSha[:7],
|
||||
},
|
||||
"HEAD": {
|
||||
Ref: "HEAD",
|
||||
},
|
||||
"hex branch named like its own commit": {
|
||||
Ref: fullSha[:4],
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
URL: "https://example.invalid/action",
|
||||
Ref: tt.Ref,
|
||||
Dir: t.TempDir(),
|
||||
Dir: dir,
|
||||
})
|
||||
|
||||
err := clone(context.Background())
|
||||
if tt.Err != nil {
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, err)
|
||||
} else {
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, clone(context.Background()), "a retry on the cache keeps the error")
|
||||
return
|
||||
}
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
marker := filepath.Join(dir, "marker")
|
||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||
require.NoError(t, clone(context.Background()))
|
||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
createRemote := func(message string) string {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -270,6 +302,11 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
nested := filepath.Join(cacheDir, "nested")
|
||||
require.NoError(t, os.Mkdir(nested, 0o755))
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context()))
|
||||
assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository")
|
||||
|
||||
markerPath := filepath.Join(cacheDir, "stale-marker")
|
||||
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
|
||||
|
||||
@@ -279,7 +316,7 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin")
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, newRemoteDir, originURL)
|
||||
|
||||
@@ -291,61 +328,36 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning")
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorNonFastForwardRef(t *testing.T) {
|
||||
// Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes
|
||||
// non-fast-forward between two fetches. Before the fix, the fetch used Force=false,
|
||||
// causing go-git to return ErrForceNeeded and short-circuit the checkout.
|
||||
|
||||
// Create a bare "remote" repo with an initial commit on main and a feature branch.
|
||||
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
// We need a working clone to push commits from.
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Create a feature branch (simulates refs/pull/N/head).
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature"))
|
||||
|
||||
// First clone via the executor — should succeed and cache the repo.
|
||||
cloneDir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: cloneDir,
|
||||
})
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
|
||||
require.NoError(t, clone(context.Background()))
|
||||
|
||||
// Now force-push the feature branch to a non-fast-forward commit (simulates
|
||||
// a PR rebase). This makes refs/heads/feature non-fast-forward.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
|
||||
require.NoError(t, clone(context.Background()), "a non-fast-forward ref must update")
|
||||
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
|
||||
|
||||
// Also advance main so we can verify the clone picks up the new commit.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
|
||||
// Second clone to the same directory — before the fix this returned ErrForceNeeded
|
||||
// and left the working tree at the old commit.
|
||||
err := clone(context.Background())
|
||||
require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true")
|
||||
|
||||
// Verify the working tree was actually updated to the latest main commit.
|
||||
out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit")
|
||||
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: t.TempDir()})
|
||||
require.Error(t, tagClone(context.Background()))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
|
||||
require.NoError(t, tagClone(context.Background()), "a tag published after a failed lookup must be fetched")
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with a single commit on main.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -392,6 +404,7 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
@@ -430,6 +443,7 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -473,12 +487,11 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
return remote
|
||||
}
|
||||
|
||||
t.Run("commit hash falls back to a full clone when the remote refuses unadvertised objects", func(t *testing.T) {
|
||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.NoFileExists(t, shallowMarker(dir))
|
||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
@@ -505,76 +518,46 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
t.Run("moving branch updates while staying shallow", func(t *testing.T) {
|
||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1,
|
||||
})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
require.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
|
||||
// Advance main on the remote, then reuse the existing shallow clone.
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorTransportSessions(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--initial-branch=main", workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "-a", "v2", "-m", "v2"))
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Ref string
|
||||
Depth int
|
||||
}{
|
||||
"shallow branch": {"main", 1},
|
||||
"full clone branch": {"main", 0},
|
||||
"full lightweight tag": {"v1", 0},
|
||||
"full annotated tag": {"v2", 0},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
counter := installCountingTransport(t)
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: workDir, Ref: tt.Ref, Dir: dir, Depth: tt.Depth,
|
||||
})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Swap(0), "cold clone")
|
||||
assert.Equal(t, gitRevParse(t, workDir, tt.Ref+"^{commit}"), gitRevParse(t, dir, "HEAD"))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Swap(0), "unchanged warm cache")
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.yml"), []byte(name), 0o644))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.yml"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", name))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "-a", "v2", "-m", "v2"))
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("staged"), 0o644))
|
||||
require.NoError(t, gitCmd("-C", dir, "add", "action.yml"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("unstaged"), 0o644))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Load(), "updated warm cache")
|
||||
assert.Equal(t, gitRevParse(t, workDir, "HEAD"), gitRevParse(t, dir, "HEAD"))
|
||||
status, err := exec.Command("git", "-C", dir, "status", "--porcelain").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, string(status))
|
||||
})
|
||||
}
|
||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
||||
work := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
||||
sha := gitRevParse(t, work, "HEAD")
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir))
|
||||
require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef)
|
||||
require.NoError(t, os.RemoveAll(remote))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
@@ -598,25 +581,6 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type countingTransport struct {
|
||||
transport.Transport
|
||||
sessions atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingTransport) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) {
|
||||
c.sessions.Add(1)
|
||||
return c.Transport.NewUploadPackSession(ep, auth)
|
||||
}
|
||||
|
||||
func installCountingTransport(t *testing.T) *countingTransport {
|
||||
t.Helper()
|
||||
counter := &countingTransport{Transport: gogitfile.DefaultClient}
|
||||
gogitclient.InstallProtocol("file", counter)
|
||||
t.Cleanup(func() { gogitclient.InstallProtocol("file", gogitfile.DefaultClient) })
|
||||
|
||||
return counter
|
||||
}
|
||||
|
||||
func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()
|
||||
@@ -717,12 +681,17 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
}
|
||||
|
||||
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
|
||||
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
block := make(chan struct{})
|
||||
reached := make(chan struct{})
|
||||
var once sync.Once
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
||||
once.Do(func() { close(reached) })
|
||||
var authorization string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
|
||||
once.Do(func() {
|
||||
authorization = request.Header.Get("Authorization")
|
||||
close(reached)
|
||||
})
|
||||
<-block
|
||||
}))
|
||||
t.Cleanup(func() {
|
||||
@@ -731,17 +700,15 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
})
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||
repo, err := gogit.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gitCmd("init", dir))
|
||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx)
|
||||
}()
|
||||
|
||||
select {
|
||||
@@ -749,6 +716,7 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
@@ -758,25 +726,3 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
t.Fatal("fetch ignored context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewGitCloneExecutorFailsOnSilentHTTP2Connection(t *testing.T) {
|
||||
server := httptest.NewUnstartedServer(nil)
|
||||
server.EnableHTTP2 = true
|
||||
server.Config.TLSNextProto = map[string]func(*http.Server, *tls.Conn, http.Handler){
|
||||
"h2": func(_ *http.Server, conn *tls.Conn, _ http.Handler) { _, _ = io.Copy(io.Discard, conn) },
|
||||
}
|
||||
server.StartTLS()
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
previous := gogitclient.Protocols["https"]
|
||||
gogitclient.InstallProtocol("https", newHTTPClient(&http.HTTP2Config{SendPingTimeout: 50 * time.Millisecond, PingTimeout: 50 * time.Millisecond}))
|
||||
t.Cleanup(func() { gogitclient.InstallProtocol("https", previous) })
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: server.URL + "/action", Ref: "main", Dir: filepath.Join(t.TempDir(), "action"), InsecureSkipTLS: true,
|
||||
})(ctx)
|
||||
require.Error(t, err)
|
||||
require.NoError(t, ctx.Err())
|
||||
}
|
||||
@@ -87,45 +87,54 @@ func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExec
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return createDockerNetwork(ctx, cli, name, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var options map[string]string
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
// the daemon's mtu only reaches its default bridge, Docker's default is left out to keep default-network-opts
|
||||
if mtu := n.Options["com.docker.network.driver.mtu"]; n.Name == network.NetworkBridge && mtu != "" && mtu != "1500" {
|
||||
options = map[string]string{"com.docker.network.driver.mtu": mtu}
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Options: options,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
|
||||
func runnerLabels(runnerUUID string) map[string]string {
|
||||
|
||||
@@ -57,6 +57,27 @@ func TestIsAddressPoolExhausted(t *testing.T) {
|
||||
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
|
||||
}
|
||||
|
||||
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
bridgeOptions, want map[string]string
|
||||
}{
|
||||
{
|
||||
map[string]string{"com.docker.network.bridge.host_binding_ipv4": "127.0.0.1", "com.docker.network.driver.mtu": "1450"},
|
||||
map[string]string{"com.docker.network.driver.mtu": "1450"},
|
||||
},
|
||||
{map[string]string{"com.docker.network.driver.mtu": "1500"}, nil},
|
||||
} {
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).
|
||||
Return(mobyclient.NetworkListResult{Items: []network.Summary{{Name: network.NetworkBridge, Options: tc.bridgeOptions}}}, nil)
|
||||
client.On("NetworkCreate", ctx, "job-network", mobyclient.NetworkCreateOptions{Driver: "bridge", Scope: "local", Options: tc.want}).
|
||||
Return(mobyclient.NetworkCreateResult{}, nil)
|
||||
require.NoError(t, createDockerNetwork(ctx, client, "job-network", NewDockerNetworkCreateExecutorInput{}))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
|
||||
return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
|
||||
}
|
||||
|
||||
+57
-33
@@ -8,6 +8,7 @@ package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -1117,6 +1118,10 @@ func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *c
|
||||
// systempaths=unconfined lands in these two rather than in SecurityOpt
|
||||
resetOption(logger, "--security-opt", &hostConfig.MaskedPaths, trusted.MaskedPaths)
|
||||
resetOption(logger, "--security-opt", &hostConfig.ReadonlyPaths, trusted.ReadonlyPaths)
|
||||
resetOption(logger, "--log-driver", &hostConfig.LogConfig.Type, trusted.LogConfig.Type) // drivers run in the daemon, egressing from its network
|
||||
resetOption(logger, "--log-opt", &hostConfig.LogConfig.Config, trusted.LogConfig.Config)
|
||||
resetOption(logger, "--oom-score-adj", &hostConfig.OomScoreAdj, trusted.OomScoreAdj) // -1000 exempts it from the host OOM killer
|
||||
resetOption(logger, "--oom-kill-disable", &hostConfig.OomKillDisable, trusted.OomKillDisable)
|
||||
|
||||
// a driver mounts what it likes, e.g. local with device= binds any host path, which
|
||||
// valid_volumes never gets to see
|
||||
@@ -1158,9 +1163,9 @@ func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
|
||||
logger := common.Logger(ctx)
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
|
||||
if len(cr.input.ValidVolumes) > 0 {
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
// sanitize binds
|
||||
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
|
||||
for _, bind := range hostConfig.Binds {
|
||||
@@ -1174,33 +1179,37 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
continue
|
||||
}
|
||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type)) {
|
||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type), parsed.ReadOnly) {
|
||||
if matcher.needsNoCopy(parsed.Source, mount.Type(parsed.Type)) && (parsed.Volume == nil || !parsed.Volume.NoCopy) {
|
||||
bind += ",nocopy"
|
||||
}
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", parsed.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Binds = sanitizedBinds
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
if matcher.isValid(mt.Source, mt.Type) {
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
} else {
|
||||
for _, bind := range hostConfig.Binds {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", bind)
|
||||
}
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
hostConfig.Binds = []string{}
|
||||
}
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
readOnly := mt.ReadOnly && (mt.BindOptions == nil || !mt.BindOptions.ReadOnlyNonRecursive) // bind-recursive=writable leaves submounts writable
|
||||
if matcher.isValid(mt.Source, mt.Type, readOnly) {
|
||||
if matcher.needsNoCopy(mt.Source, mt.Type) {
|
||||
mt.VolumeOptions = cmp.Or(mt.VolumeOptions, &mount.VolumeOptions{})
|
||||
mt.VolumeOptions.NoCopy = true
|
||||
}
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
hostConfig.Binds = []string{}
|
||||
hostConfig.Mounts = []mount.Mount{}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
|
||||
return config, hostConfig
|
||||
}
|
||||
@@ -1240,26 +1249,37 @@ func overlayVolumes(dst, src *container.HostConfig) {
|
||||
}
|
||||
|
||||
type validVolumeMatcher struct {
|
||||
allowAll bool
|
||||
named []string
|
||||
host []string
|
||||
allowAll bool
|
||||
allowAllReadOnly bool
|
||||
named []validVolumeRule
|
||||
host []validVolumeRule
|
||||
}
|
||||
|
||||
type validVolumeRule struct {
|
||||
pattern string
|
||||
readOnly bool
|
||||
}
|
||||
|
||||
func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolumeMatcher {
|
||||
logger := common.Logger(ctx)
|
||||
ret := validVolumeMatcher{
|
||||
named: make([]string, 0, len(validVolumes)),
|
||||
host: make([]string, 0, len(validVolumes)),
|
||||
named: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
host: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
}
|
||||
|
||||
for _, v := range validVolumes {
|
||||
v, readOnly := strings.CutSuffix(v, ":ro")
|
||||
if v == "**" {
|
||||
ret.allowAll = true
|
||||
if readOnly {
|
||||
ret.allowAllReadOnly = true
|
||||
} else {
|
||||
ret.allowAll = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !isHostVolumePattern(v) {
|
||||
if doublestar.ValidatePattern(v) {
|
||||
ret.named = append(ret.named, v)
|
||||
ret.named = append(ret.named, validVolumeRule{pattern: v, readOnly: readOnly})
|
||||
} else {
|
||||
logger.Errorf("invalid volume pattern %s", v)
|
||||
}
|
||||
@@ -1271,7 +1291,7 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
|
||||
continue
|
||||
}
|
||||
if doublestar.ValidatePathPattern(normalized) {
|
||||
ret.host = append(ret.host, normalized)
|
||||
ret.host = append(ret.host, validVolumeRule{pattern: normalized, readOnly: readOnly})
|
||||
} else {
|
||||
logger.Errorf("invalid volume pattern %s", normalized)
|
||||
}
|
||||
@@ -1280,30 +1300,34 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
|
||||
return ret
|
||||
}
|
||||
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type) bool {
|
||||
if m.allowAll {
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type, readOnly bool) bool {
|
||||
if m.allowAll || (m.allowAllReadOnly && readOnly) {
|
||||
return true
|
||||
}
|
||||
if sourceType == mount.TypeTmpfs && source == "" { // backed by memory, exposes nothing from the host, same as --tmpfs
|
||||
return true
|
||||
}
|
||||
rules, match := m.named, doublestar.MatchUnvalidated
|
||||
if isHostVolumeSource(source, sourceType) {
|
||||
normalized, err := normalizeHostVolumePath(source)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, pattern := range m.host {
|
||||
if doublestar.PathMatchUnvalidated(pattern, normalized) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
rules, match, source = m.host, doublestar.PathMatchUnvalidated, normalized
|
||||
}
|
||||
for _, pattern := range m.named {
|
||||
if doublestar.MatchUnvalidated(pattern, source) {
|
||||
for _, rule := range rules {
|
||||
if (readOnly || !rule.readOnly) && match(rule.pattern, source) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// needsNoCopy reports a volume only `:ro` rules allow, as dockerd fills an empty volume from the image even when mounted read-only.
|
||||
func (m validVolumeMatcher) needsNoCopy(source string, sourceType mount.Type) bool {
|
||||
return sourceType == mount.TypeVolume && !m.isValid(source, sourceType, false)
|
||||
}
|
||||
|
||||
func isHostVolumePattern(pattern string) bool {
|
||||
return filepath.IsAbs(pattern) ||
|
||||
strings.HasPrefix(pattern, "."+string(filepath.Separator)) ||
|
||||
|
||||
@@ -137,6 +137,11 @@ func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.Netw
|
||||
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkCreate(ctx context.Context, name string, opts mobyclient.NetworkCreateOptions) (mobyclient.NetworkCreateResult, error) {
|
||||
args := m.Called(ctx, name, opts)
|
||||
return args.Get(0).(mobyclient.NetworkCreateResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
|
||||
@@ -519,10 +524,12 @@ var _ ExecutionsEnvironment = &containerReference{}
|
||||
|
||||
func TestCheckVolumes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
mounts []mount.Mount
|
||||
expectedMounts []mount.Mount
|
||||
}{
|
||||
{
|
||||
desc: "match all volumes",
|
||||
@@ -578,6 +585,27 @@ func TestCheckVolumes(t *testing.T) {
|
||||
"/opt/app/cache:/cache",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only rules allow only read-only mounts and copy nothing into volumes",
|
||||
validVolumes: []string{"/etc/ssl:ro", "data:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "/etc/ssl:/ssl", "data:/data:ro", "data:/data", "shared:/shared:ro"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "data:/data:ro,nocopy", "shared:/shared:ro"},
|
||||
mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true, BindOptions: &mount.BindOptions{ReadOnlyNonRecursive: true}},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true},
|
||||
},
|
||||
expectedMounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true, VolumeOptions: &mount.VolumeOptions{NoCopy: true}},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only wildcard keeps writable rules writable",
|
||||
validVolumes: []string{"**:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "data:/data", "shared:/shared"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "shared:/shared"},
|
||||
},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
@@ -588,9 +616,10 @@ func TestCheckVolumes(t *testing.T) {
|
||||
ValidVolumes: tc.validVolumes,
|
||||
},
|
||||
}
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds})
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts})
|
||||
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
||||
assert.Len(t, hook.AllEntries(), len(tc.binds)-len(tc.expectedBinds)) // every drop is warned about
|
||||
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
||||
assert.Len(t, hook.AllEntries(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts)) // every drop is warned about
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -666,6 +695,9 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
DeviceCgroupRules: []string{"a *:* rwm"},
|
||||
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}},
|
||||
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
|
||||
LogConfig: container.LogConfig{Type: "syslog", Config: map[string]string{"syslog-address": "tcp://169.254.169.254:514"}},
|
||||
OomScoreAdj: -1000,
|
||||
OomKillDisable: new(true),
|
||||
}
|
||||
|
||||
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{})
|
||||
@@ -699,7 +731,9 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
|
||||
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
|
||||
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
|
||||
"--security-opt apparmor=unconfined --volumes-from other --isolation process " +
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1 " +
|
||||
"--log-driver=syslog --log-opt syslog-address=tcp://169.254.169.254:514 " +
|
||||
"--oom-score-adj=-1000 --oom-kill-disable"
|
||||
|
||||
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's
|
||||
// own options alone describe it, field for field
|
||||
@@ -762,6 +796,22 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
}
|
||||
|
||||
func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
|
||||
hostConfig, _ := mergeOptions(t, "", "--mount type=tmpfs,dst=/var/lib/mysql", false)
|
||||
require.Len(t, hostConfig.Mounts, 1)
|
||||
|
||||
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
||||
logger, hook := test.NewNullLogger()
|
||||
_, sanitized := (&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{Mounts: []mount.Mount{
|
||||
hostConfig.Mounts[0],
|
||||
{Type: mount.TypeVolume, Target: "/anonymous"},
|
||||
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
||||
}})
|
||||
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
||||
assert.Len(t, hook.AllEntries(), 2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerInfoFromInspect(t *testing.T) {
|
||||
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
|
||||
info := containerInfoFromInspect(container.InspectResponse{
|
||||
|
||||
@@ -433,8 +433,8 @@ func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, "/", func(writer io.Writer) error {
|
||||
return writeDirTar(ctx, writer, destPath, srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -230,6 +231,15 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||
require.Error(t, err)
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
||||
|
||||
@@ -532,7 +532,7 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
|
||||
"/jobs/" + strconv.FormatInt(jobID, 10) + "/steps/"
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
// Reuse a single client across all step uploads so connections can be pooled.
|
||||
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout}
|
||||
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout, Transport: rc.Config.instanceTransportFor(base)}
|
||||
for i := range rc.Run.Job().Steps {
|
||||
summaryPath := path.Join(actPath, "workflow", "step-summary-"+strconv.Itoa(i)+".md")
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, rc.JobContainer, summaryPath, maxJobSummaryBytes)
|
||||
@@ -544,6 +544,15 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
|
||||
}
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// instanceTransportFor returns InstanceTransport only for URLs on the registered address, as steps can rewrite the summary URL through GITHUB_ENV.
|
||||
func (c Config) instanceTransportFor(rawURL string) http.RoundTripper {
|
||||
if !strings.HasPrefix(rawURL, c.GitHubInstance+"/") {
|
||||
return nil
|
||||
}
|
||||
return c.InstanceTransport
|
||||
}
|
||||
|
||||
// extractJobIDFromRuntimeToken returns the JobID claim from an ACTIONS_RUNTIME_TOKEN JWT
|
||||
// without verifying its signature. Returns 0 if the token is unparseable or has no JobID.
|
||||
func extractJobIDFromRuntimeToken(token string) int64 {
|
||||
|
||||
@@ -858,7 +858,7 @@ func TestTryUploadJobSummaryStopsAtPhaseTimeout(t *testing.T) {
|
||||
cm.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t *testing.T) {
|
||||
runtimeToken := fakeRuntimeToken(34)
|
||||
|
||||
type upload struct {
|
||||
@@ -866,7 +866,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
body string
|
||||
}
|
||||
var got []upload
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
got = append(got, upload{r.URL.Path, string(body)})
|
||||
@@ -896,6 +896,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
"ACTIONS_RUNTIME_TOKEN": runtimeToken,
|
||||
"GITEA_RUN_ID": "12",
|
||||
}, cm, 3)
|
||||
rc.Config.GitHubInstance, rc.Config.InstanceTransport = server.URL, server.Client().Transport
|
||||
|
||||
tryUploadJobSummary(ctx, rc)
|
||||
|
||||
|
||||
@@ -39,11 +39,8 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
|
||||
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(uses))
|
||||
|
||||
// If the repository is private, we need a token to clone it
|
||||
token := rc.Config.GetToken()
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir, token),
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir),
|
||||
newReusableWorkflowExecutor(rc, workflowDir, remoteReusableWorkflow.Path),
|
||||
)
|
||||
}
|
||||
@@ -60,10 +57,8 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Owner, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref)
|
||||
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename))
|
||||
|
||||
token := getGitCloneToken(rc.Config, remoteReusableWorkflow.CloneURL())
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir, token),
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir),
|
||||
newReusableWorkflowExecutor(rc, workflowDir, remoteReusableWorkflow.Path),
|
||||
)
|
||||
}
|
||||
@@ -76,20 +71,19 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
// 2. Gitea has already full URL with rc.Config.GitHubInstance when calling newRemoteReusableWorkflow
|
||||
//
|
||||
// remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL
|
||||
func cloneRemoteReusableWorkflow(rc *RunContext, cloneURL, ref, targetDirectory, token string) common.Executor {
|
||||
func cloneRemoteReusableWorkflow(rc *RunContext, cloneURL, ref, targetDirectory string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
interpolatedURL, err := rc.NewExpressionEvaluator(ctx).Interpolate(ctx, cloneURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to interpolate the workflow clone URL: %w", err)
|
||||
}
|
||||
return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
return git.NewGitCloneExecutor(rc.Config.withInstanceAuth(git.NewGitCloneExecutorInput{
|
||||
URL: interpolatedURL,
|
||||
Ref: ref,
|
||||
Dir: targetDirectory,
|
||||
Token: token,
|
||||
OfflineMode: rc.Config.ActionOfflineMode,
|
||||
Depth: rc.Config.ActionCloneDepth,
|
||||
})(ctx)
|
||||
}, rc.Config.GetToken()))(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,13 +202,18 @@ func setReusedWorkflowCallerResult(rc *RunContext, runner *runnerImpl) common.Ex
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// getGitCloneToken returns GITEA_TOKEN when shouldCloneURLUseToken returns true,
|
||||
// otherwise returns an empty string
|
||||
func getGitCloneToken(conf *Config, cloneURL string) string {
|
||||
if !shouldCloneURLUseToken(conf.GitHubInstance, conf.trustedActionInstance(), cloneURL) {
|
||||
return ""
|
||||
// withInstanceAuth passes instance credentials only to clones from this Gitea instance, and runner.insecure only to its registered address.
|
||||
func (c Config) withInstanceAuth(input git.NewGitCloneExecutorInput, token string) git.NewGitCloneExecutorInput {
|
||||
u, err := url.Parse(input.URL)
|
||||
if err != nil || !onInstanceHost(u.Host, c.GitHubInstance, c.trustedActionInstance()) {
|
||||
return input
|
||||
}
|
||||
return conf.GetToken()
|
||||
input.ClientCertFile, input.ClientKeyFile = c.ClientCertFile, c.ClientKeyFile
|
||||
input.InsecureSkipTLS = c.InsecureSkipTLS && onInstanceHost(u.Host, c.GitHubInstance)
|
||||
if u.User == nil {
|
||||
input.Token = token
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
@@ -228,26 +227,16 @@ func (c Config) trustedActionInstance() string {
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// shouldCloneURLUseToken returns true when the following conditions are met:
|
||||
// 1. cloneURL's host matches this Gitea instance: either the registered instance
|
||||
// (instanceURL) or, for DEFAULT_ACTIONS_URL=self on a different hostname, the
|
||||
// self-hosted action instance (trustedActionInstance, "" when not trusted)
|
||||
// 2. the cloneURL does not have basic auth embedded
|
||||
func shouldCloneURLUseToken(instanceURL, trustedActionInstance, cloneURL string) bool {
|
||||
u2, err := url.Parse(cloneURL)
|
||||
if err != nil || u2.User != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, candidate := range []string{instanceURL, trustedActionInstance} {
|
||||
// onInstanceHost reports whether host is that of one of the instance URLs, which may lack a scheme.
|
||||
func onInstanceHost(host string, instanceURLs ...string) bool {
|
||||
for _, candidate := range instanceURLs {
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(candidate, "http://") &&
|
||||
!strings.HasPrefix(candidate, "https://") {
|
||||
if !strings.HasPrefix(candidate, "http://") && !strings.HasPrefix(candidate, "https://") {
|
||||
candidate = "https://" + candidate
|
||||
}
|
||||
if u1, err := url.Parse(candidate); err == nil && u1.Host == u2.Host {
|
||||
if u, err := url.Parse(candidate); err == nil && u.Host == host {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,12 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -59,7 +64,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
}
|
||||
cacheDir := t.TempDir()
|
||||
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir, "")(context.Background()))
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(context.Background()))
|
||||
got, err := os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tmpl("v1"), string(got))
|
||||
@@ -69,7 +74,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
gitMust(t, workDir, "commit", "-am", "v2")
|
||||
gitMust(t, workDir, "push", "origin", "master")
|
||||
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir, "")(context.Background()))
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(context.Background()))
|
||||
got, err = os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tmpl("v2"), string(got), "cached workflow file must reflect the updated branch tip")
|
||||
@@ -123,107 +128,79 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetGitCloneTokenWithSchemalessGiteaInstance(t *testing.T) {
|
||||
conf := &Config{
|
||||
GitHubInstance: "gitea.example.net",
|
||||
Secrets: map[string]string{
|
||||
"GITEA_TOKEN": "token-value",
|
||||
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T) {
|
||||
var gotCert bool
|
||||
var gotToken string
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotCert = len(r.TLS.PeerCertificates) > 0
|
||||
_, gotToken, _ = r.BasicAuth()
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
pair := server.TLS.Certificates[0]
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(pair.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
dir := t.TempDir()
|
||||
certFile, keyFile := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: pair.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
rc := &RunContext{
|
||||
Config: &Config{
|
||||
GitHubInstance: server.URL,
|
||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
},
|
||||
Run: &model.Run{JobID: "j1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"j1": {}}}},
|
||||
}
|
||||
|
||||
token := getGitCloneToken(conf, "https://gitea.example.net/actions/tools")
|
||||
|
||||
require.Equal(t, "token-value", token)
|
||||
require.Error(t, cloneRemoteReusableWorkflow(rc, server.URL+"/owner/repo", "main", t.TempDir())(t.Context()))
|
||||
require.True(t, gotCert)
|
||||
require.Equal(t, "token", gotToken)
|
||||
}
|
||||
|
||||
func TestGetGitCloneTokenSelfHostedActionsDifferentHost(t *testing.T) {
|
||||
// The runner registered with one hostname while DEFAULT_ACTIONS_URL=self resolves
|
||||
// actions against AppURL on a different hostname for the same instance.
|
||||
conf := &Config{
|
||||
GitHubInstance: "gitea.local",
|
||||
DefaultActionInstance: "https://gitea.my-nas.lan",
|
||||
DefaultActionInstanceIsSelfHosted: true,
|
||||
Secrets: map[string]string{
|
||||
"GITEA_TOKEN": "token-value",
|
||||
},
|
||||
}
|
||||
|
||||
token := getGitCloneToken(conf, "https://gitea.my-nas.lan/owner/action")
|
||||
|
||||
require.Equal(t, "token-value", token)
|
||||
}
|
||||
|
||||
func TestShouldCloneURLUseToken(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
instanceURL string
|
||||
trustedActionInstance string
|
||||
cloneURL string
|
||||
want bool
|
||||
func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, instanceURL, trustedActionInstance, cloneURL string
|
||||
token, cert, insecure bool
|
||||
}{
|
||||
{
|
||||
name: "same host with schemaless instance",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://gitea.example.net/actions/tools",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "same host with schemaless instance and port",
|
||||
instanceURL: "gitea.example.net:3000",
|
||||
cloneURL: "https://gitea.example.net:3000/actions/tools",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "different host",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://github.com/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "embedded basic auth",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://user:[email protected]/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "invalid clone URL",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "://gitea.example.net/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// self-hosted DEFAULT_ACTIONS_URL on a different hostname than the
|
||||
// registered instance: the token must still be attached.
|
||||
name: "self-hosted action instance on different host",
|
||||
instanceURL: "gitea.local",
|
||||
trustedActionInstance: "https://gitea.my-nas.lan",
|
||||
cloneURL: "https://gitea.my-nas.lan/owner/action",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// embedded basic auth must still be rejected even when the host matches
|
||||
// the trusted action instance.
|
||||
name: "self-hosted action instance with embedded basic auth",
|
||||
instanceURL: "gitea.local",
|
||||
trustedActionInstance: "https://gitea.my-nas.lan",
|
||||
cloneURL: "https://user:[email protected]/owner/action",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// github.com / mirror hosts are never trusted: trustedActionInstance is
|
||||
// empty in github mode, so an off-instance clone URL gets no token.
|
||||
name: "github mode does not trust mirror host",
|
||||
instanceURL: "gitea.local",
|
||||
cloneURL: "https://mirror.example.com/owner/action",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
{"same host with schemaless instance", "gitea.example.net", "", "https://gitea.example.net/actions/tools", true, true, true},
|
||||
{"same host with schemaless instance and port", "gitea.example.net:3000", "", "https://gitea.example.net:3000/actions/tools", true, true, true},
|
||||
{"different host", "gitea.example.net", "", "https://github.com/actions/tools", false, false, false},
|
||||
{"embedded basic auth keeps its own credentials", "gitea.example.net", "", "https://user:pass@gitea.example.net/actions/tools", false, true, true},
|
||||
{"invalid clone URL", "gitea.example.net", "", "://gitea.example.net/actions/tools", false, false, false},
|
||||
{"self-hosted action instance on different host skips no verification", "gitea.local", "https://gitea.my-nas.lan", "https://gitea.my-nas.lan/owner/action", true, true, false},
|
||||
{"self-hosted action instance with embedded basic auth", "gitea.local", "https://gitea.my-nas.lan", "https://user:[email protected]/owner/action", false, true, false},
|
||||
{"github mode does not trust mirror host", "gitea.local", "", "https://mirror.example.com/owner/action", false, false, false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, shouldCloneURLUseToken(tt.instanceURL, tt.trustedActionInstance, tt.cloneURL))
|
||||
conf := Config{
|
||||
GitHubInstance: tt.instanceURL,
|
||||
DefaultActionInstance: tt.trustedActionInstance,
|
||||
DefaultActionInstanceIsSelfHosted: tt.trustedActionInstance != "",
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: "/cert",
|
||||
ClientKeyFile: "/key",
|
||||
}
|
||||
want := git.NewGitCloneExecutorInput{URL: tt.cloneURL, InsecureSkipTLS: tt.insecure}
|
||||
if tt.token {
|
||||
want.Token = "token"
|
||||
}
|
||||
if tt.cert {
|
||||
want.ClientCertFile, want.ClientKeyFile = conf.ClientCertFile, conf.ClientKeyFile
|
||||
}
|
||||
require.Equal(t, want, conf.withInstanceAuth(git.NewGitCloneExecutorInput{URL: tt.cloneURL}, "token"))
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{}
|
||||
conf := Config{GitHubInstance: "https://gitea.local", InstanceTransport: transport}
|
||||
require.Same(t, transport, conf.instanceTransportFor("https://gitea.local/api/actions_pipeline/"))
|
||||
require.Nil(t, conf.instanceTransportFor("https://gitea.local.example/api/actions_pipeline/"))
|
||||
require.Nil(t, conf.instanceTransportFor("https://other.example/x#/api/actions_pipeline/"))
|
||||
}
|
||||
|
||||
func gitMust(t *testing.T, dir string, args ...string) {
|
||||
|
||||
@@ -725,7 +725,7 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
||||
logger.Infof("Cleaning up services for job %s", rc.JobName)
|
||||
errs = append(errs, rc.stopServiceContainers()(ctx))
|
||||
}
|
||||
if !common.Dryrun(ctx) && (preclean || rc.hadDockerProxy) {
|
||||
if !common.Dryrun(ctx) && !rc.Config.NoSweep && (preclean || rc.hadDockerProxy) {
|
||||
errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName()))
|
||||
}
|
||||
if preclean || rc.deferVolumeCleanup == nil {
|
||||
@@ -1236,10 +1236,6 @@ func (rc *RunContext) Executor() (common.Executor, error) {
|
||||
}
|
||||
|
||||
func (rc *RunContext) runsOnImage(ctx context.Context) (string, error) {
|
||||
if rc.Run.Job().RunsOn() == nil && rc.containerSpec.Image == "" {
|
||||
common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String())
|
||||
}
|
||||
|
||||
runsOn := rc.Run.Job().RunsOn()
|
||||
for i, v := range runsOn {
|
||||
var err error
|
||||
@@ -1708,9 +1704,7 @@ func parentDir(p string) string {
|
||||
// still names a release when the label is a rolling one such as ubuntu-latest.
|
||||
func (rc *RunContext) imageOS(ctx context.Context) string {
|
||||
if rc.Run.Job().RunsOn() == nil {
|
||||
// A composite action runs on a synthetic job, and resolving its image would only
|
||||
// log that runs-on is missing.
|
||||
return ""
|
||||
return "" // a composite action runs on a synthetic job without runs-on
|
||||
}
|
||||
if imageOS := imageOSFromImage(rc.platformImage); imageOS != "" {
|
||||
return imageOS
|
||||
|
||||
@@ -778,9 +778,10 @@ func TestRemoveDockerActionImagesByTag(t *testing.T) {
|
||||
func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
t.Setenv("TMPDIR", "/tmp")
|
||||
proxyDir := t.TempDir()
|
||||
for _, name := range []string{"synchronous", "proxy", "closed proxy", "deferred", "preclean deferred", "canceled"} {
|
||||
for _, name := range []string{"synchronous", "proxy", "closed proxy", "kept proxy", "deferred", "preclean deferred", "canceled"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
proxy, preclean, deferred := strings.HasSuffix(name, "proxy"), strings.HasPrefix(name, "preclean"), strings.HasSuffix(name, "deferred")
|
||||
removesLabelled := (proxy || preclean) && name != "kept proxy"
|
||||
if proxy && runtime.GOOS == "windows" {
|
||||
t.Skip("Unix socket ownership is unavailable on Windows")
|
||||
}
|
||||
@@ -790,7 +791,7 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
service.On("Remove").Return(func(context.Context) error { return removeError }).Once()
|
||||
service.On("Close").Return(func(context.Context) error { return closeError }).Once()
|
||||
rc := &RunContext{
|
||||
Config: &Config{CacheContainer: "cache-container"},
|
||||
Config: &Config{CacheContainer: "cache-container", NoSweep: name == "kept proxy"},
|
||||
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
|
||||
JobContainer: job,
|
||||
serviceContainers: []*serviceContainer{{name: "svc", container: service}},
|
||||
@@ -846,8 +847,8 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, 2, volumeRemovals)
|
||||
}
|
||||
assert.Equal(t, proxy || preclean, strings.Contains(err.Error(), "labelled GET /containers/json"))
|
||||
if proxy || preclean {
|
||||
assert.Equal(t, removesLabelled, strings.Contains(err.Error(), "labelled GET /containers/json"))
|
||||
if removesLabelled {
|
||||
require.ErrorContains(t, err, "labelled GET /networks")
|
||||
require.ErrorContains(t, err, "labelled GET /volumes")
|
||||
}
|
||||
|
||||
@@ -7,8 +7,10 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -76,13 +78,16 @@ type Config struct {
|
||||
DefaultActionInstanceIsSelfHosted bool
|
||||
PlatformPicker func(labels []string) string
|
||||
KubernetesPicker func(labels []string) container.KubernetesOptions
|
||||
InstanceTransport http.RoundTripper
|
||||
JobLoggerLevel *log.Level // the level of job logger
|
||||
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
||||
SharedToolCache bool // one tool cache for all jobs instead of one per job
|
||||
InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance
|
||||
ClientCertFile, ClientKeyFile string // PEM client certificate for the Gitea instance
|
||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
||||
NoSweep bool // keep the containers, networks and volumes a job creates through its docker proxy
|
||||
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
||||
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
||||
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
||||
@@ -173,6 +178,15 @@ func maxParallelFor(strategy *model.Strategy, combinations int) int {
|
||||
|
||||
// NewPlanExecutor ...
|
||||
func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
|
||||
for _, stage := range plan.Stages {
|
||||
for _, run := range stage.Runs {
|
||||
job := run.Job()
|
||||
if runsOn := job.RunsOn(); job.Uses == "" && len(job.Steps) > 0 && (len(runsOn) == 0 || slices.Contains(runsOn, "")) { // Gitea's needs placeholders carry no steps
|
||||
return common.NewErrorExecutor(fmt.Errorf("job %q requires a non-empty 'runs-on'", run.JobID))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
maxJobNameLen := 0
|
||||
|
||||
stagePipeline := make([]common.Executor, 0)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
log "github.com/sirupsen/logrus"
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -151,6 +152,21 @@ func TestGraphWithSomeMissing(t *testing.T) {
|
||||
log.SetOutput(out)
|
||||
}
|
||||
|
||||
func TestNewPlanExecutorRequiresRunsOnBeforeRunningAnyJob(t *testing.T) {
|
||||
for _, runsOn := range []string{"", "if: false", "runs-on: [ubuntu-latest, '']"} {
|
||||
workflow, err := model.ReadWorkflow(strings.NewReader("jobs:\n first:\n runs-on: ubuntu-latest\n steps: [run: echo]\n test:\n needs: first\n " + runsOn + "\n steps: [run: echo]\n"))
|
||||
require.NoError(t, err)
|
||||
plan, err := model.CombineWorkflowPlanner(workflow).PlanJob("test")
|
||||
require.NoError(t, err)
|
||||
require.ErrorContains(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(plan)(t.Context()), `job "test" requires a non-empty 'runs-on'`, runsOn)
|
||||
assert.Empty(t, workflow.Jobs["first"].Result, runsOn)
|
||||
}
|
||||
|
||||
workflow := &model.Workflow{Jobs: map[string]*model.Job{"build": {Result: "success"}}}
|
||||
require.NoError(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "build"}}}}})(t.Context()))
|
||||
assert.Equal(t, "success", workflow.Jobs["build"].Result)
|
||||
}
|
||||
|
||||
func TestGraphEvent(t *testing.T) {
|
||||
planner, err := model.NewWorkflowPlanner("testdata/basic", true)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
@@ -38,6 +38,7 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
defer rawLogger.Infof("::endgroup::")
|
||||
|
||||
rc := sab.RunContext
|
||||
rc.ApplyExtraPath(ctx, &sab.env)
|
||||
return sab.run(ctx, &action.Context{
|
||||
Container: rc.JobContainer,
|
||||
Github: rc.getGithubContext(ctx),
|
||||
@@ -48,6 +49,9 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
NoSkipCheckout: rc.Config.NoSkipCheckout,
|
||||
UseGitIgnore: rc.Config.UseGitIgnore,
|
||||
InsecureSkipTLS: rc.Config.InsecureSkipTLS,
|
||||
ClientCertFile: rc.Config.ClientCertFile,
|
||||
ClientKeyFile: rc.Config.ClientKeyFile,
|
||||
Env: sab.env,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
)
|
||||
|
||||
type stepActionRemote struct {
|
||||
@@ -68,36 +67,22 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
actionDir := sar.actionDir()
|
||||
defaultActionURL := sar.RunContext.Config.DefaultActionURL()
|
||||
// For Gitea
|
||||
// A composite RunContext nils Config.Secrets, so getGitCloneToken would yield an
|
||||
// empty token and clone the action anonymously (401 against the authenticated
|
||||
// instance). github.Token survives the composite config copy and matches the
|
||||
// top-level token; keep the shouldCloneURLUseToken host gate to avoid leaking it.
|
||||
cloneURL := sar.remoteAction.CloneURL(defaultActionURL)
|
||||
token := ""
|
||||
if shouldCloneURLUseToken(sar.RunContext.Config.GitHubInstance, sar.RunContext.Config.trustedActionInstance(), cloneURL) {
|
||||
token = github.Token
|
||||
}
|
||||
gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
URL: cloneURL,
|
||||
// github.Token, unlike Config.GetToken, survives the composite config copy, which nils Config.Secrets.
|
||||
gitClone := stepActionRemoteNewCloneExecutor(sar.RunContext.Config.withInstanceAuth(git.NewGitCloneExecutorInput{
|
||||
URL: sar.remoteAction.CloneURL(defaultActionURL),
|
||||
Ref: sar.remoteAction.Ref,
|
||||
Dir: actionDir,
|
||||
Token: token,
|
||||
OfflineMode: sar.RunContext.Config.ActionOfflineMode,
|
||||
Depth: sar.RunContext.Config.ActionCloneDepth,
|
||||
// printPrepareActions reports the download with its resolved commit.
|
||||
Quiet: true,
|
||||
|
||||
InsecureSkipTLS: sar.cloneSkipTLS(), // For Gitea
|
||||
})
|
||||
var ntErr common.Executor
|
||||
}, github.Token))
|
||||
if err := gitClone(ctx); err != nil {
|
||||
var refErr *git.Error
|
||||
switch {
|
||||
case errors.As(err, &refErr) && errors.Is(err, git.ErrShortRef):
|
||||
return fmt.Errorf("unable to resolve action `%s`, the provided ref `%s` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `%s` instead",
|
||||
sar.Step.Uses, sar.remoteAction.Ref, refErr.Commit())
|
||||
case errors.Is(err, gogit.ErrForceNeeded): // TODO: figure out if it will be easy to shadow/alias go-git err's
|
||||
ntErr = common.NewInfoExecutor("Non-terminating error while running 'git clone': %v", err)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
@@ -115,15 +100,10 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
return f, f, err
|
||||
}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
ntErr,
|
||||
func(ctx context.Context) error {
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile)
|
||||
sar.action = actionModel
|
||||
return err
|
||||
},
|
||||
)(ctx)
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile)
|
||||
sar.action = actionModel
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,22 +239,6 @@ func (sar *stepActionRemote) getCompositeSteps() *compositeSteps {
|
||||
return sar.compositeSteps
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// cloneSkipTLS returns true if the runner can clone an action from the Gitea instance
|
||||
func (sar *stepActionRemote) cloneSkipTLS() bool {
|
||||
if !sar.RunContext.Config.InsecureSkipTLS {
|
||||
// Return false if the Gitea instance is not an insecure instance
|
||||
return false
|
||||
}
|
||||
if sar.remoteAction.URL == "" {
|
||||
// Empty URL means the default action instance should be used
|
||||
// Return true if the URL of the Gitea instance is the same as the URL of the default action instance
|
||||
return sar.RunContext.Config.DefaultActionURL() == sar.RunContext.Config.GitHubInstance
|
||||
}
|
||||
// Return true if the URL of the remote action is the same as the URL of the Gitea instance
|
||||
return sar.remoteAction.URL == sar.RunContext.Config.GitHubInstance
|
||||
}
|
||||
|
||||
type remoteAction struct {
|
||||
URL string
|
||||
Org string
|
||||
|
||||
+22
-4
@@ -8,14 +8,32 @@ Built-in actions ship with the runner, so a job needs no action download and no
|
||||
|
||||
## checkout
|
||||
|
||||
Works like `actions/checkout` with these inputs. Any other input fails the step.
|
||||
Works like `actions/checkout` with these inputs. Any other input fails the step. It runs `git` where the job runs, see the [prerequisites](../README.md#prerequisites).
|
||||
|
||||
| Input | Default |
|
||||
| --- | --- |
|
||||
| `repository` | `github.repository` |
|
||||
| `ref` | the triggering commit, or the default branch of another repository |
|
||||
| `ref` | the triggering ref and commit, or the default branch of another repository |
|
||||
| `token` | `github.token` |
|
||||
| `ssh-key` | empty, a private key fetches over SSH instead |
|
||||
| `ssh-known-hosts` | empty, host keys trusted besides `~/.ssh/known_hosts` |
|
||||
| `ssh-strict` | `true`, rejects unknown host keys |
|
||||
| `ssh-user` | the user of the repository's SSH URL, else `git` |
|
||||
| `persist-credentials` | `true`, later steps of the job can fetch and push with `token` or `ssh-key` |
|
||||
| `path` | workspace root |
|
||||
| `fetch-depth` | `1`, `0` for full history |
|
||||
| `clean` | `true`, removes untracked files, also in submodules |
|
||||
| `filter` | empty, a partial clone filter such as `blob:none` |
|
||||
| `sparse-checkout` | empty, the paths to check out, one per line |
|
||||
| `sparse-checkout-cone-mode` | `true`, `false` reads `sparse-checkout` as patterns |
|
||||
| `fetch-depth` | `1`, `0` for the full history of every branch and tag |
|
||||
| `fetch-tags` | `false`, `true` fetches every tag |
|
||||
| `show-progress` | `true`, shows the fetch progress |
|
||||
| `lfs` | `false`, `true` downloads Git LFS files |
|
||||
| `submodules` | `false`, `true` checks out submodules, `recursive` also theirs |
|
||||
| `set-safe-directory` | `true`, lets checkout's `git` work in a directory another user owns |
|
||||
|
||||
With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` is set.
|
||||
A branch is checked out as a local branch tracking `origin`, a tag or commit as a detached `HEAD`. A triggering tag that moved since the event fails the step. The step outputs `ref` and `commit` like `actions/checkout`.
|
||||
|
||||
Persisted credentials end with the job. `token` is only sent to `github.server_url`, while `ssh-key` is offered to any SSH host, as with `actions/checkout`. The runner's `client_cert_file` is only offered to `github.server_url` and only during the step, so later steps do not get it. Without `ssh-key`, submodule URLs of the form `git@<server host>:` are fetched from `github.server_url` with `token`. With `ssh-key`, the repository is fetched from the SSH URL in the event's `repository.ssh_url`, or from the server's host on port 22.
|
||||
|
||||
With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` or `ssh-key` is set.
|
||||
+1
-1
@@ -45,7 +45,7 @@ kubernetes:
|
||||
nvidia.com/gpu.present: "true"
|
||||
```
|
||||
|
||||
A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`.
|
||||
A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. Like every config value, templates expand `${NAME}` from the runner's environment, so a shell command in a template writes `$${NAME}` for the pod's own variable.
|
||||
|
||||
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
||||
|
||||
|
||||
+18
-2
@@ -31,7 +31,7 @@ Except for the endpoint, each `OTEL_EXPORTER_OTLP_*` variable has `TRACES` and `
|
||||
|
||||
- A span per job, named `RUN {workflow} {job}`, with the repository, ref, commit, run URL, run number, attempt and result.
|
||||
- A span per step, named as in Gitea's step list, including `Pre` and `Post` stages and the steps of composite actions, with the step's result.
|
||||
- The runner's name, UUID and version.
|
||||
- The runner's name, UUID, version, process ID and start time, and the Go version.
|
||||
|
||||
Logs, step output and secrets are never sent. Step names are taken from the workflow file as written, without expanding `${{ }}` expressions.
|
||||
|
||||
@@ -41,7 +41,23 @@ Jobs receive `TRACEPARENT`, so tools in a job that use OpenTelemetry can add the
|
||||
|
||||
## Metrics
|
||||
|
||||
The metrics of the Prometheus endpoint are also exported, under the same names and whether or not `metrics.enabled` is set. They are sent every export interval and once more on shutdown. If your collector only accepts traces, set `OTEL_METRICS_EXPORTER=none`.
|
||||
The metrics of the Prometheus endpoint are also exported, whether or not `metrics.enabled` is set, except `gitea_runner_info`, `go_info` and `process_start_time_seconds`, which are sent as resource attributes. They are sent every export interval and once more on shutdown. To turn them off, set `OTEL_METRICS_EXPORTER=none`.
|
||||
|
||||
Metrics are named after the [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) where one exists, and under `gitea.runner.*` otherwise. Other Go and process metrics, such as `go_memstats_heap_alloc_bytes`, keep their Prometheus names.
|
||||
|
||||
| Metric | Attributes |
|
||||
| --- | --- |
|
||||
| `cicd.worker.count` | `cicd.worker.state` |
|
||||
| `cicd.system.errors` | `cicd.system.component`, `rpc.method`, `error.type` |
|
||||
| `gitea.runner.jobs` | `cicd.pipeline.result` |
|
||||
| `gitea.runner.job.active`, `gitea.runner.job.limit`, `gitea.runner.job.utilization`, `gitea.runner.job.duration` | |
|
||||
| `gitea.runner.poll.fetches` | `gitea.runner.poll.result` |
|
||||
| `gitea.runner.uptime`, `gitea.runner.poll.backoff`, `gitea.runner.poll.fetch.duration` | |
|
||||
| `gitea.runner.report.log.requests`, `gitea.runner.report.state.requests` | `gitea.runner.report.result` |
|
||||
| `gitea.runner.report.log.buffer`, `gitea.runner.report.log.duration`, `gitea.runner.report.state.duration` | |
|
||||
| `go.memory.used` | `go.memory.type` |
|
||||
| `go.goroutine.count`, `go.memory.allocated`, `go.memory.gc.goal`, `go.memory.gc.cycles`, `go.memory.limit`, `go.config.gogc`, `go.processor.limit` | |
|
||||
| `process.memory.usage`, `process.memory.virtual`, `process.unix.file_descriptor.count` (`process.windows.handle.count` on Windows) | |
|
||||
|
||||
## Security
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import "testing"
|
||||
|
||||
func testBuiltinCheckout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
api, repo := newScenario(t)
|
||||
pushWorkflow(t, api, repo, "checkout.yml")
|
||||
branchRun := waitForRun(t, api, repo)
|
||||
if err := api.CreateTag(t.Context(), repo, "v1"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tagRun := waitForRunAfter(t, api, repo, branchRun.ID)
|
||||
requireSuccess(t, api, repo, branchRun.ID)
|
||||
requireSuccess(t, api, repo, tagRun.ID)
|
||||
}
|
||||
@@ -30,6 +30,7 @@ func TestCompatibility(t *testing.T) {
|
||||
t.Run("artifact", testArtifactRoundTrip)
|
||||
t.Run("cancellation_and_log_streaming", testRunCancellation)
|
||||
t.Run("dispatch", testWorkflowDispatch)
|
||||
t.Run("builtin_checkout", testBuiltinCheckout)
|
||||
t.Run("ephemeral", testEphemeralRunner)
|
||||
t.Run("kubernetes", testKubernetes)
|
||||
}
|
||||
|
||||
@@ -53,6 +53,12 @@ func (a *GiteaAPI) CreateFile(ctx context.Context, repo, path, content, message
|
||||
return a.doJSON(ctx, http.MethodPost, url, body, nil)
|
||||
}
|
||||
|
||||
func (a *GiteaAPI) CreateTag(ctx context.Context, repo, name string) error {
|
||||
body := map[string]any{"tag_name": name}
|
||||
url := fmt.Sprintf("/api/v1/repos/%s/%s/tags", giteaAdminUser, repo)
|
||||
return a.doJSON(ctx, http.MethodPost, url, body, nil)
|
||||
}
|
||||
|
||||
func (a *GiteaAPI) CreateSecret(ctx context.Context, repo, name, value string) error {
|
||||
body := map[string]any{"data": value}
|
||||
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/secrets/%s", giteaAdminUser, repo, name)
|
||||
|
||||
@@ -188,6 +188,7 @@ func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, e
|
||||
"GITEA__security__INSTALL_LOCK=true",
|
||||
"GITEA__database__DB_TYPE=sqlite3",
|
||||
"GITEA__actions__ENABLED=true",
|
||||
"GITEA__server__LFS_START_SERVER=true",
|
||||
"GITEA__server__ROOT_URL=" + baseURL + "/",
|
||||
},
|
||||
ExposedPorts: network.PortSet{containerPort: struct{}{}},
|
||||
|
||||
+6
-1
@@ -141,6 +141,11 @@ func dumpRunLogs(t *testing.T, api *GiteaAPI, repo string, runID int64) {
|
||||
}
|
||||
|
||||
func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun {
|
||||
t.Helper()
|
||||
return waitForRunAfter(t, api, repo, 0)
|
||||
}
|
||||
|
||||
func waitForRunAfter(t *testing.T, api *GiteaAPI, repo string, previousID int64) *ActionRun {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
|
||||
defer cancel()
|
||||
@@ -150,7 +155,7 @@ func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun {
|
||||
if err != nil {
|
||||
t.Fatalf("get latest run: %v", err)
|
||||
}
|
||||
if len(runs) > 0 {
|
||||
if len(runs) > 0 && runs[0].ID > previousID {
|
||||
return &runs[0]
|
||||
}
|
||||
select {
|
||||
|
||||
@@ -75,7 +75,7 @@ func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *p
|
||||
}
|
||||
labelNames := []string{label.Name}
|
||||
|
||||
pingCli := client.New(fixture.baseURL, cfg.Runner.Insecure, "", "", config.RequestTimeout)
|
||||
pingCli := client.New(fixture.baseURL, cfg.Runner.TLSConfig(), "", "", config.RequestTimeout, cfg.Runner.ExtraHeaders)
|
||||
if _, err := pingCli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{Data: t.Name()})); err != nil {
|
||||
t.Fatalf("ping %s: %v", fixture.baseURL, err)
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *p
|
||||
Ephemeral: regResp.Msg.Runner.Ephemeral,
|
||||
}
|
||||
|
||||
cli := client.New(fixture.baseURL, cfg.Runner.Insecure, reg.UUID, reg.Token, config.RequestTimeout)
|
||||
cli := client.New(fixture.baseURL, cfg.Runner.TLSConfig(), reg.UUID, reg.Token, config.RequestTimeout, cfg.Runner.ExtraHeaders)
|
||||
|
||||
runner := run.NewRunner(cfg, reg, cli)
|
||||
declResp, err := runner.Declare(ctx, labelNames)
|
||||
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
name: checkout
|
||||
on: push
|
||||
jobs:
|
||||
checkout:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- id: checkout
|
||||
uses: builtin:checkout
|
||||
- uses: builtin:checkout
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
path: short
|
||||
- run: test "$(git -C short rev-parse HEAD)" = "$GITHUB_SHA"
|
||||
- if: github.ref_type == 'branch'
|
||||
run: |
|
||||
test "$(git symbolic-ref HEAD)" = "$GITHUB_REF"
|
||||
test "${{ steps.checkout.outputs.ref }}" = "$GITHUB_REF"
|
||||
test "${{ steps.checkout.outputs.commit }}" = "$GITHUB_SHA"
|
||||
git -c user.name=e2e -c [email protected] commit --allow-empty -m pushed
|
||||
git push
|
||||
git tag moved
|
||||
git push --force origin moved
|
||||
git switch --orphan child
|
||||
echo child > child.txt
|
||||
git add child.txt
|
||||
git -c user.name=e2e -c [email protected] commit -m child
|
||||
git push origin child
|
||||
git switch -c content main
|
||||
git submodule add -b child "../${{ github.event.repository.name }}.git" sub
|
||||
git lfs install --local
|
||||
git lfs track '*.bin'
|
||||
mkdir keep drop
|
||||
echo big > keep/big.bin
|
||||
echo big > drop/big.bin
|
||||
git add .gitattributes keep drop
|
||||
git -c user.name=e2e -c [email protected] commit -m content
|
||||
git push origin content
|
||||
- if: github.ref_type == 'branch'
|
||||
uses: builtin:checkout
|
||||
with:
|
||||
ref: content
|
||||
path: full
|
||||
submodules: true
|
||||
lfs: true
|
||||
- if: github.ref_type == 'branch'
|
||||
uses: builtin:checkout
|
||||
with:
|
||||
ref: content
|
||||
path: sparse
|
||||
sparse-checkout: keep
|
||||
lfs: true
|
||||
- if: github.ref_type == 'branch'
|
||||
run: |
|
||||
test "$(cat full/sub/child.txt)" = child
|
||||
test "$(cat full/keep/big.bin)" = big
|
||||
test "$(cat sparse/keep/big.bin)" = big
|
||||
test ! -e sparse/drop
|
||||
- if: github.ref_type == 'tag'
|
||||
run: test "$(git describe --tags)" = "$GITHUB_REF_NAME"
|
||||
@@ -7,17 +7,17 @@ toolchain go1.27.1
|
||||
require (
|
||||
connectrpc.com/connect v1.21.0
|
||||
dario.cat/mergo v1.0.2
|
||||
gitea.dev/actionslib v1.2.1
|
||||
gitea.dev/actionslib v1.3.0
|
||||
github.com/avast/retry-go/v5 v5.0.0
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/containerd/errdefs v1.0.0
|
||||
github.com/creack/pty v1.1.24
|
||||
github.com/distribution/reference v0.6.0
|
||||
github.com/docker/cli v29.8.1+incompatible
|
||||
github.com/docker/cli v29.8.2+incompatible
|
||||
github.com/docker/go-connections v0.8.1
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/go-git/go-billy/v5 v5.9.1
|
||||
github.com/go-git/go-billy/v5 v5.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.2
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/johannesboyne/gofakes3 v1.2.0
|
||||
|
||||
@@ -4,8 +4,8 @@ cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik=
|
||||
cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
gitea.dev/actionslib v1.2.1 h1:GL//K/0zIZV6h1OOksv1awvFVg0rg7fug2xWcG7mkG0=
|
||||
gitea.dev/actionslib v1.2.1/go.mod h1:1+gqOKGSEPn2IFHgY8S3GC5Ld+Hn8jF3OxiFHQ/fpx4=
|
||||
gitea.dev/actionslib v1.3.0 h1:xZRoTL1+sK/PaglT9uXkZJ9g0nk1WLMEH4FUXdXpKKw=
|
||||
gitea.dev/actionslib v1.3.0/go.mod h1:svCefEsavx4jmn8vJ4wTG7Q0KBcGfrFmGUjrhxuNUAQ=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -74,8 +74,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/docker/cli v29.8.1+incompatible h1:qYL1bCp6cRw2SB1xmLlIOPyV171dilw9W2Jew38vy9c=
|
||||
github.com/docker/cli v29.8.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ=
|
||||
github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q=
|
||||
github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
|
||||
@@ -92,8 +92,8 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.9.1 h1:8U73XiOTfINdItHVa6z4Gv7ToObcZ6grkqQbLryLCdA=
|
||||
github.com/go-git/go-billy/v5 v5.9.1/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0=
|
||||
github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE=
|
||||
github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY=
|
||||
|
||||
@@ -170,10 +170,11 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
|
||||
cli := client.New(
|
||||
reg.Address,
|
||||
cfg.Runner.Insecure,
|
||||
cfg.Runner.TLSConfig(),
|
||||
reg.UUID,
|
||||
reg.Token,
|
||||
config.RequestTimeout,
|
||||
cfg.Runner.ExtraHeaders,
|
||||
)
|
||||
|
||||
runner := run.NewRunner(cfg, reg, cli)
|
||||
@@ -208,6 +209,15 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
metrics.RunnerCapacity.Set(float64(cfg.Runner.Capacity))
|
||||
metrics.RegisterUptimeFunc(time.Now())
|
||||
metrics.RegisterRunningJobsFunc(runner.RunningCount, cfg.Runner.Capacity)
|
||||
metrics.RegisterStateFunc(func() string {
|
||||
if runner.RunningCount() > 0 {
|
||||
return metrics.LabelStateBusy
|
||||
}
|
||||
if ready, _ := poller.Ready(cfg.Metrics.ReadinessGrace); !ready {
|
||||
return metrics.LabelStateUnavailable
|
||||
}
|
||||
return metrics.LabelStateIdle
|
||||
})
|
||||
if cfg.Metrics.Enabled {
|
||||
metrics.StartServer(ctx, cfg.Metrics.Addr, func() (bool, string) {
|
||||
return poller.Ready(cfg.Metrics.ReadinessGrace)
|
||||
|
||||
@@ -128,6 +128,7 @@ func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
|
||||
{"ephemeral interrupted before task", true, true, nil, nil, false},
|
||||
{"persistent once", false, false, nil, nil, false},
|
||||
{"ephemeral rejected on fetch", true, false, nil, errUnregisteredRunner, true},
|
||||
{"ephemeral rejected on fetch with 401", true, false, nil, connect.NewError(connect.CodeUnauthenticated, errors.New("unregistered runner")), true},
|
||||
{"ephemeral proxy 401 on fetch", true, false, nil, connect.NewError(connect.CodeUnauthenticated, errors.New("HTTP status 401 Unauthorized")), false},
|
||||
{"ephemeral rejected on declare", true, false, errUnregisteredRunner, nil, true},
|
||||
{"persistent rejected on declare", false, false, errUnregisteredRunner, nil, false},
|
||||
|
||||
@@ -363,10 +363,11 @@ func doRegister(ctx context.Context, cfg *config.Config, inputs *registerInputs)
|
||||
// initial http client
|
||||
cli := client.New(
|
||||
inputs.InstanceAddr,
|
||||
cfg.Runner.Insecure,
|
||||
cfg.Runner.TLSConfig(),
|
||||
"",
|
||||
"",
|
||||
config.RequestTimeout,
|
||||
cfg.Runner.ExtraHeaders,
|
||||
)
|
||||
|
||||
if _, err := cli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{
|
||||
|
||||
@@ -206,6 +206,9 @@ func (p *Poller) Ready(grace time.Duration) (bool, string) {
|
||||
if p.unregistered.Load() {
|
||||
return false, "runner is no longer registered"
|
||||
}
|
||||
if p.pollingCtx.Err() != nil {
|
||||
return false, "runner is shutting down"
|
||||
}
|
||||
p.availabilityMu.Lock()
|
||||
ready, reason := p.availabilityReady, p.availabilityReason
|
||||
p.availabilityMu.Unlock()
|
||||
@@ -394,7 +397,7 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
|
||||
p.lastPollFailed.Store(true)
|
||||
s.consecutiveErrors++
|
||||
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask, connect.CodeOf(err).String()).Inc()
|
||||
return nil, false
|
||||
}
|
||||
p.markHealthyPoll()
|
||||
|
||||
@@ -110,8 +110,8 @@ func TestPoller_FetchUnauthenticatedStopsPolling(t *testing.T) {
|
||||
fetchErr error
|
||||
rejected bool
|
||||
}{
|
||||
"connect": {connect_go.NewError(connect_go.CodeUnauthenticated, errors.New("unregistered runner")), false},
|
||||
"gitea": {connect_go.NewWireError(connect_go.CodeUnknown, errors.New("rpc error: code = Unauthenticated desc = unregistered runner")), true},
|
||||
"proxy": {connect_go.NewError(connect_go.CodeUnauthenticated, errors.New("401 Unauthorized")), false},
|
||||
"gitea": {connect_go.NewWireError(connect_go.CodeUnknown, errors.New("rpc error: code = Unauthenticated desc = unregistered runner")), true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
client := mocks.NewClient(t)
|
||||
@@ -227,6 +227,11 @@ func TestPollerReady(t *testing.T) {
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "unable to poll Gitea", reason)
|
||||
|
||||
poller.shutdownPolling()
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "runner is shutting down", reason)
|
||||
|
||||
poller.unregistered.Store(true)
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
|
||||
@@ -74,6 +74,7 @@ type Runner struct {
|
||||
capabilities string
|
||||
|
||||
isolatedCacheContainer func() string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
runningTasks sync.Map
|
||||
runningCount atomic.Int64
|
||||
@@ -140,6 +141,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
runHealthCheck: executeHealthCheck,
|
||||
}
|
||||
runner.isolatedCacheContainer = sync.OnceValue(runner.detectIsolatedCacheContainer)
|
||||
runner.instanceTransport = client.NewTransport(cli.Address(), cfg.Runner.TLSConfig())
|
||||
return runner
|
||||
}
|
||||
|
||||
@@ -332,7 +334,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
||||
}
|
||||
|
||||
metrics.JobDuration.Observe(time.Since(start).Seconds())
|
||||
metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc()
|
||||
metrics.JobsTotal.WithLabelValues(metrics.JobStatus(ctx, reporter.Result())).Inc()
|
||||
endJob(reporter.Result())
|
||||
}()
|
||||
reporter.RunDaemon()
|
||||
@@ -615,6 +617,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
},
|
||||
ContainerOptions: r.cfg.Container.Options,
|
||||
ServiceReadyTimeout: r.cfg.Container.ServiceReadyTimeout,
|
||||
NoSweep: r.cfg.Container.Sweep != nil && !*r.cfg.Container.Sweep,
|
||||
ContainerDaemonSocket: r.cfg.Container.DockerHost,
|
||||
Privileged: r.cfg.Container.Privileged,
|
||||
DefaultActionInstance: r.getDefaultActionsURL(task),
|
||||
@@ -626,6 +629,9 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
ValidVolumes: r.cfg.Container.ValidVolumes,
|
||||
SharedToolCache: r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
|
||||
InsecureSkipTLS: r.cfg.Runner.Insecure,
|
||||
ClientCertFile: r.cfg.Runner.ClientCertFile,
|
||||
ClientKeyFile: r.cfg.Runner.ClientKeyFile,
|
||||
InstanceTransport: r.instanceTransport,
|
||||
RunnerName: r.name,
|
||||
KubernetesPicker: r.kubernetesOptions,
|
||||
}
|
||||
|
||||
@@ -20,4 +20,7 @@ type Context struct {
|
||||
NoSkipCheckout bool
|
||||
UseGitIgnore bool
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile string
|
||||
ClientKeyFile string
|
||||
Env map[string]string
|
||||
}
|
||||
@@ -6,21 +6,37 @@ package checkout
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/kballard/go-shellquote"
|
||||
)
|
||||
|
||||
var inputNames = []string{"repository", "ref", "token", "path", "fetch-depth"}
|
||||
var inputNames = []string{
|
||||
"repository", "ref", "token", "ssh-key", "ssh-known-hosts", "ssh-strict", "ssh-user", "persist-credentials", "path", "clean",
|
||||
"filter", "sparse-checkout", "sparse-checkout-cone-mode", "fetch-depth", "fetch-tags", "show-progress", "lfs", "submodules",
|
||||
"set-safe-directory",
|
||||
}
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) error {
|
||||
const credentialsInclude = "includeIf.gitdir:**.path" // the form actions/checkout removes from a repository it checks out again
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) (err error) {
|
||||
for name := range c.Inputs {
|
||||
if !slices.Contains(inputNames, name) {
|
||||
return fmt.Errorf("checkout: unsupported input %q", name)
|
||||
@@ -37,38 +53,402 @@ func Main(ctx context.Context, c *action.Context) error {
|
||||
|
||||
ghc := c.Github
|
||||
repository := cmp.Or(c.Inputs["repository"], ghc.Repository)
|
||||
ref := shortBranch(c.Inputs["ref"])
|
||||
isWorkflowRepo := repository == ghc.Repository
|
||||
local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, ref) && !c.NoSkipCheckout
|
||||
if local && !c.BindWorkdir {
|
||||
return c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx)
|
||||
}
|
||||
if local && dst == c.Workspace {
|
||||
ref := c.Inputs["ref"]
|
||||
sshKey := strings.TrimSpace(c.Inputs["ssh-key"])
|
||||
isWorkflowRepo := strings.EqualFold(repository, ghc.Repository)
|
||||
local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, shortBranch(ref)) && !c.NoSkipCheckout && sshKey == ""
|
||||
if local && (!c.BindWorkdir || dst == c.Workspace) {
|
||||
if !c.BindWorkdir {
|
||||
if err := c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if output := c.Env["GITHUB_OUTPUT"]; output != "" {
|
||||
return c.Container.Copy(path.Dir(output), &container.FileEntry{Name: path.Base(output), Mode: 0o666, Body: fmt.Sprintf("ref=%s\ncommit=%s\n", ghc.Ref, ghc.Sha)})(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if ref == "" {
|
||||
ref = "HEAD"
|
||||
if isWorkflowRepo {
|
||||
ref = ghc.Sha
|
||||
co := &checkout{
|
||||
url: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository,
|
||||
dst: dst,
|
||||
ref: ref,
|
||||
depth: depth,
|
||||
fetchTags: boolInput(c.Inputs, "fetch-tags", false),
|
||||
clean: boolInput(c.Inputs, "clean", true),
|
||||
lfs: boolInput(c.Inputs, "lfs", false),
|
||||
progress: boolInput(c.Inputs, "show-progress", true),
|
||||
cone: boolInput(c.Inputs, "sparse-checkout-cone-mode", true),
|
||||
filter: strings.TrimSpace(c.Inputs["filter"]),
|
||||
output: c.Env["GITHUB_OUTPUT"],
|
||||
}
|
||||
for pattern := range strings.Lines(c.Inputs["sparse-checkout"]) {
|
||||
if pattern = strings.TrimSpace(pattern); pattern != "" {
|
||||
co.sparse = append(co.sparse, pattern)
|
||||
}
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "checkout-")
|
||||
if err != nil {
|
||||
if len(co.sparse) > 0 {
|
||||
co.filter = cmp.Or(co.filter, "blob:none")
|
||||
}
|
||||
if submodules := strings.ToLower(strings.TrimSpace(c.Inputs["submodules"])); submodules == "true" || submodules == "recursive" {
|
||||
co.submodules = submodules
|
||||
}
|
||||
if sshKey != "" {
|
||||
if co.url, err = sshURL(ghc, repository, strings.TrimSpace(c.Inputs["ssh-user"])); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
co.objectFormat = git.ObjectFormat(ref)
|
||||
switch {
|
||||
case co.objectFormat != "":
|
||||
co.sha, co.ref = ref, ""
|
||||
case ref == "" && isWorkflowRepo && strings.HasPrefix(ghc.Ref, "refs/"):
|
||||
co.sha, co.ref = ghc.Sha, ghc.Ref
|
||||
case ref == "" && isWorkflowRepo: // a merged pull request's ref is its merge commit
|
||||
co.sha = ghc.Sha
|
||||
case ref == "HEAD":
|
||||
co.ref = ""
|
||||
}
|
||||
if isWorkflowRepo {
|
||||
co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(ghc.Sha))
|
||||
}
|
||||
|
||||
// named like actions/checkout's, which create-pull-request knows to hide, and gone with the job
|
||||
runnerTemp, _ := c.Container.GetRunnerContext(ctx)["temp"].(string)
|
||||
digest := sha256.Sum256([]byte(dst))
|
||||
credentials := fmt.Sprintf("git-credentials-%x.config", digest[:8])
|
||||
co.credentials = path.Join(runnerTemp, credentials)
|
||||
persist := boolInput(c.Inputs, "persist-credentials", true)
|
||||
config, remoteConfig := serverConfig(ghc.ServerURL, cmp.Or(c.Inputs["token"], ghc.Token), c.InsecureSkipTLS, co.submodules != "" && sshKey == "")
|
||||
if boolInput(c.Inputs, "set-safe-directory", true) {
|
||||
remoteConfig = append(remoteConfig, "safe.directory=*") // a bound workdir belongs to the host user
|
||||
}
|
||||
if c.ClientCertFile != "" { // only for this step, later steps never get the runner's key
|
||||
var certPEM, keyPEM []byte
|
||||
if certPEM, err = os.ReadFile(c.ClientCertFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if keyPEM, err = os.ReadFile(c.ClientKeyFile); err != nil {
|
||||
return err
|
||||
}
|
||||
cert, key := fmt.Sprintf("git-client-cert-%x.pem", digest[:8]), fmt.Sprintf("git-client-key-%x.pem", digest[:8])
|
||||
defer func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx))
|
||||
}()
|
||||
if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: cert, Mode: 0o600, Body: string(certPEM)}, &container.FileEntry{Name: key, Mode: 0o600, Body: string(keyPEM)})(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
scope := "http." + strings.TrimSuffix(ghc.ServerURL, "/") + "/."
|
||||
remoteConfig = append(remoteConfig, scope+"sslCert="+path.Join(runnerTemp, cert), scope+"sslKey="+path.Join(runnerTemp, key))
|
||||
}
|
||||
env := gitEnv(c, remoteConfig)
|
||||
env["GIT_LFS_SKIP_SMUDGE"] = strconv.FormatBool(!co.lfs || len(co.sparse) == 0) // lfs pull fetches the objects of a full checkout in one go
|
||||
if sshKey != "" {
|
||||
key := fmt.Sprintf("git-ssh-key-%x", digest[:8])
|
||||
files := []*container.FileEntry{{Name: key, Mode: 0o600, Body: sshKey + "\n"}}
|
||||
ssh := []string{"ssh", "-i", path.Join(runnerTemp, key)}
|
||||
if boolInput(c.Inputs, "ssh-strict", true) {
|
||||
ssh = append(ssh, "-o", "StrictHostKeyChecking=yes", "-o", "CheckHostIP=no")
|
||||
}
|
||||
if knownHosts := strings.TrimSpace(c.Inputs["ssh-known-hosts"]); knownHosts != "" {
|
||||
name := fmt.Sprintf("git-ssh-known-hosts-%x", digest[:8])
|
||||
files = append(files, &container.FileEntry{Name: name, Mode: 0o600, Body: knownHosts + "\n"})
|
||||
ssh = append(ssh, "-o", "UserKnownHostsFile=~/.ssh/known_hosts "+strconv.Quote(path.Join(runnerTemp, name)))
|
||||
}
|
||||
if !persist {
|
||||
defer func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx))
|
||||
}()
|
||||
}
|
||||
if err := c.Container.Copy(runnerTemp, files...)(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
env["GIT_SSH_COMMAND"] = shellquote.Join(ssh...)
|
||||
config += "[core]\n\tsshCommand = " + strconv.Quote(env["GIT_SSH_COMMAND"]) + "\n"
|
||||
}
|
||||
if !persist {
|
||||
config = ""
|
||||
}
|
||||
|
||||
if co.objectFormat == "" || co.sha == "" && !strings.HasPrefix(co.ref, "refs/") {
|
||||
advertised, err := lsRemote(ctx, c, env, co.url, co.ref)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checkout: %s@%s: %w", repository, ref, err)
|
||||
}
|
||||
if co.resolve(advertised); co.ref == "" && co.sha == "" {
|
||||
return fmt.Errorf("checkout: %s has no ref %q", repository, ref)
|
||||
}
|
||||
}
|
||||
co.outputRef = co.ref
|
||||
if ref != co.sha {
|
||||
co.outputRef = cmp.Or(ref, co.ref) // as given, like actions/checkout
|
||||
}
|
||||
if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: credentials, Mode: 0o600, Body: config})(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
if err := git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
URL: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository,
|
||||
Ref: ref,
|
||||
Dir: dir,
|
||||
Token: cmp.Or(c.Inputs["token"], ghc.Token),
|
||||
Depth: depth,
|
||||
InsecureSkipTLS: c.InsecureSkipTLS,
|
||||
})(ctx); err != nil {
|
||||
return fmt.Errorf("checkout: clone %s@%s: %w", repository, ref, err)
|
||||
|
||||
clone := func(commands [][]string) error {
|
||||
if err := run(ctx, c, env, commands); err != nil {
|
||||
return fmt.Errorf("checkout: clone %s@%s: %w", repository, cmp.Or(co.ref, co.sha), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.Container.CopyDir(dst, dir+string(filepath.Separator)+".", false, false)(ctx)
|
||||
commands, fetched := co.commands()
|
||||
if co.sha != "" && strings.HasPrefix(co.ref, "refs/tags/") {
|
||||
if err := clone(commands[:fetched]); err != nil {
|
||||
return err
|
||||
}
|
||||
tagged, err := output(ctx, c, env, "git", "-C", dst, "rev-parse", "--verify", "--quiet", co.ref+"^{commit}")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checkout: %s: %w", co.ref, err)
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(tagged), co.sha) {
|
||||
return fmt.Errorf("checkout: %s does not point to the expected commit %s, it may have been moved after the workflow was triggered", co.ref, co.sha)
|
||||
}
|
||||
commands = commands[fetched:]
|
||||
}
|
||||
return clone(commands)
|
||||
}
|
||||
|
||||
func gitEnv(c *action.Context, config []string) map[string]string {
|
||||
env := map[string]string{"GIT_TERMINAL_PROMPT": "0"}
|
||||
maps.Copy(env, c.Env)
|
||||
maps.DeleteFunc(env, func(name, _ string) bool { return slices.Contains(git.LocalEnvVars, name) })
|
||||
maps.Copy(env, git.ConfigEnv(append([]string{"protocol.version=2", "init.defaultBranch=main", "advice.detachedHead=false", "maintenance.auto=false"}, config...)...))
|
||||
return env
|
||||
}
|
||||
|
||||
func run(ctx context.Context, c *action.Context, env map[string]string, commands [][]string) error {
|
||||
if _, host := c.Container.(*container.HostEnvironment); !host || runtime.GOOS != "windows" {
|
||||
script := []string{"set -e", `command -v git >/dev/null || { echo "builtin:checkout needs git on PATH" >&2; exit 127; }`}
|
||||
for _, command := range commands {
|
||||
script = append(script, shellquote.Join(command...))
|
||||
}
|
||||
return c.Container.Exec([]string{"sh", "-c", strings.Join(script, "\n")}, env, "", "")(ctx)
|
||||
}
|
||||
for _, command := range commands {
|
||||
if err := c.Container.Exec(command, env, "", "")(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// output runs where the job runs, so the job's network and git config apply as for the fetch.
|
||||
func output(ctx context.Context, c *action.Context, env map[string]string, command ...string) (string, error) {
|
||||
var out strings.Builder
|
||||
stdout, stderr := c.Container.ReplaceLogWriter(&out, &out)
|
||||
defer c.Container.ReplaceLogWriter(stdout, stderr)
|
||||
if err := c.Container.Exec(command, env, "", "")(ctx); err != nil {
|
||||
return "", fmt.Errorf("%w: %s", err, strings.TrimSpace(out.String()))
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func lsRemote(ctx context.Context, c *action.Context, env map[string]string, url, ref string) (string, error) {
|
||||
patterns := []string{ref}
|
||||
switch {
|
||||
case ref == "":
|
||||
patterns = []string{"HEAD"}
|
||||
case !strings.HasPrefix(ref, "refs/"):
|
||||
patterns = []string{"refs/heads/" + ref, "refs/tags/" + ref}
|
||||
}
|
||||
return output(ctx, c, env, append([]string{"git", "ls-remote", "--symref", url}, patterns...)...)
|
||||
}
|
||||
|
||||
// serverConfig scopes the token to the server, so no other host like a repository's LFS endpoint gets it.
|
||||
func serverConfig(serverURL, token string, insecureSkipTLS, rewriteSSH bool) (file string, env []string) {
|
||||
scope := strings.TrimSuffix(serverURL, "/") + "/"
|
||||
file = fmt.Sprintf("[http %q]\n", scope)
|
||||
for _, entry := range git.RemoteConfig(token, insecureSkipTLS) {
|
||||
setting := strings.TrimPrefix(entry, "http.")
|
||||
key, value, _ := strings.Cut(setting, "=")
|
||||
file += "\t" + key + " = " + value + "\n"
|
||||
env = append(env, "http."+scope+"."+setting)
|
||||
}
|
||||
if server, err := url.Parse(serverURL); rewriteSSH && err == nil && server.Hostname() != "" { // submodules on the server fetch over HTTP with the token
|
||||
file += fmt.Sprintf("[url %q]\n\tinsteadOf = git@%s:\n", scope, server.Hostname())
|
||||
env = append(env, "url."+scope+".insteadOf=git@"+server.Hostname()+":")
|
||||
}
|
||||
return file, env
|
||||
}
|
||||
|
||||
func sshURL(ghc *model.GithubContext, repository, user string) (string, error) {
|
||||
endpoint := &url.URL{Scheme: "ssh"}
|
||||
if advertised, _ := model.NestedMapLookup(ghc.Event, "repository", "ssh_url").(string); advertised != "" {
|
||||
if colon := strings.LastIndex(advertised, ":"); !strings.HasPrefix(advertised, "ssh://") && colon >= 0 {
|
||||
advertised = "ssh://" + advertised[:colon] + "/" + advertised[colon+1:] // scp-like git@host:owner/repo.git
|
||||
}
|
||||
var err error
|
||||
if endpoint, err = url.Parse(advertised); err != nil {
|
||||
return "", fmt.Errorf("checkout: parse the event's ssh_url: %w", err)
|
||||
}
|
||||
} else if server, err := url.Parse(ghc.ServerURL); err == nil {
|
||||
endpoint.Host = server.Hostname()
|
||||
}
|
||||
endpoint.User = url.User(cmp.Or(user, endpoint.User.Username(), "git"))
|
||||
endpoint.Path = "/" + repository + ".git"
|
||||
return endpoint.String(), nil
|
||||
}
|
||||
|
||||
type checkout struct {
|
||||
url, dst, objectFormat, credentials, output, outputRef string
|
||||
ref, sha string // ref is fully qualified or empty, sha pins it when known
|
||||
filter, submodules string
|
||||
sparse []string
|
||||
depth int
|
||||
fetchTags, clean, lfs, progress, cone bool
|
||||
}
|
||||
|
||||
const fullDepth = "2147483647" // git's INFINITE_DEPTH, also deepens a shallow dst
|
||||
|
||||
// commands checks out co.dst, the first `fetched` of them fetch so a fetched tag can be verified before the rest.
|
||||
func (co *checkout) commands() (commands [][]string, fetched int) {
|
||||
repo := func(args ...string) []string { return append([]string{"git", "-C", co.dst}, args...) }
|
||||
commands = [][]string{
|
||||
{"git", "init", "--quiet", "--object-format=" + co.objectFormat, co.dst},
|
||||
repo("config", "remote.origin.url", co.url), // `remote add` fails on a repeated checkout
|
||||
repo("config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"),
|
||||
repo("config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials),
|
||||
}
|
||||
if co.lfs {
|
||||
commands = append(commands, repo("lfs", "install", "--local"))
|
||||
}
|
||||
|
||||
depth := strconv.Itoa(co.depth)
|
||||
var refSpecs []string
|
||||
if co.depth == 0 {
|
||||
depth = fullDepth
|
||||
refSpecs = append(refSpecs, "+refs/heads/*:refs/remotes/origin/*")
|
||||
}
|
||||
fetch := []string{"fetch", "--force", "--no-tags", "--no-recurse-submodules", "--depth", depth}
|
||||
if co.progress {
|
||||
fetch = append(fetch, "--progress")
|
||||
} else {
|
||||
fetch = append(fetch, "--no-progress")
|
||||
}
|
||||
if co.filter != "" {
|
||||
fetch = append(fetch, "--filter="+co.filter)
|
||||
}
|
||||
if co.fetchTags || co.depth == 0 {
|
||||
refSpecs = append(refSpecs, "+refs/tags/*:refs/tags/*")
|
||||
}
|
||||
tag := strings.HasPrefix(co.ref, "refs/tags/") // fetched itself, which keeps an annotated tag's object
|
||||
pinned := co.sha != "" && co.ref != "" && !tag
|
||||
if co.ref != "" && !pinned {
|
||||
refSpecs = append(refSpecs, "+"+co.ref+":"+trackingRef(co.ref))
|
||||
}
|
||||
if co.sha != "" && !tag {
|
||||
refSpecs = append(refSpecs, co.sha)
|
||||
}
|
||||
commands = append(commands, repo(append(append(fetch, "origin"), refSpecs...)...))
|
||||
if pinned {
|
||||
commands = append(commands, repo("update-ref", trackingRef(co.ref), co.sha))
|
||||
}
|
||||
fetched = len(commands)
|
||||
|
||||
if len(co.sparse) > 0 {
|
||||
mode := "--cone"
|
||||
if !co.cone {
|
||||
mode = "--no-cone"
|
||||
}
|
||||
commands = append(commands, repo("sparse-checkout", "init", mode), repo(append([]string{"sparse-checkout", "set", "--"}, co.sparse...)...))
|
||||
}
|
||||
if branch, ok := strings.CutPrefix(co.ref, "refs/heads/"); ok {
|
||||
commands = append(commands, repo("checkout", "--force", "--ignore-other-worktrees", "-B", branch, "--track", trackingRef(co.ref)))
|
||||
} else {
|
||||
commands = append(commands, repo("checkout", "--force", cmp.Or(co.sha, trackingRef(co.ref))))
|
||||
}
|
||||
if len(co.sparse) == 0 { // after the checkout, so a sparse dst of another repository needs nothing of it
|
||||
commands = append(commands, repo("sparse-checkout", "disable"), repo("config", "--local", "--unset-all", "extensions.worktreeConfig"))
|
||||
}
|
||||
if co.lfs && len(co.sparse) == 0 {
|
||||
commands = append(commands, repo("lfs", "pull", "origin"))
|
||||
}
|
||||
if co.clean {
|
||||
commands = append(commands, repo("clean", "-ffdx"))
|
||||
}
|
||||
if co.submodules != "" {
|
||||
commands = append(commands, co.submoduleCommands(repo)...)
|
||||
}
|
||||
if co.output != "" {
|
||||
commands = append(commands, repo("log", "-1", "--format=commit=%H%nref="+strings.ReplaceAll(co.outputRef, "%", "%%"), "--output="+co.output))
|
||||
}
|
||||
return commands, fetched
|
||||
}
|
||||
|
||||
func (co *checkout) submoduleCommands(repo func(args ...string) []string) [][]string {
|
||||
var recursive []string
|
||||
if co.submodules == "recursive" {
|
||||
recursive = []string{"--recursive"}
|
||||
}
|
||||
foreach := func(commands ...[]string) []string {
|
||||
joined := make([]string, len(commands))
|
||||
for i, command := range commands {
|
||||
joined[i] = shellquote.Join(command...)
|
||||
}
|
||||
return repo(append(append([]string{"submodule", "foreach"}, recursive...), strings.Join(joined, " && "))...)
|
||||
}
|
||||
commands := [][]string{repo(append([]string{"submodule", "sync"}, recursive...)...)}
|
||||
if co.clean {
|
||||
commands = append(commands, foreach([]string{"git", "clean", "-ffdx"}))
|
||||
}
|
||||
update := append([]string{"submodule", "update", "--init", "--force"}, recursive...)
|
||||
if co.depth > 0 {
|
||||
update = append(update, "--depth="+strconv.Itoa(co.depth))
|
||||
}
|
||||
include := []string{"git", "config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials}
|
||||
if co.lfs {
|
||||
return append(commands, repo(update...), foreach([]string{"git", "lfs", "install", "--local"}, []string{"git", "lfs", "pull", "origin"}, include))
|
||||
}
|
||||
return append(commands, repo(update...), foreach(include))
|
||||
}
|
||||
|
||||
func (co *checkout) resolve(advertised string) {
|
||||
ref := co.ref
|
||||
if !strings.HasPrefix(ref, "refs/") {
|
||||
co.ref = ""
|
||||
}
|
||||
for line := range strings.Lines(advertised) {
|
||||
value, name, _ := strings.Cut(strings.TrimSpace(line), "\t")
|
||||
if target, ok := strings.CutPrefix(value, "ref: "); ok {
|
||||
if name == "HEAD" {
|
||||
co.ref = target
|
||||
}
|
||||
continue
|
||||
}
|
||||
co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(value))
|
||||
switch name {
|
||||
case "HEAD":
|
||||
if co.ref == "" {
|
||||
co.sha = value
|
||||
}
|
||||
case "refs/heads/" + ref:
|
||||
co.ref = name
|
||||
case "refs/tags/" + ref:
|
||||
co.ref = cmp.Or(co.ref, name)
|
||||
}
|
||||
}
|
||||
co.objectFormat = cmp.Or(co.objectFormat, "sha1")
|
||||
}
|
||||
|
||||
func trackingRef(ref string) string {
|
||||
if branch, ok := strings.CutPrefix(ref, "refs/heads/"); ok {
|
||||
return "refs/remotes/origin/" + branch
|
||||
}
|
||||
if pull, ok := strings.CutPrefix(ref, "refs/pull/"); ok {
|
||||
return "refs/remotes/pull/" + pull
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
func boolInput(inputs map[string]string, name string, fallback bool) bool {
|
||||
value := strings.TrimSpace(inputs[name])
|
||||
return strings.EqualFold(value, "true") || value == "" && fallback
|
||||
}
|
||||
|
||||
func workspaceDestination(workspace, checkoutPath string) (string, error) {
|
||||
@@ -91,7 +471,6 @@ func fetchDepth(input string) (int, error) {
|
||||
return depth, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutor resolves branches by short name, a full tag ref stays exact.
|
||||
func shortBranch(ref string) string {
|
||||
func shortBranch(ref string) string { // full tag refs stay exact
|
||||
return strings.TrimPrefix(ref, "refs/heads/")
|
||||
}
|
||||
@@ -4,14 +4,21 @@
|
||||
package checkout
|
||||
|
||||
import (
|
||||
"context"
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
@@ -20,22 +27,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakeContainer struct {
|
||||
container.ExecutionsEnvironment
|
||||
dst, src string
|
||||
log []string
|
||||
}
|
||||
|
||||
func (f *fakeContainer) CopyDir(dst, src string, _, _ bool) common.Executor {
|
||||
return func(context.Context) error {
|
||||
f.dst, f.src = dst, src
|
||||
if out, err := exec.Command("git", "--git-dir", filepath.Join(src, ".git"), "log", "--format=%s").Output(); err == nil {
|
||||
f.log = strings.Fields(string(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
@@ -49,67 +40,317 @@ func runGit(t *testing.T, args ...string) string {
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func gitLog(dir string) []string {
|
||||
out, err := exec.Command("git", "--git-dir", filepath.Join(dir, ".git"), "log", "--format=%s").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return strings.Fields(string(out))
|
||||
}
|
||||
|
||||
func TestCheckout(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1")
|
||||
sha := runGit(t, "-C", work, "rev-parse", "HEAD")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2")
|
||||
runGit(t, "-C", work, "checkout", "-b", "feature")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1")
|
||||
runGit(t, "-C", work, "checkout", "--orphan", "trunk")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2")
|
||||
runGit(t, "-C", work, "tag", "trunk", "main")
|
||||
server := t.TempDir()
|
||||
runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo"))
|
||||
runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo"))
|
||||
var sha string
|
||||
_, lfsErr := exec.LookPath("git-lfs")
|
||||
require.True(t, t.Run("fixtures", func(t *testing.T) {
|
||||
t.Run("history", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1")
|
||||
sha = runGit(t, "-C", work, "rev-parse", "HEAD")
|
||||
runGit(t, "-C", work, "tag", "v1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2")
|
||||
runGit(t, "-C", work, "checkout", "-b", "feature")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1")
|
||||
runGit(t, "-C", work, "checkout", "--orphan", "trunk")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2")
|
||||
runGit(t, "-C", work, "tag", "trunk", "main")
|
||||
runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo"))
|
||||
runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo"))
|
||||
runGit(t, "-C", filepath.Join(server, "other", "repo"), "symbolic-ref", "refs/remotes/origin/HEAD", "refs/heads/main")
|
||||
})
|
||||
t.Run("sha256", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sha256Repo := filepath.Join(server, "org", "sha256")
|
||||
runGit(t, "init", "--object-format=sha256", "--initial-branch=main", sha256Repo)
|
||||
runGit(t, "-C", sha256Repo, "commit", "--allow-empty", "-m", "s1")
|
||||
runGit(t, "-C", sha256Repo, "checkout", "--detach")
|
||||
})
|
||||
t.Run("submodules", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
publish(t, server, "org/nested", map[string]string{"nested.txt": "nested"}, nil)
|
||||
publish(t, server, "org/child", map[string]string{"child.txt": "child"}, map[string]string{"nested": "../nested"})
|
||||
publish(t, server, "org/content", map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b"}, map[string]string{"sub": "../child"})
|
||||
runGit(t, "-C", filepath.Join(server, "org", "content"), "config", "uploadpack.allowFilter", "true")
|
||||
})
|
||||
t.Run("lfs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
oid := fmt.Sprintf("%x", sha256.Sum256([]byte("big")))
|
||||
pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize 3\n"
|
||||
publish(t, server, "org/lfs", map[string]string{".gitattributes": "*.bin filter=lfs diff=lfs merge=lfs -text\n", "keep/big.bin": pointer, "drop/big.bin": pointer}, nil)
|
||||
object := filepath.Join(server, "org", "lfs", "lfs", "objects", oid[:2], oid[2:4], oid)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(object), 0o755))
|
||||
require.NoError(t, os.WriteFile(object, []byte("big"), 0o644))
|
||||
})
|
||||
}))
|
||||
host := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(host, "marker"), []byte("marker"), 0o644))
|
||||
tools := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tools, "ssh"), []byte(`#!/bin/sh
|
||||
test -s "$2" || exit 1
|
||||
printf '%s\n' "$@" > "$SSH_ARGS"
|
||||
for last; do :; done
|
||||
exec sh -c "$(printf '%s' "$last" | sed "s#git-upload-pack '/\(.*\)\.git'#git upload-pack '$SSH_SERVER/\1'#")"
|
||||
`), 0o755))
|
||||
globalConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(globalConfig, []byte("[protocol \"file\"]\n\tallow = always\n"), 0o644))
|
||||
sshEvent := func(url string) map[string]any { return map[string]any{"repository": map[string]any{"ssh_url": url}} }
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
bind, noSkip bool
|
||||
inputs map[string]string
|
||||
dst string
|
||||
log []string
|
||||
hostCopy, err bool
|
||||
name string
|
||||
bind, noSkip bool
|
||||
eventRef string
|
||||
event map[string]any
|
||||
before, inputs map[string]string
|
||||
dst, head, err string
|
||||
log, refs []string
|
||||
files, config map[string]string
|
||||
}{
|
||||
{name: "bound workspace is skipped", bind: true},
|
||||
{name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "/workspace/src", hostCopy: true},
|
||||
{name: "bound path clones event sha", bind: true, inputs: map[string]string{"path": "src"}, dst: "/workspace/src", log: []string{"c1"}},
|
||||
{name: "no-skip clones event sha", noSkip: true, dst: "/workspace", log: []string{"c1"}},
|
||||
{name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, dst: "/workspace", log: []string{"f1"}},
|
||||
{name: "tag ref wins over same-named branch", inputs: map[string]string{"ref": "refs/tags/trunk"}, dst: "/workspace", log: []string{"c2"}},
|
||||
{name: "fetch-depth 0 clones full history", inputs: map[string]string{"ref": "feature", "fetch-depth": "0"}, dst: "/workspace", log: []string{"f1", "c2", "c1"}},
|
||||
{name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, dst: "/workspace", log: []string{"t2"}},
|
||||
{name: "unsupported input", inputs: map[string]string{"submodules": "true"}, err: true},
|
||||
{name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: true},
|
||||
{name: "absolute path", inputs: map[string]string{"path": "/x"}, err: true},
|
||||
{name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: true},
|
||||
{name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "src", files: map[string]string{"marker": "marker"}},
|
||||
{name: "bound path checks out the event branch", bind: true, inputs: map[string]string{"path": "src"}, dst: "src", head: "main", log: []string{"c1"}},
|
||||
{name: "no-skip checks out the event branch", noSkip: true, head: "main", log: []string{"c1"}, refs: []string{"refs/remotes/origin/main"}},
|
||||
{name: "event tag is kept", noSkip: true, eventRef: "refs/tags/v1", log: []string{"c1"}, refs: []string{"refs/tags/v1"}},
|
||||
{name: "moved event tag fails", noSkip: true, eventRef: "refs/tags/trunk", err: "does not point to the expected commit " + sha},
|
||||
{name: "deleted event tag fails", noSkip: true, eventRef: "refs/tags/deleted", err: "checkout: clone"},
|
||||
{
|
||||
name: "sha input removes previously persisted credentials without clean", noSkip: true, before: map[string]string{"ref": sha},
|
||||
inputs: map[string]string{"ref": sha, "persist-credentials": "false", "clean": "false"},
|
||||
log: []string{"c1"},
|
||||
},
|
||||
{name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, head: "feature", log: []string{"f1"}},
|
||||
{name: "tag ref wins over same-named branch and is kept", inputs: map[string]string{"ref": "refs/tags/trunk"}, log: []string{"c2"}, refs: []string{"refs/tags/trunk"}},
|
||||
{name: "branch ref wins over same-named tag", inputs: map[string]string{"ref": "refs/heads/trunk"}, head: "trunk", log: []string{"t2"}},
|
||||
{
|
||||
name: "short branch wins over same-named tag and fetch-tags fetches every tag", inputs: map[string]string{"ref": "trunk", "fetch-tags": "true"},
|
||||
head: "trunk", log: []string{"t2"}, refs: []string{"refs/tags/trunk", "refs/tags/v1"},
|
||||
},
|
||||
{name: "short tag", inputs: map[string]string{"ref": "v1"}, log: []string{"c1"}, refs: []string{"refs/tags/v1"}},
|
||||
{
|
||||
name: "fetch-depth 0 deepens a shallow checkout to every branch and tag", before: map[string]string{"ref": "feature"},
|
||||
inputs: map[string]string{"ref": "refs/heads/feature", "fetch-depth": "0"}, head: "feature", log: []string{"f1", "c2", "c1"},
|
||||
refs: []string{"refs/remotes/origin/feature", "refs/remotes/origin/main", "refs/remotes/origin/trunk", "refs/tags/trunk", "refs/tags/v1"},
|
||||
},
|
||||
{name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, head: "trunk", log: []string{"t2"}},
|
||||
{name: "explicit HEAD checks out the default branch", inputs: map[string]string{"repository": "other/repo", "ref": "HEAD"}, head: "trunk", log: []string{"t2"}},
|
||||
{name: "detached sha256 HEAD is checked out", inputs: map[string]string{"repository": "org/sha256"}, log: []string{"s1"}},
|
||||
{
|
||||
name: "ssh key with the advertised endpoint persists", event: sshEvent("ssh://[email protected]:2222/org/repo.git"),
|
||||
inputs: map[string]string{"ssh-key": "KEY", "ssh-user": "me", "ssh-known-hosts": "ssh.example ssh-ed25519 AAAA"},
|
||||
head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://[email protected]:2222/org/repo.git"},
|
||||
},
|
||||
{
|
||||
name: "ssh key without persisting is emptied", event: sshEvent("[email protected]:org/repo.git"),
|
||||
inputs: map[string]string{"ssh-key": "KEY", "persist-credentials": "false"},
|
||||
head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://[email protected]/org/repo.git"},
|
||||
},
|
||||
{
|
||||
name: "sparse cone checkout filters blobs", inputs: map[string]string{"repository": "org/content", "sparse-checkout": "a\n", "show-progress": "false"},
|
||||
head: "main", log: []string{"content"}, files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": ""},
|
||||
config: map[string]string{"remote.origin.partialclonefilter": "blob:none"},
|
||||
},
|
||||
{
|
||||
name: "non-cone sparse checkout with an explicit filter", head: "main", log: []string{"content"},
|
||||
inputs: map[string]string{"repository": "org/content", "sparse-checkout": "b/b.txt", "sparse-checkout-cone-mode": "false", "filter": "blob:limit=1024"},
|
||||
files: map[string]string{"root.txt": "", "a/a.txt": "", "b/b.txt": "b"}, config: map[string]string{"remote.origin.partialclonefilter": "blob:limit=1024"},
|
||||
},
|
||||
{
|
||||
name: "full checkout after a sparse one restores every file", before: map[string]string{"repository": "org/content", "sparse-checkout": "a"},
|
||||
inputs: map[string]string{"repository": "org/content", "set-safe-directory": "false"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b", "sub/child.txt": ""},
|
||||
},
|
||||
{
|
||||
name: "recursive submodules after plain ones", before: map[string]string{"repository": "org/content", "submodules": "true"},
|
||||
inputs: map[string]string{"repository": "org/content", "submodules": "Recursive"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"sub/child.txt": "child", "sub/nested/nested.txt": "nested"},
|
||||
},
|
||||
{
|
||||
name: "lfs content stays when narrowing a full checkout to a sparse one", before: map[string]string{"repository": "org/lfs", "lfs": "true"},
|
||||
inputs: map[string]string{"repository": "org/lfs", "lfs": "true", "sparse-checkout": "keep"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"keep/big.bin": "big", "drop/big.bin": ""},
|
||||
},
|
||||
{name: "unknown ref", inputs: map[string]string{"ref": "missing"}, err: "has no ref"},
|
||||
{name: "unsupported input", inputs: map[string]string{"github-server-url": "https://example.com"}, err: "unsupported input"},
|
||||
{name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: "must be relative"},
|
||||
{name: "absolute path", inputs: map[string]string{"path": "/x"}, err: "must be relative"},
|
||||
{name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: "non-negative"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fc := &fakeContainer{}
|
||||
err := Main(t.Context(), &action.Context{
|
||||
Container: fc,
|
||||
Github: &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", Sha: sha, ServerURL: server},
|
||||
Inputs: tc.inputs,
|
||||
HostWorkdir: host,
|
||||
Workspace: "/workspace",
|
||||
BindWorkdir: tc.bind,
|
||||
NoSkipCheckout: tc.noSkip,
|
||||
})
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, fc.dst)
|
||||
t.Parallel()
|
||||
if tc.inputs["lfs"] == "true" && lfsErr != nil {
|
||||
t.Skip("needs git-lfs")
|
||||
}
|
||||
workspace, runnerTemp := t.TempDir(), t.TempDir()
|
||||
output := filepath.Join(runnerTemp, "output")
|
||||
checkout := func(inputs map[string]string) error {
|
||||
return Main(t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard},
|
||||
Github: &model.GithubContext{
|
||||
Repository: "org/repo", Ref: cmp.Or(tc.eventRef, "refs/heads/main"), Sha: sha, ServerURL: "file://" + server, Token: "tok", Event: tc.event,
|
||||
},
|
||||
Inputs: inputs,
|
||||
HostWorkdir: host,
|
||||
Workspace: workspace,
|
||||
BindWorkdir: tc.bind,
|
||||
NoSkipCheckout: tc.noSkip,
|
||||
Env: map[string]string{
|
||||
"PATH": tools + string(os.PathListSeparator) + os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": globalConfig, "GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_DIR": host, "GITHUB_OUTPUT": output, "SSH_SERVER": server, "SSH_ARGS": filepath.Join(runnerTemp, "ssh-args"),
|
||||
},
|
||||
})
|
||||
}
|
||||
dst := filepath.Join(workspace, tc.dst)
|
||||
if tc.err != "" {
|
||||
require.ErrorContains(t, checkout(tc.inputs), tc.err)
|
||||
if tc.eventRef == "" {
|
||||
entries, _ := os.ReadDir(workspace)
|
||||
assert.Empty(t, entries)
|
||||
} else {
|
||||
assert.Nil(t, gitLog(dst))
|
||||
}
|
||||
return
|
||||
}
|
||||
first := tc.before
|
||||
if first == nil {
|
||||
first = tc.inputs
|
||||
}
|
||||
require.NoError(t, checkout(first))
|
||||
repos := []string{dst}
|
||||
if fileExists(filepath.Join(dst, "sub", ".git")) {
|
||||
repos = append(repos, filepath.Join(dst, "sub"))
|
||||
}
|
||||
for _, repo := range repos {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repo, "untracked"), nil, 0o644))
|
||||
}
|
||||
require.NoError(t, checkout(tc.inputs))
|
||||
outputs, err := os.ReadFile(output)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.dst, fc.dst)
|
||||
assert.Equal(t, tc.log, fc.log)
|
||||
assert.Equal(t, tc.hostCopy, strings.HasPrefix(fc.src, host))
|
||||
if tc.log != nil {
|
||||
assert.NoDirExists(t, fc.src)
|
||||
assert.Equal(t, tc.log, gitLog(dst))
|
||||
for name, content := range tc.files {
|
||||
body, err := os.ReadFile(filepath.Join(dst, name))
|
||||
assert.Equal(t, content, strings.TrimSpace(string(body)), name)
|
||||
assert.Equal(t, content == "", os.IsNotExist(err), name)
|
||||
}
|
||||
assert.Equal(t, tc.files["marker"] != "", fileExists(filepath.Join(dst, "marker")))
|
||||
for key, value := range tc.config {
|
||||
assert.Equal(t, value, gitOutput(dst, "config", key), key)
|
||||
}
|
||||
if tc.refs != nil {
|
||||
assert.Subset(t, strings.Fields(runGit(t, "-C", dst, "for-each-ref", "--format=%(refname)")), tc.refs)
|
||||
}
|
||||
if tc.log == nil {
|
||||
assert.Contains(t, string(outputs), "commit="+sha+"\n")
|
||||
assert.Contains(t, string(outputs), "ref="+cmp.Or(tc.eventRef, "refs/heads/main")+"\n")
|
||||
return
|
||||
}
|
||||
if tc.head == "" {
|
||||
assert.Empty(t, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
} else {
|
||||
assert.Equal(t, "refs/heads/"+tc.head, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
assert.Equal(t, "origin/"+tc.head, gitOutput(dst, "rev-parse", "--abbrev-ref", "@{u}"))
|
||||
}
|
||||
assert.Contains(t, string(outputs), "commit="+gitOutput(dst, "rev-parse", "HEAD")+"\n")
|
||||
if inputRef := tc.inputs["ref"]; inputRef != "" && inputRef != sha {
|
||||
assert.Contains(t, string(outputs), "ref="+inputRef+"\n")
|
||||
}
|
||||
assert.Equal(t, tc.inputs["sparse-checkout"] != "", gitOutput(dst, "config", "--local", "extensions.worktreeConfig") == "true")
|
||||
persisted := tc.inputs["persist-credentials"] != "false"
|
||||
for _, repo := range repos {
|
||||
assert.Equal(t, tc.inputs["clean"] == "false", fileExists(filepath.Join(repo, "untracked")), repo)
|
||||
assert.Len(t, strings.Fields(gitOutput(repo, "config", "--local", "--get-all", credentialsInclude)), 1, repo)
|
||||
headers, _ := exec.Command("git", "-C", repo, "config", "--get-all", "http.file://"+server+"/.extraheader").Output()
|
||||
assert.Equal(t, persisted, string(headers) == "\nAuthorization: Basic dG9rZW46dG9r\n", repo)
|
||||
}
|
||||
if tc.inputs["ssh-key"] != "" {
|
||||
keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-ssh-key-*"))
|
||||
require.Len(t, keys, 1)
|
||||
key, _ := os.ReadFile(keys[0])
|
||||
if persisted {
|
||||
assert.Equal(t, "KEY\n", string(key))
|
||||
} else {
|
||||
assert.Empty(t, key)
|
||||
}
|
||||
args, _ := os.ReadFile(filepath.Join(runnerTemp, "ssh-args"))
|
||||
assert.Contains(t, string(args), "StrictHostKeyChecking=yes")
|
||||
assert.Equal(t, tc.inputs["ssh-known-hosts"] != "", strings.Contains(string(args), "UserKnownHostsFile="))
|
||||
assert.Equal(t, persisted, gitOutput(dst, "config", "core.sshCommand") != "")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutOffersClientCertOnlyDuringTheStep(t *testing.T) {
|
||||
var gotCert bool
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotCert = len(r.TLS.PeerCertificates) > 0
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
pair := server.TLS.Certificates[0]
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(pair.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
dir, workspace, runnerTemp := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
certFile, keyFile := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: pair.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
|
||||
require.ErrorContains(t, Main(t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard},
|
||||
Github: &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", ServerURL: server.URL},
|
||||
Inputs: map[string]string{"repository": "other/repo"},
|
||||
Workspace: workspace,
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
Env: map[string]string{"PATH": os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_NOSYSTEM": "1"},
|
||||
}), "checkout: other/repo@")
|
||||
require.True(t, gotCert)
|
||||
keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-client-key-*"))
|
||||
require.Len(t, keys, 1)
|
||||
key, err := os.ReadFile(keys[0])
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, key)
|
||||
}
|
||||
|
||||
func publish(t *testing.T, server, name string, files, submodules map[string]string) {
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
for file, content := range files {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(work, file)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(work, file), []byte(content), 0o644))
|
||||
}
|
||||
runGit(t, "-C", work, "add", ".")
|
||||
for path, url := range submodules {
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".path", path)
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".url", url)
|
||||
commit := runGit(t, "--git-dir", filepath.Join(server, name, url), "rev-parse", "main")
|
||||
runGit(t, "-C", work, "update-index", "--add", "--cacheinfo", "160000,"+commit+","+path)
|
||||
runGit(t, "-C", work, "add", ".gitmodules")
|
||||
}
|
||||
runGit(t, "-C", work, "commit", "-m", "content")
|
||||
runGit(t, "clone", "--bare", "--quiet", work, filepath.Join(server, name))
|
||||
}
|
||||
|
||||
func gitOutput(dir string, args ...string) string {
|
||||
out, _ := exec.Command("git", append([]string{"-C", dir}, args...)...).Output()
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func fileExists(name string) bool {
|
||||
_, err := os.Stat(name)
|
||||
return err == nil
|
||||
}
|
||||
@@ -6,6 +6,7 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"gitea.dev/actionslib/runner/v1"
|
||||
@@ -20,9 +21,8 @@ type Client interface {
|
||||
UpdateTask(context.Context, *connect.Request[runnerv1.UpdateTaskRequest]) (*connect.Response[runnerv1.UpdateTaskResponse], error)
|
||||
}
|
||||
|
||||
// IsRegistrationRejected reports whether Gitea itself answered "unregistered runner", unlike a proxy's bare 401.
|
||||
// IsRegistrationRejected reports whether Gitea itself answered "unregistered runner", unlike a proxy's 401.
|
||||
func IsRegistrationRejected(err error) bool {
|
||||
var connectErr *connect.Error
|
||||
return errors.As(err, &connectErr) && connectErr.Code() == connect.CodeUnknown && connect.IsWireError(err) &&
|
||||
connectErr.Message() == "rpc error: code = Unauthenticated desc = unregistered runner"
|
||||
return errors.As(err, &connectErr) && strings.HasSuffix(connectErr.Message(), "unregistered runner")
|
||||
}
|
||||
@@ -6,6 +6,7 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,29 +19,54 @@ import (
|
||||
"gitea.dev/actionslib/runner/v1/runnerv1connect"
|
||||
)
|
||||
|
||||
func getHTTPClient(endpoint string, insecure bool, timeout time.Duration) *http.Client {
|
||||
func NewTransport(endpoint string, tlsConfig *tls.Config) *http.Transport {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
MaxIdleConns: 10,
|
||||
MaxIdleConnsPerHost: 10, // All requests go to one host; default is 2 which causes frequent reconnects.
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "https://") && insecure {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "https://") {
|
||||
transport.TLSClientConfig = tlsConfig
|
||||
}
|
||||
return transport
|
||||
}
|
||||
|
||||
func getHTTPClient(endpoint string, tlsConfig *tls.Config, timeout time.Duration, extraHeaders map[string]string) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: NewTransport(endpoint, tlsConfig),
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 { // a custom CheckRedirect replaces net/http's own limit
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
from, to := via[0].URL, req.URL
|
||||
if from.Hostname() != to.Hostname() || from.Scheme == "https" && to.Scheme != "https" {
|
||||
for name, value := range extraHeaders {
|
||||
if req.Header.Get(name) == value {
|
||||
req.Header.Del(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: timeout}
|
||||
}
|
||||
|
||||
// New returns a new runner client. timeout bounds every RPC: without it a
|
||||
// stalled connection parks the reporter for the whole job context, so logs and
|
||||
// heartbeats stop together and the task is reaped as a zombie.
|
||||
func New(endpoint string, insecure bool, uuid, token string, timeout time.Duration, opts ...connect.ClientOption) *HTTPClient {
|
||||
func New(endpoint string, tlsConfig *tls.Config, uuid, token string, timeout time.Duration, extraHeaders map[string]string, opts ...connect.ClientOption) *HTTPClient {
|
||||
baseURL := strings.TrimRight(endpoint, "/") + "/api/actions"
|
||||
|
||||
opts = append(opts, connect.WithInterceptors(connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc {
|
||||
return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
|
||||
for name, value := range extraHeaders {
|
||||
if req.Header().Get(name) == "" { // keep protocol headers such as Content-Type
|
||||
req.Header().Set(name, value)
|
||||
}
|
||||
}
|
||||
req.Header().Set("User-Agent", "gitea-runner/"+ver.Version())
|
||||
if uuid != "" {
|
||||
req.Header().Set(protocol.UUIDHeader, uuid)
|
||||
@@ -52,7 +78,7 @@ func New(endpoint string, insecure bool, uuid, token string, timeout time.Durati
|
||||
}
|
||||
})))
|
||||
|
||||
httpClient := getHTTPClient(endpoint, insecure, timeout)
|
||||
httpClient := getHTTPClient(endpoint, tlsConfig, timeout, extraHeaders)
|
||||
return &HTTPClient{
|
||||
PingServiceClient: pingv1connect.NewPingServiceClient(
|
||||
httpClient,
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -19,7 +21,7 @@ import (
|
||||
func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
|
||||
t.Setenv("HTTP_PROXY", "http://proxy.example.com:8080")
|
||||
|
||||
client := getHTTPClient("http://gitea.example.com", false, time.Minute)
|
||||
client := getHTTPClient("http://gitea.example.com", nil, time.Minute, nil)
|
||||
require.Equal(t, time.Minute, client.Timeout)
|
||||
transport, ok := client.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
@@ -33,29 +35,10 @@ func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
|
||||
require.Equal(t, "http://proxy.example.com:8080", proxyURL.String())
|
||||
}
|
||||
|
||||
func TestGetHTTPClientInsecureTLS(t *testing.T) {
|
||||
// insecure only takes effect for https endpoints
|
||||
httpsInsecure := getHTTPClient("https://gitea.example.com", true, time.Minute)
|
||||
transport, ok := httpsInsecure.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, transport.TLSClientConfig)
|
||||
require.True(t, transport.TLSClientConfig.InsecureSkipVerify)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
endpoint string
|
||||
insecure bool
|
||||
}{
|
||||
{"https secure", "https://gitea.example.com", false},
|
||||
{"http insecure ignored", "http://gitea.example.com", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := getHTTPClient(tc.endpoint, tc.insecure, time.Minute)
|
||||
tr, ok := c.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.Nil(t, tr.TLSClientConfig)
|
||||
})
|
||||
}
|
||||
func TestGetHTTPClientAppliesTLSConfigOnlyToHTTPS(t *testing.T) {
|
||||
tlsConfig := &tls.Config{}
|
||||
require.Same(t, tlsConfig, getHTTPClient("https://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
|
||||
require.Nil(t, getHTTPClient("http://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
|
||||
}
|
||||
|
||||
func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
@@ -69,7 +52,12 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
// trailing slash must be trimmed before "/api/actions" is appended
|
||||
c := New(server.URL+"/", false, "the-uuid", "the-token", time.Minute)
|
||||
c := New(server.URL+"/", nil, "the-uuid", "the-token", time.Minute, map[string]string{
|
||||
"X-Proxy-Token": "proxy-token",
|
||||
protocol.UUIDHeader: "other-uuid",
|
||||
protocol.TokenHeader: "other-token",
|
||||
"Content-Type": "text/plain",
|
||||
})
|
||||
// Address returns the endpoint as supplied (untrimmed)
|
||||
require.Equal(t, server.URL+"/", c.Address())
|
||||
|
||||
@@ -79,6 +67,8 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
require.True(t, strings.HasPrefix(gotPath, "/api/actions/"), "unexpected path %q", gotPath)
|
||||
require.Equal(t, "the-uuid", gotHeaders.Get(protocol.UUIDHeader))
|
||||
require.Equal(t, "the-token", gotHeaders.Get(protocol.TokenHeader))
|
||||
require.Equal(t, "proxy-token", gotHeaders.Get("X-Proxy-Token"))
|
||||
require.Equal(t, "application/proto", gotHeaders.Get("Content-Type"))
|
||||
}
|
||||
|
||||
func TestNewOmitsEmptyHeaders(t *testing.T) {
|
||||
@@ -89,9 +79,52 @@ func TestNewOmitsEmptyHeaders(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := New(server.URL, false, "", "", time.Minute)
|
||||
c := New(server.URL, nil, "", "", time.Minute, nil)
|
||||
_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
|
||||
require.Empty(t, gotHeaders.Get(protocol.UUIDHeader))
|
||||
require.Empty(t, gotHeaders.Get(protocol.TokenHeader))
|
||||
}
|
||||
|
||||
func TestNewStopsRedirectLoop(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.Path, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := New(server.URL, nil, "", "", time.Minute, nil)
|
||||
_, err := c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
require.ErrorContains(t, err, "stopped after 10 redirects")
|
||||
}
|
||||
|
||||
func TestNewRedirectKeepsExtraHeadersOnlyOnSameHost(t *testing.T) {
|
||||
var gotHeaders http.Header
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotHeaders = r.Header.Clone()
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer target.Close()
|
||||
_, port, err := net.SplitHostPort(target.Listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tc := range []struct {
|
||||
newOrigin func(http.Handler) *httptest.Server
|
||||
host string
|
||||
kept bool
|
||||
}{
|
||||
{httptest.NewServer, "127.0.0.1", true},
|
||||
{httptest.NewServer, "localhost", false},
|
||||
{httptest.NewTLSServer, "127.0.0.1", false},
|
||||
} {
|
||||
gotHeaders = nil
|
||||
origin := tc.newOrigin(http.RedirectHandler("http://"+net.JoinHostPort(tc.host, port), http.StatusPermanentRedirect))
|
||||
c := New(origin.URL, &tls.Config{InsecureSkipVerify: true}, "the-uuid", "the-token", time.Minute, map[string]string{"X-Proxy-Token": "proxy-token", "Content-Type": "text/plain"})
|
||||
_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
origin.Close()
|
||||
|
||||
require.NotNil(t, gotHeaders, origin.URL)
|
||||
require.Equal(t, tc.kept, gotHeaders.Get("X-Proxy-Token") != "", "%s to %s", origin.URL, tc.host)
|
||||
require.Equal(t, "the-token", gotHeaders.Get(protocol.TokenHeader))
|
||||
require.Equal(t, "application/proto", gotHeaders.Get("Content-Type"))
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
# Every option with its default value, all commented out. Read this file, do not copy it.
|
||||
# `./gitea-runner config init` writes a config file to copy the lines you change into.
|
||||
# Values may read the runner's environment as ${NAME}, an unset NAME fails the load. $${NAME} is a literal ${NAME}.
|
||||
|
||||
# Logging for the runner process itself (messages printed to stderr), plus the copy of
|
||||
# each task's log kept under log.job. Neither controls how workflow step output is streamed
|
||||
@@ -40,6 +41,9 @@ runner:
|
||||
#shutdown_timeout: 0s
|
||||
# Whether skip verifying the TLS certificate of the Gitea instance.
|
||||
#insecure: false
|
||||
# PEM client certificate and key for a Gitea instance that requires mutual TLS.
|
||||
#client_cert_file: /etc/gitea-runner/client.crt
|
||||
#client_key_file: /etc/gitea-runner/client.key
|
||||
# The timeout for fetching the job from the Gitea instance.
|
||||
# Values above the 60s RPC timeout are capped to it.
|
||||
#fetch_timeout: 5s
|
||||
@@ -83,6 +87,11 @@ runner:
|
||||
# and github_mirror is not empty. In this case,
|
||||
# it replaces https://github.com with the value here, which is useful for some special network environments.
|
||||
#github_mirror: ''
|
||||
# Extra HTTP headers for runner API requests to Gitea, e.g. reverse proxy authentication.
|
||||
# Not used for action clones, artifacts, job summaries or jobs. Runner headers take precedence.
|
||||
# Dropped on redirects to another hostname, or to HTTP if the instance URL uses HTTPS.
|
||||
#extra_headers:
|
||||
# X-Proxy-Token: 'example'
|
||||
# When true (the default), fetch only the requested ref of an action repository (e.g. actions/checkout@v4) at depth 1 instead of cloning every branch's full history.
|
||||
# Set to false to clone the full history.
|
||||
#action_shallow_clone: true
|
||||
@@ -217,6 +226,7 @@ container:
|
||||
# created by the runner. Omit the entire block to use Docker's defaults. An auto-created
|
||||
# network is labelled com.gitea.runner.uuid=<this runner's uuid>, which is how the idle
|
||||
# cleanup tells its own leftovers apart from those of other runners on the same daemon.
|
||||
# It also takes the `mtu` set in the Docker daemon config.
|
||||
#network_create_options:
|
||||
# enable_ipv4: true # Omit to use Docker's default (IPv4 enabled). Set false to disable IPv4.
|
||||
# enable_ipv6: false # Omit to use Docker's default (IPv6 disabled). Enabling it requires dockerd started with --ipv6.
|
||||
@@ -237,14 +247,15 @@ container:
|
||||
# If it's empty, /workspace will be used.
|
||||
# Purely numeric subdirectories under this path are reserved for task workspaces and may be removed by idle cleanup.
|
||||
#workdir_parent:
|
||||
# Volumes (including bind mounts) can be mounted to containers. Glob syntax is supported, see https://github.com/bmatcuk/doublestar#patterns
|
||||
# `*` matches within a single path segment, `**` also matches across path separators.
|
||||
# You can specify multiple volumes. If the sequence is empty, no volumes can be mounted.
|
||||
# For example, if you only allow containers to mount the `data` volume and all the json files in `/src`, you should change the config to:
|
||||
# Volumes and bind mounts containers may mount, as globs (https://github.com/bmatcuk/doublestar#patterns).
|
||||
# `*` matches within a path segment, `**` across segments. Empty allows none. tmpfs mounts are always allowed.
|
||||
# A `:ro` suffix allows only read-only mounts, unless an entry without it matches too.
|
||||
# For example, to allow the `data` volume and all json files in `/src`, and `/etc/ssl/certs` only read-only:
|
||||
# valid_volumes:
|
||||
# - data
|
||||
# - /src/*.json
|
||||
# If you want to allow any volume, please use the following configuration:
|
||||
# - /etc/ssl/certs:ro
|
||||
# To allow any volume:
|
||||
# valid_volumes:
|
||||
# - '**'
|
||||
#valid_volumes: []
|
||||
@@ -274,6 +285,9 @@ container:
|
||||
# How long a job waits for a service container that declares a healthcheck to become
|
||||
# healthy. A negative value (e.g. -1s) starts the steps without waiting.
|
||||
#service_ready_timeout: 5m
|
||||
# Remove the containers, networks and volumes a job creates through its Docker proxy when
|
||||
# the job ends. Set to false when jobs deploy services to the runner's Docker daemon.
|
||||
#sweep: true
|
||||
|
||||
host:
|
||||
# The parent directory of a job's working directory.
|
||||
|
||||
+101
-19
@@ -4,12 +4,13 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.yaml.in/yaml/v4"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
)
|
||||
|
||||
// RequestTimeout bounds every RPC to Gitea, and with it runner.fetch_timeout.
|
||||
@@ -32,6 +34,8 @@ const DefaultImage = "docker.gitea.com/runner-images:ubuntu-latest"
|
||||
// (so a programmatically built config still gets a sane bound).
|
||||
const DefaultPostTaskScriptTimeout = 5 * time.Minute
|
||||
|
||||
var environmentReference = regexp.MustCompile(`\$?\$\{[A-Za-z_][A-Za-z0-9_]*\}`)
|
||||
|
||||
// Minimal is the smallest config file that runs the runner: options it does not
|
||||
// name keep their default, and it names none.
|
||||
const Minimal = `# Minimal config file. Every option it does not set keeps its default.
|
||||
@@ -60,6 +64,9 @@ type Runner struct {
|
||||
Timeout time.Duration `yaml:"timeout"` // Timeout specifies the duration for runner timeout.
|
||||
ShutdownTimeout time.Duration `yaml:"shutdown_timeout"` // ShutdownTimeout specifies the duration to wait for running jobs to complete during a shutdown of the runner.
|
||||
Insecure bool `yaml:"insecure"` // Insecure indicates whether the runner operates in an insecure mode.
|
||||
ClientCertFile string `yaml:"client_cert_file"` // ClientCertFile is the PEM certificate presented to a Gitea instance that requires mutual TLS.
|
||||
ClientKeyFile string `yaml:"client_key_file"` // ClientKeyFile is the PEM private key paired with ClientCertFile.
|
||||
ExtraHeaders map[string]string `yaml:"extra_headers"` // ExtraHeaders adds HTTP headers to runner API requests to Gitea.
|
||||
FetchTimeout time.Duration `yaml:"fetch_timeout"` // FetchTimeout specifies the timeout duration for fetching resources.
|
||||
FetchInterval time.Duration `yaml:"fetch_interval"` // FetchInterval specifies the interval duration for fetching resources.
|
||||
FetchIntervalMax time.Duration `yaml:"fetch_interval_max"` // FetchIntervalMax specifies the maximum backoff interval when idle.
|
||||
@@ -174,6 +181,7 @@ type Container struct {
|
||||
DockerTimeout time.Duration `yaml:"docker_timeout"` // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner
|
||||
BindWorkdir bool `yaml:"bind_workdir"` // BindWorkdir mounts the workspace from a host directory instead of a Docker volume.
|
||||
ServiceReadyTimeout time.Duration `yaml:"service_ready_timeout"` // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting.
|
||||
Sweep *bool `yaml:"sweep"` // Sweep removes the containers, networks and volumes a job creates through its Docker proxy when the job ends. Unset means enabled.
|
||||
}
|
||||
|
||||
// Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses.
|
||||
@@ -242,14 +250,18 @@ func LoadDefault(file string) (*Config, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open config file %q: %w", file, err)
|
||||
}
|
||||
if err := yaml.Unmarshal(content, cfg); err != nil {
|
||||
var document yaml.Node
|
||||
if err := yaml.Unmarshal(content, &document); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
warnUnknownKeys(file, content)
|
||||
definedRunnerKeys, err = definedRunnerConfigKeys(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
|
||||
if err := expandEnvironment(&document); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
if err := document.Load(cfg, yaml.WithV3Defaults()); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
warnUnknownKeys(file, &document)
|
||||
definedRunnerKeys = definedRunnerConfigKeys(&document)
|
||||
}
|
||||
|
||||
if cfg.Runner.EnvFile != "" {
|
||||
@@ -264,6 +276,21 @@ func LoadDefault(file string) (*Config, error) {
|
||||
maps.Copy(cfg.Runner.Envs, envs)
|
||||
}
|
||||
}
|
||||
if (cfg.Runner.ClientCertFile == "") != (cfg.Runner.ClientKeyFile == "") {
|
||||
return nil, errors.New("runner.client_cert_file and runner.client_key_file must be set together")
|
||||
}
|
||||
if cfg.Runner.ClientCertFile != "" {
|
||||
if _, err := tls.LoadX509KeyPair(cfg.Runner.ClientCertFile, cfg.Runner.ClientKeyFile); err != nil {
|
||||
return nil, fmt.Errorf("load runner client certificate: %w", err)
|
||||
}
|
||||
for _, file := range []*string{&cfg.Runner.ClientCertFile, &cfg.Runner.ClientKeyFile} {
|
||||
abs, err := filepath.Abs(*file) // git resolves a relative path against the repository it runs in
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
*file = abs
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Log.Level == "" {
|
||||
cfg.Log.Level = "info"
|
||||
@@ -320,6 +347,9 @@ func LoadDefault(file string) (*Config, error) {
|
||||
if !slices.Contains(ToolCacheModes, cfg.Runner.ToolCacheMode) {
|
||||
return nil, fmt.Errorf("invalid runner.tool_cache_mode %q: must be one of %q", cfg.Runner.ToolCacheMode, ToolCacheModes)
|
||||
}
|
||||
if err := validateExtraHeaders(cfg.Runner.ExtraHeaders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Host.WorkdirParent == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
@@ -411,30 +441,67 @@ func LoadDefault(file string) (*Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (r *Runner) TLSConfig() *tls.Config {
|
||||
tlsConfig := &tls.Config{InsecureSkipVerify: r.Insecure}
|
||||
if r.ClientCertFile != "" {
|
||||
tlsConfig.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
pair, err := tls.LoadX509KeyPair(r.ClientCertFile, r.ClientKeyFile) // per handshake, so renewed files apply
|
||||
if err != nil || info.SupportsCertificate(&pair) != nil {
|
||||
return &tls.Certificate{}, err
|
||||
}
|
||||
return &pair, nil
|
||||
}
|
||||
}
|
||||
return tlsConfig
|
||||
}
|
||||
|
||||
// expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset.
|
||||
func expandEnvironment(node *yaml.Node) error {
|
||||
var errs []error
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
expanded := environmentReference.ReplaceAllStringFunc(node.Value, func(match string) string {
|
||||
if strings.HasPrefix(match, "$$") {
|
||||
return match[1:]
|
||||
}
|
||||
value, ok := os.LookupEnv(strings.Trim(match, "${}"))
|
||||
if !ok {
|
||||
errs = append(errs, fmt.Errorf("line %d: %s is not set", node.Line, match))
|
||||
}
|
||||
return value
|
||||
})
|
||||
if expanded != node.Value && node.Style&yaml.TaggedStyle == 0 {
|
||||
node.Tag = ""
|
||||
if expanded == "" && node.Style == 0 {
|
||||
node.Tag = "!!null"
|
||||
}
|
||||
}
|
||||
node.Value = expanded
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
for index, child := range node.Content {
|
||||
if node.Kind != yaml.MappingNode || index%2 == 1 {
|
||||
errs = append(errs, expandEnvironment(child))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// warnUnknownKeys reports keys the config does not define, which are otherwise ignored
|
||||
// without a trace. It only warns, so a config carrying keys from another runner version
|
||||
// still loads.
|
||||
func warnUnknownKeys(file string, content []byte) {
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
decoder.KnownFields(true)
|
||||
|
||||
func warnUnknownKeys(file string, document *yaml.Node) {
|
||||
var loadErrs *yaml.LoadErrors
|
||||
if err := decoder.Decode(&Config{}); errors.As(err, &loadErrs) {
|
||||
if err := document.Load(&Config{}, yaml.WithV3Defaults(), yaml.WithKnownFields()); errors.As(err, &loadErrs) {
|
||||
for _, loadErr := range loadErrs.Errors {
|
||||
log.Warnf("config file %q: %s, it will be ignored", file, loadErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal(content, &root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func definedRunnerConfigKeys(root *yaml.Node) map[string]bool {
|
||||
defined := map[string]bool{}
|
||||
if len(root.Content) == 0 {
|
||||
return defined, nil
|
||||
return defined
|
||||
}
|
||||
|
||||
doc := root.Content[0]
|
||||
@@ -450,7 +517,22 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
||||
break
|
||||
}
|
||||
|
||||
return defined, nil
|
||||
return defined
|
||||
}
|
||||
|
||||
func validateExtraHeaders(headers map[string]string) error {
|
||||
seen := make(map[string]string, len(headers))
|
||||
for name, value := range headers {
|
||||
if !httpguts.ValidHeaderFieldName(name) || !httpguts.ValidHeaderFieldValue(value) {
|
||||
return fmt.Errorf("invalid runner.extra_headers entry %q", name)
|
||||
}
|
||||
key := strings.ToLower(name)
|
||||
if other, ok := seen[key]; ok {
|
||||
return fmt.Errorf("runner.extra_headers sets %q and %q, which name the same header", other, name)
|
||||
}
|
||||
seen[key] = name
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveSecretFile reads key from key_file, so deployments can mount a secret instead of committing it.
|
||||
|
||||
@@ -4,6 +4,13 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -42,6 +49,81 @@ cache:
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClientCertIsValidatedAndAbsoluteOnLoadReadOnUseAndOfferedOnlyWhenAccepted(t *testing.T) {
|
||||
require.True(t, (&Runner{Insecure: true}).TLSConfig().InsecureSkipVerify)
|
||||
|
||||
dir := t.TempDir()
|
||||
certPath, keyPath := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
writePair := func() []byte {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(1)}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
return der
|
||||
}
|
||||
load := func(certFile, keyFile string) (*Config, error) {
|
||||
return LoadDefault(write(t, "runner:\n client_cert_file: '"+certFile+"'\n client_key_file: '"+keyFile+"'\n"))
|
||||
}
|
||||
writePair()
|
||||
|
||||
_, err := load(certPath, "")
|
||||
require.ErrorContains(t, err, "must be set together")
|
||||
_, err = load(keyPath, keyPath)
|
||||
require.ErrorContains(t, err, "load runner client certificate")
|
||||
|
||||
t.Chdir(dir)
|
||||
cfg, err := load("client.crt", "client.key")
|
||||
require.NoError(t, err)
|
||||
require.True(t, filepath.IsAbs(cfg.Runner.ClientCertFile))
|
||||
require.True(t, filepath.IsAbs(cfg.Runner.ClientKeyFile))
|
||||
renewed := writePair()
|
||||
info := &tls.CertificateRequestInfo{Version: tls.VersionTLS13, SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}}
|
||||
pair, err := cfg.Runner.TLSConfig().GetClientCertificate(info)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, renewed, pair.Certificate[0])
|
||||
info.AcceptableCAs = [][]byte{[]byte("another CA")}
|
||||
pair, err = cfg.Runner.TLSConfig().GetClientCertificate(info)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, pair.Certificate)
|
||||
}
|
||||
|
||||
func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) {
|
||||
t.Setenv("RUNNER_CAPACITY", "4")
|
||||
t.Setenv("RUNNER_LABEL", "linux:host")
|
||||
t.Setenv("RUNNER_KEY", "fetch_timeout")
|
||||
t.Setenv("RUNNER_EMPTY", "")
|
||||
t.Setenv("RUNNER_MISSING", "")
|
||||
require.NoError(t, os.Unsetenv("RUNNER_MISSING"))
|
||||
path := write(t, `runner:
|
||||
capacity: ${RUNNER_CAPACITY}
|
||||
labels:
|
||||
- ${RUNNER_LABEL}
|
||||
envs:
|
||||
LITERAL: $${RUNNER_LABEL}
|
||||
${RUNNER_KEY}: 1s
|
||||
cache:
|
||||
enabled: ${RUNNER_EMPTY}
|
||||
`)
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 4, cfg.Runner.Capacity)
|
||||
assert.Equal(t, []string{"linux:host"}, cfg.Runner.Labels)
|
||||
assert.Equal(t, map[string]string{"LITERAL": "${RUNNER_LABEL}"}, cfg.Runner.Envs)
|
||||
assert.Equal(t, 5*time.Second, cfg.Runner.FetchTimeout)
|
||||
assert.True(t, *cfg.Cache.Enabled)
|
||||
|
||||
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_MISSING}\n file: ${RUNNER_MISSING}\n"))
|
||||
require.ErrorContains(t, err, "line 3: ${RUNNER_MISSING} is not set\nline 4: ${RUNNER_MISSING} is not set")
|
||||
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_LABEL}\n"))
|
||||
require.ErrorContains(t, err, "line 3:")
|
||||
}
|
||||
|
||||
func TestLoadDefault_CacheS3(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
accessPath := filepath.Join(dir, "access.key")
|
||||
@@ -89,6 +171,25 @@ func TestLoadDefault_ToolCacheMode(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "tool_cache_mode")
|
||||
}
|
||||
|
||||
func TestLoadDefault_LoadsExtraHeaders(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte("runner:\n extra_headers:\n X-Proxy-Token: proxy-token\n"), 0o600))
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]string{"X-Proxy-Token": "proxy-token"}, cfg.Runner.ExtraHeaders)
|
||||
|
||||
for _, headers := range []string{
|
||||
"X-Proxy Token: proxy-token\n",
|
||||
"X-Proxy-Token: |\n proxy-token\n",
|
||||
"X-Proxy-Token: a\n x-proxy-token: b\n",
|
||||
} {
|
||||
require.NoError(t, os.WriteFile(path, []byte("runner:\n extra_headers:\n "+headers), 0o600))
|
||||
_, err = LoadDefault(path)
|
||||
assert.ErrorContains(t, err, "runner.extra_headers", headers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) {
|
||||
cfg, err := LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +22,7 @@ const Namespace = "gitea_runner"
|
||||
//
|
||||
// LabelResult* values are used on metrics with label key "result" (RPC outcomes).
|
||||
// LabelStatus* values are used on metrics with label key "status" (job outcomes).
|
||||
// LabelState* values are used on metrics with label key "state" (runner state).
|
||||
const (
|
||||
LabelResultTask = "task"
|
||||
LabelResultEmpty = "empty"
|
||||
@@ -35,6 +38,11 @@ const (
|
||||
LabelStatusCancelled = "cancelled"
|
||||
LabelStatusSkipped = "skipped"
|
||||
LabelStatusUnknown = "unknown"
|
||||
LabelStatusTimeout = "timeout"
|
||||
|
||||
LabelStateBusy = "busy"
|
||||
LabelStateIdle = "idle"
|
||||
LabelStateUnavailable = "unavailable"
|
||||
)
|
||||
|
||||
// rpcDurationBuckets covers the expected latency range for short-running
|
||||
@@ -57,6 +65,15 @@ func ResultToStatusLabel(r runnerv1.Result) string {
|
||||
}
|
||||
}
|
||||
|
||||
// JobStatus reports an unsuccessful job past its deadline as timeout, like the job's trace span.
|
||||
func JobStatus(ctx context.Context, r runnerv1.Result) string {
|
||||
status := ResultToStatusLabel(r)
|
||||
if status != LabelStatusSuccess && status != LabelStatusSkipped && errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return LabelStatusTimeout
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
var (
|
||||
RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
@@ -96,7 +113,7 @@ var (
|
||||
Namespace: Namespace,
|
||||
Subsystem: "job",
|
||||
Name: "total",
|
||||
Help: "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
|
||||
Help: "Total jobs processed by status (success, failure, cancelled, skipped, timeout, unknown).",
|
||||
}, []string{"status"})
|
||||
|
||||
JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
@@ -148,8 +165,8 @@ var (
|
||||
Namespace: Namespace,
|
||||
Subsystem: "client",
|
||||
Name: "errors_total",
|
||||
Help: "Total client RPC errors by method.",
|
||||
}, []string{"method"})
|
||||
Help: "Total client RPC errors by method and connect code.",
|
||||
}, []string{"method", "code"})
|
||||
)
|
||||
|
||||
// Registry is the custom Prometheus registry used by the runner.
|
||||
@@ -220,3 +237,28 @@ func register(collector prometheus.Collector) {
|
||||
Registry.Unregister(collector)
|
||||
Registry.MustRegister(collector)
|
||||
}
|
||||
|
||||
// RegisterStateFunc registers gitea_runner_state, 1 for the state stateFn reports and 0 for the others.
|
||||
func RegisterStateFunc(stateFn func() string) {
|
||||
register(stateCollector{prometheus.NewDesc(prometheus.BuildFQName(Namespace, "", "state"),
|
||||
"Runner state (busy, idle, unavailable). 1 for the current state.", []string{"state"}, nil), stateFn})
|
||||
}
|
||||
|
||||
// stateCollector reads the state once per collection, so a scrape never reports two states.
|
||||
type stateCollector struct {
|
||||
desc *prometheus.Desc
|
||||
state func() string
|
||||
}
|
||||
|
||||
func (c stateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
|
||||
|
||||
func (c stateCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
current := c.state()
|
||||
for _, state := range []string{LabelStateBusy, LabelStateIdle, LabelStateUnavailable} {
|
||||
value := 0.0
|
||||
if state == current {
|
||||
value = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, value, state)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,11 @@ func TestResultToStatusLabel(t *testing.T) {
|
||||
require.Equal(t, tt.want, ResultToStatusLabel(tt.result))
|
||||
})
|
||||
}
|
||||
expired, cancel := context.WithDeadline(t.Context(), time.Time{})
|
||||
defer cancel()
|
||||
require.Equal(t, LabelStatusTimeout, JobStatus(expired, runnerv1.Result_RESULT_FAILURE))
|
||||
require.Equal(t, LabelStatusSuccess, JobStatus(expired, runnerv1.Result_RESULT_SUCCESS))
|
||||
require.Equal(t, LabelStatusCancelled, JobStatus(t.Context(), runnerv1.Result_RESULT_CANCELLED))
|
||||
}
|
||||
|
||||
func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
@@ -50,6 +55,8 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
RunnerInfo.WithLabelValues("test", "runner").Set(1)
|
||||
RegisterUptimeFunc(time.Now().Add(-time.Second))
|
||||
RegisterRunningJobsFunc(func() int64 { return 2 }, 4)
|
||||
RegisterStateFunc(func() string { return LabelStateIdle })
|
||||
RegisterStateFunc(func() string { return LabelStateBusy })
|
||||
|
||||
metrics, err := Registry.Gather()
|
||||
require.NoError(t, err)
|
||||
@@ -58,6 +65,15 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_uptime_seconds"))
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_job_running"))
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_job_capacity_utilization_ratio"))
|
||||
states := map[string]float64{}
|
||||
for _, mf := range metrics {
|
||||
if mf.GetName() == "gitea_runner_state" {
|
||||
for _, m := range mf.GetMetric() {
|
||||
states[m.GetLabel()[0].GetValue()] = m.GetGauge().GetValue()
|
||||
}
|
||||
}
|
||||
}
|
||||
require.Equal(t, map[string]float64{LabelStateBusy: 1, LabelStateIdle: 0, LabelStateUnavailable: 0}, states)
|
||||
}
|
||||
|
||||
func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {
|
||||
|
||||
@@ -36,9 +36,8 @@ func processAlive(pid int) bool {
|
||||
if b, readErr := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); readErr == nil {
|
||||
// Format: "pid (comm) state ..." — state follows the closing ')' of the
|
||||
// command name (which may itself contain spaces and parens).
|
||||
rest := string(b)
|
||||
if idx := strings.LastIndex(rest, ") "); idx >= 0 {
|
||||
fields := strings.Fields(rest[idx+2:])
|
||||
if _, after, found := strings.CutLast(string(b), ") "); found {
|
||||
fields := strings.Fields(after)
|
||||
if len(fields) > 0 && fields[0] == "Z" {
|
||||
return false // zombie: terminated but not yet reaped
|
||||
}
|
||||
|
||||
@@ -624,7 +624,7 @@ func (r *Reporter) reportLog(noMore bool) (bool, error) {
|
||||
metrics.ReportLogDuration.Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog, connect.CodeOf(err).String()).Inc()
|
||||
return false, err
|
||||
}
|
||||
metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultSuccess).Inc()
|
||||
@@ -701,7 +701,7 @@ func (r *Reporter) ReportState(reportResult bool) error {
|
||||
metrics.ReportStateDuration.Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
metrics.ReportStateTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask, connect.CodeOf(err).String()).Inc()
|
||||
r.stateMu.Lock()
|
||||
r.stateChanged = true
|
||||
r.stateMu.Unlock()
|
||||
|
||||
@@ -7,12 +7,18 @@ import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/metrics"
|
||||
|
||||
"gitea.dev/actionslib/runner/v1/runnerv1connect"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
commonpb "go.opentelemetry.io/proto/otlp/common/v1"
|
||||
metricpb "go.opentelemetry.io/proto/otlp/metrics/v1"
|
||||
@@ -26,7 +32,11 @@ const cumulative = metricpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMUL
|
||||
func pushMetrics(exporter *client, res *resource.Resource, gatherer prometheus.Gatherer) func(context.Context) error {
|
||||
attrs := make([]*commonpb.KeyValue, 0, res.Len())
|
||||
for _, kv := range res.Attributes() {
|
||||
attrs = append(attrs, keyValue(string(kv.Key), kv.Value.String()))
|
||||
attr := keyValue(string(kv.Key), kv.Value.String())
|
||||
if kv.Value.Type() == attribute.INT64 {
|
||||
attr.Value.Value = &commonpb.AnyValue_IntValue{IntValue: kv.Value.AsInt64()}
|
||||
}
|
||||
attrs = append(attrs, attr)
|
||||
}
|
||||
interval, timeout := millis("OTEL_METRIC_EXPORT_INTERVAL", time.Minute, time.Minute), millis("OTEL_METRIC_EXPORT_TIMEOUT", 30*time.Second, noLimit)
|
||||
push := func(ctx context.Context) error {
|
||||
@@ -70,39 +80,132 @@ var ucum = map[string]string{
|
||||
"meters": "m", "volts": "V", "amperes": "A", "joules": "J", "watts": "W", "grams": "g", "celsius": "Cel", "hertz": "Hz", "percent": "%",
|
||||
}
|
||||
|
||||
// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp
|
||||
type otlpMetric struct {
|
||||
name, unit string
|
||||
upDown bool // a gauge semconv defines as an UpDownCounter
|
||||
labels map[string]string // label → attribute
|
||||
values map[string]string // label value → attribute value
|
||||
extra []*commonpb.KeyValue
|
||||
unlimited float64 // gauge value meaning no limit, not sent
|
||||
}
|
||||
|
||||
var rpcMethods = map[string]string{
|
||||
metrics.LabelMethodFetchTask: strings.TrimPrefix(runnerv1connect.RunnerServiceFetchTaskProcedure, "/"),
|
||||
metrics.LabelMethodUpdateLog: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateLogProcedure, "/"),
|
||||
metrics.LabelMethodUpdateTask: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateTaskProcedure, "/"),
|
||||
}
|
||||
|
||||
// openFiles maps process_open_fds, which client_golang fills with the handle count on Windows.
|
||||
func openFiles() otlpMetric {
|
||||
if runtime.GOOS == "windows" {
|
||||
return otlpMetric{name: "process.windows.handle.count", unit: "{handle}", upDown: true}
|
||||
}
|
||||
return otlpMetric{name: "process.unix.file_descriptor.count", unit: "{file_descriptor}", upDown: true}
|
||||
}
|
||||
|
||||
// otlpMetrics maps to semconv names, unlisted metrics keep theirs and an empty name drops one the resource already carries.
|
||||
var otlpMetrics = map[string]otlpMetric{
|
||||
"gitea_runner_info": {},
|
||||
"go_info": {},
|
||||
"process_start_time_seconds": {},
|
||||
"gitea_runner_capacity": {name: "gitea.runner.job.limit", unit: "{job}", upDown: true},
|
||||
"gitea_runner_uptime_seconds": {name: "gitea.runner.uptime", unit: "s"},
|
||||
"gitea_runner_job_running": {name: "gitea.runner.job.active", unit: "{job}", upDown: true},
|
||||
"gitea_runner_job_capacity_utilization_ratio": {name: "gitea.runner.job.utilization", unit: "1"},
|
||||
"gitea_runner_job_duration_seconds": {name: "gitea.runner.job.duration", unit: "s"},
|
||||
"gitea_runner_job_total": {
|
||||
name: "gitea.runner.jobs", unit: "{job}",
|
||||
labels: map[string]string{"status": "cicd.pipeline.result"},
|
||||
values: map[string]string{metrics.LabelStatusCancelled: "cancellation", metrics.LabelStatusSkipped: "skip", metrics.LabelStatusUnknown: "error"},
|
||||
},
|
||||
"gitea_runner_poll_fetch_total": {
|
||||
name: "gitea.runner.poll.fetches", unit: "{fetch}",
|
||||
labels: map[string]string{"result": "gitea.runner.poll.result"},
|
||||
},
|
||||
"gitea_runner_poll_fetch_duration_seconds": {name: "gitea.runner.poll.fetch.duration", unit: "s"},
|
||||
"gitea_runner_poll_backoff_seconds": {name: "gitea.runner.poll.backoff", unit: "s"},
|
||||
"gitea_runner_report_log_duration_seconds": {name: "gitea.runner.report.log.duration", unit: "s"},
|
||||
"gitea_runner_report_state_duration_seconds": {name: "gitea.runner.report.state.duration", unit: "s"},
|
||||
"gitea_runner_report_log_buffer_rows": {name: "gitea.runner.report.log.buffer", unit: "{row}", upDown: true},
|
||||
"gitea_runner_report_log_total": {
|
||||
name: "gitea.runner.report.log.requests", unit: "{request}",
|
||||
labels: map[string]string{"result": "gitea.runner.report.result"},
|
||||
},
|
||||
"gitea_runner_report_state_total": {
|
||||
name: "gitea.runner.report.state.requests", unit: "{request}",
|
||||
labels: map[string]string{"result": "gitea.runner.report.result"},
|
||||
},
|
||||
"gitea_runner_client_errors_total": {
|
||||
name: "cicd.system.errors", unit: "{error}",
|
||||
labels: map[string]string{"method": "rpc.method", "code": "error.type"}, values: rpcMethods,
|
||||
extra: []*commonpb.KeyValue{keyValue("cicd.system.component", "runner")},
|
||||
},
|
||||
"gitea_runner_state": {
|
||||
name: "cicd.worker.count", unit: "{worker}", upDown: true,
|
||||
labels: map[string]string{"state": "cicd.worker.state"},
|
||||
values: map[string]string{metrics.LabelStateIdle: "available", metrics.LabelStateUnavailable: "offline"},
|
||||
},
|
||||
"go_goroutines": {name: "go.goroutine.count", unit: "{goroutine}", upDown: true},
|
||||
"go_memstats_alloc_bytes_total": {name: "go.memory.allocated", unit: "By"},
|
||||
"go_memstats_next_gc_bytes": {name: "go.memory.gc.goal", unit: "By", upDown: true},
|
||||
"go_gc_gomemlimit_bytes": {name: "go.memory.limit", unit: "By", upDown: true, unlimited: math.MaxInt64},
|
||||
"go_gc_gogc_percent": {name: "go.config.gogc", unit: "%", upDown: true},
|
||||
"go_sched_gomaxprocs_threads": {name: "go.processor.limit", unit: "{thread}", upDown: true},
|
||||
"process_resident_memory_bytes": {name: "process.memory.usage", unit: "By", upDown: true},
|
||||
"process_virtual_memory_bytes": {name: "process.memory.virtual", unit: "By", upDown: true},
|
||||
"process_open_fds": openFiles(),
|
||||
}
|
||||
|
||||
// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp except for otlpMetrics.
|
||||
func convert(families []*dto.MetricFamily) []*metricpb.Metric {
|
||||
now := uint64(time.Now().UnixNano())
|
||||
metrics := make([]*metricpb.Metric, 0, len(families))
|
||||
byFamily := make(map[string]*dto.MetricFamily, len(families))
|
||||
for _, family := range families {
|
||||
byFamily[family.GetName()] = family
|
||||
mapping, ok := otlpMetrics[family.GetName()]
|
||||
if !ok {
|
||||
mapping.name = family.GetName()
|
||||
} else if mapping.name == "" {
|
||||
continue
|
||||
}
|
||||
metric := &metricpb.Metric{
|
||||
Name: family.GetName(), Description: family.GetHelp(), Unit: cmp.Or(ucum[family.GetUnit()], family.GetUnit()),
|
||||
Name: mapping.name, Description: family.GetHelp(), Unit: cmp.Or(mapping.unit, ucum[family.GetUnit()], family.GetUnit()),
|
||||
Metadata: []*commonpb.KeyValue{keyValue("prometheus.type", strings.ToLower(family.GetType().String()))},
|
||||
}
|
||||
switch family.GetType() {
|
||||
case dto.MetricType_COUNTER:
|
||||
sum := &metricpb.Sum{AggregationTemporality: cumulative, IsMonotonic: true}
|
||||
for _, series := range family.GetMetric() {
|
||||
sum.DataPoints = append(sum.DataPoints, number(series, unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
|
||||
sum.DataPoints = append(sum.DataPoints, number(labels(series, mapping), unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Sum{Sum: sum}
|
||||
case dto.MetricType_GAUGE:
|
||||
gauge := &metricpb.Gauge{}
|
||||
var points []*metricpb.NumberDataPoint
|
||||
for _, series := range family.GetMetric() {
|
||||
gauge.DataPoints = append(gauge.DataPoints, number(series, 0, now, series.GetGauge().GetValue()))
|
||||
if value := series.GetGauge().GetValue(); mapping.unlimited == 0 || value != mapping.unlimited {
|
||||
points = append(points, number(labels(series, mapping), 0, now, value))
|
||||
}
|
||||
}
|
||||
if len(points) == 0 {
|
||||
continue
|
||||
}
|
||||
if mapping.upDown {
|
||||
metric.Data = &metricpb.Metric_Sum{Sum: &metricpb.Sum{AggregationTemporality: cumulative, DataPoints: points}}
|
||||
} else {
|
||||
metric.Data = &metricpb.Metric_Gauge{Gauge: &metricpb.Gauge{DataPoints: points}}
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Gauge{Gauge: gauge}
|
||||
case dto.MetricType_HISTOGRAM:
|
||||
histogram := &metricpb.Histogram{AggregationTemporality: cumulative}
|
||||
for _, series := range family.GetMetric() {
|
||||
histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, now))
|
||||
histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, labels(series, mapping), now))
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Histogram{Histogram: histogram}
|
||||
case dto.MetricType_SUMMARY:
|
||||
summary := &metricpb.Summary{}
|
||||
for _, series := range family.GetMetric() {
|
||||
point := &metricpb.SummaryDataPoint{
|
||||
Attributes: labels(series), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Attributes: labels(series, mapping), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Count: series.GetSummary().GetSampleCount(), Sum: series.GetSummary().GetSampleSum(),
|
||||
}
|
||||
for _, quantile := range series.GetSummary().GetQuantile() {
|
||||
@@ -116,11 +219,44 @@ func convert(families []*dto.MetricFamily) []*metricpb.Metric {
|
||||
}
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
return append(metrics, derived(byFamily, now)...)
|
||||
}
|
||||
|
||||
// derived adds the semconv Go metrics that are computed from other metrics rather than renamed.
|
||||
func derived(families map[string]*dto.MetricFamily, now uint64) []*metricpb.Metric {
|
||||
first := func(name string) *dto.Metric {
|
||||
if series := families[name].GetMetric(); len(series) > 0 {
|
||||
return series[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
gauge := func(name string) (float64, bool) {
|
||||
series := first(name)
|
||||
return series.GetGauge().GetValue(), series != nil
|
||||
}
|
||||
sum := func(name, unit, description string, monotonic bool, points ...*metricpb.NumberDataPoint) *metricpb.Metric {
|
||||
return &metricpb.Metric{Name: name, Unit: unit, Description: description, Data: &metricpb.Metric_Sum{Sum: &metricpb.Sum{
|
||||
AggregationTemporality: cumulative, IsMonotonic: monotonic, DataPoints: points,
|
||||
}}}
|
||||
}
|
||||
var metrics []*metricpb.Metric
|
||||
total, ok1 := gauge("go_memstats_sys_bytes")
|
||||
released, ok2 := gauge("go_memstats_heap_released_bytes")
|
||||
stack, ok3 := gauge("go_memstats_stack_inuse_bytes")
|
||||
if ok1 && ok2 && ok3 {
|
||||
metrics = append(metrics, sum("go.memory.used", "By", "Memory used by the Go runtime.", false,
|
||||
number([]*commonpb.KeyValue{keyValue("go.memory.type", "stack")}, 0, now, stack),
|
||||
number([]*commonpb.KeyValue{keyValue("go.memory.type", "other")}, 0, now, total-released-stack)))
|
||||
}
|
||||
if gc := first("go_gc_duration_seconds"); gc != nil {
|
||||
metrics = append(metrics, sum("go.memory.gc.cycles", "{gc_cycle}", "Number of completed GC cycles.", true,
|
||||
number(nil, unixNano(gc.GetSummary().GetCreatedTimestamp()), now, float64(gc.GetSummary().GetSampleCount()))))
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func number(series *dto.Metric, start, now uint64, value float64) *metricpb.NumberDataPoint {
|
||||
return &metricpb.NumberDataPoint{Attributes: labels(series), StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
|
||||
func number(attrs []*commonpb.KeyValue, start, now uint64, value float64) *metricpb.NumberDataPoint {
|
||||
return &metricpb.NumberDataPoint{Attributes: attrs, StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
|
||||
}
|
||||
|
||||
// unixNano maps a nil timestamp to 0, leaving the start time unset as the spec asks.
|
||||
@@ -128,10 +264,10 @@ func unixNano(timestamp *timestamppb.Timestamp) uint64 {
|
||||
return uint64(timestamp.AsTime().UnixNano())
|
||||
}
|
||||
|
||||
func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint {
|
||||
func histogramPoint(series *dto.Metric, attrs []*commonpb.KeyValue, now uint64) *metricpb.HistogramDataPoint {
|
||||
histogram := series.GetHistogram()
|
||||
point := &metricpb.HistogramDataPoint{
|
||||
Attributes: labels(series), StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Attributes: attrs, StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Count: histogram.GetSampleCount(), Sum: histogram.SampleSum,
|
||||
BucketCounts: make([]uint64, len(histogram.GetBucket())+1),
|
||||
}
|
||||
@@ -144,12 +280,12 @@ func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint
|
||||
return point
|
||||
}
|
||||
|
||||
func labels(series *dto.Metric) []*commonpb.KeyValue {
|
||||
attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel()))
|
||||
func labels(series *dto.Metric, mapping otlpMetric) []*commonpb.KeyValue {
|
||||
attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel())+len(mapping.extra))
|
||||
for _, label := range series.GetLabel() {
|
||||
attrs = append(attrs, keyValue(label.GetName(), label.GetValue()))
|
||||
attrs = append(attrs, keyValue(cmp.Or(mapping.labels[label.GetName()], label.GetName()), cmp.Or(mapping.values[label.GetValue()], label.GetValue())))
|
||||
}
|
||||
return attrs
|
||||
return append(attrs, mapping.extra...)
|
||||
}
|
||||
|
||||
func keyValue(key, value string) *commonpb.KeyValue {
|
||||
|
||||
@@ -45,6 +45,8 @@ import (
|
||||
|
||||
const scope = "gitea.com/gitea/runner"
|
||||
|
||||
var processCreation = time.Now() // package init, within milliseconds of the process start
|
||||
|
||||
type taskRunKey struct{}
|
||||
|
||||
type taskRun struct {
|
||||
@@ -102,12 +104,13 @@ func Setup(ctx context.Context, uuid, name string, gatherer prometheus.Gatherer)
|
||||
if len(clients) == 0 {
|
||||
return noop, nil
|
||||
}
|
||||
res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
|
||||
res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithProcessPID(), resource.WithProcessRuntimeName(), resource.WithProcessRuntimeVersion(), resource.WithAttributes(
|
||||
semconv.ServiceName("gitea-runner"),
|
||||
semconv.ServiceVersion(ver.Version()),
|
||||
semconv.ServiceInstanceID(uuid),
|
||||
semconv.CICDWorkerID(uuid),
|
||||
semconv.CICDWorkerName(name),
|
||||
semconv.ProcessCreationTime(processCreation.UTC().Format(time.RFC3339Nano)),
|
||||
), resource.WithFromEnv())
|
||||
if err != nil {
|
||||
return noop, err
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -154,9 +156,10 @@ func TestExportRetriesAndMatchesOTLP(t *testing.T) {
|
||||
assert.Equal(t, "op", tracesData.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
|
||||
|
||||
resourceMetrics := metricsData.GetResourceMetrics()[0]
|
||||
assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool {
|
||||
return proto.Equal(kv, keyValue("service.name", "gitea-runner"))
|
||||
}))
|
||||
pid := &commonpb.KeyValue{Key: "process.pid", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: int64(os.Getpid())}}}
|
||||
for _, want := range []*commonpb.KeyValue{keyValue("service.name", "gitea-runner"), keyValue("process.creation.time", processCreation.UTC().Format(time.RFC3339Nano)), pid} {
|
||||
assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool { return proto.Equal(kv, want) }), want.GetKey())
|
||||
}
|
||||
assert.InDelta(t, 2, resourceMetrics.GetScopeMetrics()[0].GetMetrics()[0].GetGauge().GetDataPoints()[0].GetAsDouble(), 0)
|
||||
}
|
||||
|
||||
@@ -165,31 +168,76 @@ func TestConvert(t *testing.T) {
|
||||
counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "jobs_total", Help: "Jobs."}, []string{"status"})
|
||||
histogram := prometheus.NewHistogram(prometheus.HistogramOpts{Name: "duration_seconds", Unit: "seconds", Buckets: []float64{1, 2}})
|
||||
summary := prometheus.NewSummary(prometheus.SummaryOpts{Name: "gc_seconds", Objectives: map[float64]float64{0.5: 0.05}})
|
||||
registry.MustRegister(counter, histogram, summary)
|
||||
jobs := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_job_total"}, []string{"status"})
|
||||
running := prometheus.NewGauge(prometheus.GaugeOpts{Name: "gitea_runner_job_running"})
|
||||
gauges := map[string]float64{
|
||||
"go_info": 1, "go_gc_gomemlimit_bytes": math.MaxInt64,
|
||||
"go_memstats_sys_bytes": 100, "go_memstats_heap_released_bytes": 30, "go_memstats_stack_inuse_bytes": 20,
|
||||
}
|
||||
for name, value := range gauges {
|
||||
registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: name}, func() float64 { return value }))
|
||||
}
|
||||
state := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "gitea_runner_state"}, []string{"state"})
|
||||
clientErrors := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_client_errors_total"}, []string{"method", "code"})
|
||||
registry.MustRegister(counter, histogram, summary, jobs, running, state, clientErrors)
|
||||
counter.WithLabelValues("success").Add(3)
|
||||
for _, value := range []float64{0.5, 1.5, 3} {
|
||||
histogram.Observe(value)
|
||||
}
|
||||
summary.Observe(4)
|
||||
jobs.WithLabelValues("cancelled").Inc()
|
||||
running.Set(2)
|
||||
state.WithLabelValues("idle").Set(1)
|
||||
state.WithLabelValues("unavailable").Set(0)
|
||||
clientErrors.WithLabelValues("FetchTask", "unavailable").Inc()
|
||||
families, err := registry.Gather()
|
||||
require.NoError(t, err)
|
||||
|
||||
exported := convert(families)
|
||||
assert.Equal(t, "s", exported[0].GetUnit())
|
||||
buckets := exported[0].GetHistogram().GetDataPoints()[0]
|
||||
exported := map[string]*metricpb.Metric{}
|
||||
for _, metric := range convert(families) {
|
||||
exported[metric.GetName()] = metric
|
||||
}
|
||||
assert.ElementsMatch(t, []string{
|
||||
"duration_seconds", "gc_seconds", "jobs_total", "gitea.runner.jobs", "gitea.runner.job.active", "cicd.system.errors",
|
||||
"go_memstats_sys_bytes", "go_memstats_heap_released_bytes", "go_memstats_stack_inuse_bytes", "go.memory.used", "cicd.worker.count",
|
||||
}, slices.Collect(maps.Keys(exported)))
|
||||
assert.Equal(t, "s", exported["duration_seconds"].GetUnit())
|
||||
buckets := exported["duration_seconds"].GetHistogram().GetDataPoints()[0]
|
||||
assert.Equal(t, []float64{1, 2}, buckets.GetExplicitBounds())
|
||||
assert.Equal(t, []uint64{1, 1, 1}, buckets.GetBucketCounts())
|
||||
assert.Equal(t, uint64(3), buckets.GetCount())
|
||||
assert.InDelta(t, 5, buckets.GetSum(), 0)
|
||||
assert.InDelta(t, 4, exported[1].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
|
||||
sum := exported[2].GetSum()
|
||||
assert.Equal(t, "Jobs.", exported[2].GetDescription())
|
||||
assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported[2].GetMetadata()[0]))
|
||||
assert.InDelta(t, 4, exported["gc_seconds"].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
|
||||
sum := exported["jobs_total"].GetSum()
|
||||
assert.Equal(t, "Jobs.", exported["jobs_total"].GetDescription())
|
||||
assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported["jobs_total"].GetMetadata()[0]))
|
||||
assert.NotZero(t, sum.GetDataPoints()[0].GetStartTimeUnixNano())
|
||||
assert.True(t, sum.GetIsMonotonic())
|
||||
assert.Equal(t, cumulative, sum.GetAggregationTemporality())
|
||||
assert.InDelta(t, 3, sum.GetDataPoints()[0].GetAsDouble(), 0)
|
||||
assert.True(t, proto.Equal(keyValue("status", "success"), sum.GetDataPoints()[0].GetAttributes()[0]))
|
||||
|
||||
assert.True(t, proto.Equal(keyValue("cicd.pipeline.result", "cancellation"), exported["gitea.runner.jobs"].GetSum().GetDataPoints()[0].GetAttributes()[0]))
|
||||
active := exported["gitea.runner.job.active"]
|
||||
assert.Equal(t, "{job}", active.GetUnit())
|
||||
assert.False(t, active.GetSum().GetIsMonotonic())
|
||||
assert.InDelta(t, 2, active.GetSum().GetDataPoints()[0].GetAsDouble(), 0)
|
||||
attributes := func(points []*metricpb.NumberDataPoint, key string) map[string]float64 {
|
||||
values := map[string]float64{}
|
||||
for _, point := range points {
|
||||
for _, kv := range point.GetAttributes() {
|
||||
if kv.GetKey() == key {
|
||||
values[kv.GetValue().GetStringValue()] = point.GetAsDouble()
|
||||
}
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
assert.Equal(t, map[string]float64{"stack": 20, "other": 50}, attributes(exported["go.memory.used"].GetSum().GetDataPoints(), "go.memory.type"))
|
||||
assert.Equal(t, map[string]float64{"available": 1, "offline": 0}, attributes(exported["cicd.worker.count"].GetSum().GetDataPoints(), "cicd.worker.state"))
|
||||
assert.True(t, proto.Equal(&commonpb.KeyValueList{Values: []*commonpb.KeyValue{
|
||||
keyValue("error.type", "unavailable"), keyValue("rpc.method", "runner.v1.RunnerService/FetchTask"), keyValue("cicd.system.component", "runner"),
|
||||
}}, &commonpb.KeyValueList{Values: exported["cicd.system.errors"].GetSum().GetDataPoints()[0].GetAttributes()}))
|
||||
}
|
||||
|
||||
func TestExportRefusesRedirects(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user