mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 06:45:44 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b14a38d4a | ||
|
|
179e435cd0 | ||
|
|
f963e9cfe8 | ||
|
|
e5aa9add02 | ||
|
|
329115b148 | ||
|
|
ff30f39b1b | ||
|
|
3802feb3fe | ||
|
|
27e9b4de87 | ||
|
|
4bedadef9d | ||
|
|
83b63250a3 | ||
|
|
2e59141ae9 | ||
|
|
020f0d2861 | ||
|
|
711b72735b | ||
|
|
436d0d2942 | ||
|
|
a284a62d96 | ||
|
|
2b1e64e414 | ||
|
|
ddee2e054a | ||
|
|
629e062f86 | ||
|
|
73e9552f06 | ||
|
|
05162998eb | ||
|
|
568f89d186 | ||
|
|
d93b7d278a | ||
|
|
0cedd98051 | ||
|
|
07ca4bfa7f | ||
|
|
78610b01f6 |
No files matched your search
@@ -1,9 +1,7 @@
|
||||
name: checks
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
# The runner image ships a stale docker.io login; point docker at an empty config so
|
||||
|
||||
@@ -9,9 +9,10 @@ XGO_VERSION := go-1.27.x
|
||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
|
||||
|
||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
||||
@@ -23,7 +24,7 @@ DOCKER_TAG ?= nightly
|
||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
||||
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]3.2 # renovate: datasource=go
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/[email protected] # renovate: datasource=go
|
||||
|
||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||
@@ -170,7 +171,7 @@ E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
|
||||
.PHONY: install
|
||||
install: $(GOFILES) ## install the runner binary via `go install`
|
||||
|
||||
@@ -4,7 +4,19 @@
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
On the runner:
|
||||
|
||||
- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows.
|
||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
|
||||
Where steps run, in the job image or on the runner in host mode:
|
||||
|
||||
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
|
||||
- `node` for JavaScript actions and `hashFiles()`.
|
||||
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
|
||||
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
|
||||
|
||||
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
|
||||
|
||||
### Download pre-built binary
|
||||
|
||||
@@ -79,53 +91,24 @@ If the registry succeed, it will run immediately. Next time, you could run the r
|
||||
|
||||
### Run with docker
|
||||
|
||||
```bash
|
||||
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> -v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:nightly
|
||||
```
|
||||
|
||||
Mount a volume on `/data` if you want the registration file and optional config to survive container recreation (see [scripts/run.sh](scripts/run.sh)).
|
||||
|
||||
> **`/data` does not hold the image cache.** It is the runner's working directory and contains only the `.runner` registration file and, optionally, your config file. Images pulled for jobs live in the *Docker daemon's* data root, which for the `dind` flavours is inside the container (`/var/lib/docker`, or `/home/rootless/.local/share/docker` for `dind-rootless`). To keep the image cache across restarts, give that path its own volume as well — otherwise every new container re-pulls the job images. With the `basic` flavour the images live on whichever daemon you point the runner at, so there is nothing extra to persist.
|
||||
|
||||
### Image flavours
|
||||
|
||||
The image is published in three flavours, all built from the single multi-stage [Dockerfile](Dockerfile) in this repository. They differ only in how a Docker daemon is made available to the jobs the runner executes; the `gitea-runner` binary inside them is identical.
|
||||
|
||||
| Tag | Build target | Base image | Docker daemon | Process supervisor | Runs as |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| `latest` (and `<version>`) | `basic` | `alpine` | none — uses an external daemon you provide | [`tini`](https://github.com/krallin/tini) | `root` |
|
||||
| `latest-dind` | `dind` | `docker:dind` | bundled, started inside the container | [`s6`](https://skarnet.org/software/s6/) | `root` (privileged) |
|
||||
| `latest-dind-rootless` | `dind-rootless` | `docker:dind-rootless` | bundled, started rootless inside the container | [`s6`](https://skarnet.org/software/s6/) | `rootless` (UID 1000) |
|
||||
|
||||
#### `latest` — basic
|
||||
|
||||
The default flavour ships only the runner on a minimal Alpine base. It contains **no Docker daemon of its own**: jobs that use `docker://` images need a daemon supplied from outside the container, typically by bind-mounting the host's socket:
|
||||
|
||||
```bash
|
||||
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:latest
|
||||
```
|
||||
|
||||
`tini` is the entrypoint (it reaps zombie processes), and it just runs [`scripts/run.sh`](scripts/run.sh), which registers the runner on first start and then execs `gitea-runner daemon`. This flavour does not need `--privileged`. The trade-off is that jobs share the host's daemon, so they can see other containers and images on that daemon.
|
||||
|
||||
#### `latest-dind` — Docker-in-Docker
|
||||
|
||||
This flavour is based on the official `docker:dind` image and bundles its own Docker daemon, so it needs no external socket — only the `--privileged` flag that Docker-in-Docker requires:
|
||||
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
|
||||
|
||||
```bash
|
||||
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
--name my_runner gitea/runner:latest-dind
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
|
||||
```
|
||||
|
||||
Two processes have to run side by side here (the Docker daemon and the runner), so the entrypoint is the [`s6`](https://skarnet.org/software/s6/) supervision tree under [`scripts/s6`](scripts/s6) instead of `tini`. `s6` starts `dockerd`, and the runner service waits for the daemon to come up (`s6-svwait`) before launching [`run.sh`](scripts/run.sh). Each container has a private daemon isolated from the host's, at the cost of running privileged.
|
||||
The volumes keep the runner's registration and the job image cache across container recreation.
|
||||
|
||||
#### `latest-dind-rootless` — rootless Docker-in-Docker
|
||||
### Image flavours
|
||||
|
||||
Same idea as `dind`, but built on `docker:dind-rootless` so the bundled daemon and the runner run as an unprivileged user (`rootless`, UID 1000) rather than `root`. `DOCKER_HOST` is preset to `unix:///run/user/1000/docker.sock` so the runner talks to the rootless daemon. This reduces the blast radius compared to the privileged `dind` flavour, but rootless Docker carries the usual rootless limitations (networking, cgroups, storage drivers, and some operations that need additional host configuration such as `/etc/subuid` / `/etc/subgid` mappings and unprivileged user-namespace support).
|
||||
| Tag | Docker daemon for jobs |
|
||||
| --- | --- |
|
||||
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
|
||||
> **The UID is fixed at 1000.** It comes from the `rootless` user baked into the upstream `docker:dind-rootless` base image, and the bundled daemon always listens on `/run/user/1000/docker.sock` inside the container, so running this flavour as a different user (`--user 1001`) does not work. If you need the runner to talk to a *host* rootless daemon that runs under some other UID, use the `basic` flavour instead and bind-mount that daemon's socket (see [examples/vm/rootless-docker.md](examples/vm/rootless-docker.md)); pointing `DOCKER_HOST` at a host socket from inside `dind-rootless` will not work. Changing the UID otherwise means rebuilding the image from a base with a different `rootless` user.
|
||||
|
||||
> **Note on Podman:** these images target the Docker daemon. The bundled `dind`/`dind-rootless` daemons are `dockerd`, not Podman, and the `basic` flavour expects a Docker-compatible socket. Running them under rootless Podman is not a supported configuration, though pointing the `basic` flavour at a Podman socket that emulates the Docker API may work for some workloads.
|
||||
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -186,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
|
||||
|
||||
#### Environment variables
|
||||
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
|
||||
### Labels
|
||||
|
||||
@@ -201,10 +184,10 @@ A label is written as:
|
||||
| Part | Meaning |
|
||||
| --- | --- |
|
||||
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
|
||||
| `schema` | Either `docker` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` schema: the image to run the job in. |
|
||||
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
|
||||
|
||||
Two schemas are supported:
|
||||
Three schemas are supported:
|
||||
|
||||
- **`docker://<image>`** — the job runs inside a container created from `<image>`:
|
||||
|
||||
@@ -212,6 +195,12 @@ Two schemas are supported:
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
|
||||
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
|
||||
|
||||
```text
|
||||
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
|
||||
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
|
||||
|
||||
```text
|
||||
@@ -226,9 +215,9 @@ ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
|
||||
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
|
||||
|
||||
Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host` and `docker` are treated as schemas.
|
||||
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
|
||||
|
||||
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs: in `runner.default_image` where docker is available, on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
|
||||
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
|
||||
|
||||
@@ -275,7 +264,7 @@ volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. The proxy removes containers, networks and volumes after post steps and the completed hook. Named volumes created through it are job-scoped. To retain resources, mount the daemon socket explicitly in `container.options`. Host jobs use their existing Docker access, where `.` works.
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
#### Proxy
|
||||
|
||||
@@ -380,6 +369,7 @@ While the runner is idle it cleans up after earlier jobs:
|
||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||
|
||||
#### Post-task script (`runner.post_task_script`)
|
||||
@@ -404,7 +394,7 @@ See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environm
|
||||
|
||||
#### OpenTelemetry (`OTEL_*`)
|
||||
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
|
||||
|
||||
#### Local job logs (`log.job.dir`)
|
||||
|
||||
|
||||
+213
-357
@@ -6,24 +6,23 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/storer"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/mattn/go-isatty"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -32,6 +31,13 @@ var (
|
||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||
|
||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
|
||||
"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
|
||||
"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
|
||||
}
|
||||
|
||||
cloneLocks lock.Keyed[string] // key: clone target directory
|
||||
|
||||
@@ -51,146 +57,60 @@ type Error struct {
|
||||
commit string
|
||||
}
|
||||
|
||||
func (e *Error) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e *Error) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func (e *Error) Commit() string {
|
||||
return e.commit
|
||||
}
|
||||
|
||||
// goGitMu serializes go-git repository access across the process. go-git is not safe for
|
||||
// concurrent use of the same repository (even read access decodes packfiles into shared
|
||||
// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded
|
||||
// operations are fast local reads; gitea runs one job per process, so the lock is effectively
|
||||
// uncontended in production.
|
||||
var goGitMu sync.Mutex
|
||||
func (e *Error) Error() string { return e.err.Error() }
|
||||
func (e *Error) Unwrap() error { return e.err }
|
||||
func (e *Error) Commit() string { return e.commit }
|
||||
|
||||
// FindGitRevision get the current git revision
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
return findGitRevision(ctx, file)
|
||||
}
|
||||
|
||||
func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
gitDir, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
head, err := gitDir.Reference(plumbing.HEAD, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
if len(sha) < 7 {
|
||||
return "", "", errors.New("HEAD could not be resolved")
|
||||
}
|
||||
|
||||
if head.Hash().IsZero() {
|
||||
return "", "", errors.New("HEAD sha1 could not be resolved")
|
||||
}
|
||||
|
||||
hash := head.Hash().String()
|
||||
|
||||
logger.Debugf("Found revision: %s", hash)
|
||||
return hash[:7], strings.TrimSpace(hash), nil
|
||||
logger.Debugf("Found revision: %s", sha)
|
||||
return sha[:7], sha, nil
|
||||
}
|
||||
|
||||
// FindGitRef get the current git ref
|
||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
logger.Debugf("Loading revision from git directory")
|
||||
_, ref, err := findGitRevision(ctx, file)
|
||||
_, revision, err := FindGitRevision(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
logger.Debugf("HEAD points to '%s'", revision)
|
||||
|
||||
logger.Debugf("HEAD points to '%s'", ref)
|
||||
|
||||
// Prefer the git library to iterate over the references and find a matching tag or branch.
|
||||
refTag := ""
|
||||
refBranch := ""
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
iter, err := repo.References()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// find the reference that matches the revision's has
|
||||
err = iter.ForEach(func(r *plumbing.Reference) error {
|
||||
/* tags and branches will have the same hash
|
||||
* when a user checks out a tag, it is not mentioned explicitly
|
||||
* in the go-git package, we must identify the revision
|
||||
* then check if any tag matches that revision,
|
||||
* if so then we checked out a tag
|
||||
* else we look for branches and if matches,
|
||||
* it means we checked out a branch
|
||||
*
|
||||
* If a branches matches first we must continue and check all tags (all references)
|
||||
* in case we match with a tag later in the interation
|
||||
*/
|
||||
if r.Hash().String() == ref {
|
||||
if r.Name().IsTag() {
|
||||
refTag = r.Name().String()
|
||||
}
|
||||
if r.Name().IsBranch() {
|
||||
refBranch = r.Name().String()
|
||||
}
|
||||
var tag, branch string
|
||||
for ref := range strings.FieldsSeq(refs) {
|
||||
switch {
|
||||
case strings.HasPrefix(ref, "refs/tags/"):
|
||||
tag = ref
|
||||
case strings.HasPrefix(ref, "refs/heads/"):
|
||||
branch = ref
|
||||
}
|
||||
|
||||
// we found what we where looking for
|
||||
if refTag != "" && refBranch != "" {
|
||||
return storer.ErrStop
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// order matters here see above comment.
|
||||
if refTag != "" {
|
||||
return refTag, nil
|
||||
if tag != "" {
|
||||
return tag, nil
|
||||
}
|
||||
if refBranch != "" {
|
||||
return refBranch, nil
|
||||
if branch != "" {
|
||||
return branch, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
|
||||
}
|
||||
|
||||
// FindGithubRepo get the repo
|
||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
url, err := findGitRemoteURL(ctx, file, "origin")
|
||||
url, err := findGitRemoteURL(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -198,28 +118,16 @@ func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, e
|
||||
return slug, nil
|
||||
}
|
||||
|
||||
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||
urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
remote, err := repo.Remote(remoteName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
if urls == "" {
|
||||
return "", errors.New("remote 'origin' exists but has no URL")
|
||||
}
|
||||
|
||||
if len(remote.Config().URLs) < 1 {
|
||||
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)
|
||||
}
|
||||
|
||||
return remote.Config().URLs[0], nil
|
||||
url, _, _ := strings.Cut(urls, "\n")
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func findGitSlug(url, githubInstance string) (string, string) {
|
||||
@@ -260,69 +168,35 @@ type NewGitCloneExecutorInput struct {
|
||||
Quiet bool
|
||||
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
}
|
||||
|
||||
// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) {
|
||||
r, err := git.PlainOpen(input.Dir)
|
||||
// CloneIfRequired reports whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) {
|
||||
var origin string
|
||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||
if err == nil {
|
||||
// Verify the cached clone still points to the resolved URL before reusing it.
|
||||
remote, err := r.Remote("origin")
|
||||
if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL {
|
||||
// Reuse existing clone
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
case len(remote.Config().URLs) == 0:
|
||||
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
|
||||
default:
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
|
||||
}
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
origin, err = findGitRemoteURL(ctx, input.Dir)
|
||||
}
|
||||
|
||||
var progressWriter io.Writer
|
||||
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
|
||||
if entry, ok := logger.(*log.Entry); ok {
|
||||
progressWriter = entry.WriterLevel(log.DebugLevel)
|
||||
} else if lgr, ok := logger.(*log.Logger); ok {
|
||||
progressWriter = lgr.WriterLevel(log.DebugLevel)
|
||||
} else {
|
||||
log.Errorf("Unable to get writer from logger (type=%T)", logger)
|
||||
progressWriter = os.Stdout
|
||||
}
|
||||
if err == nil && origin == input.URL {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
cloneOptions := git.CloneOptions{
|
||||
URL: input.URL,
|
||||
Progress: progressWriter,
|
||||
|
||||
InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea
|
||||
if err == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
}
|
||||
if input.Token != "" {
|
||||
cloneOptions.Auth = &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: input.Token,
|
||||
}
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
|
||||
r, err = cloneAtDepth(ctx, input, cloneOptions, logger)
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err)
|
||||
return nil, false, err
|
||||
if err := clone(ctx, input); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err = os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return nil, false, err
|
||||
if err := os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return r, false, nil
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutor creates an executor to clone git repos
|
||||
@@ -335,134 +209,125 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||
}
|
||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||
|
||||
defer AcquireCloneLock(input.Dir)()
|
||||
|
||||
refName := plumbing.ReferenceName("refs/heads/" + input.Ref)
|
||||
r, reused, err := CloneIfRequired(ctx, refName, input, logger)
|
||||
reused, err := CloneIfRequired(ctx, input, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isOfflineMode := input.OfflineMode
|
||||
|
||||
fetchOptions := git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD", "refs/heads/*:refs/remotes/origin/*"},
|
||||
Force: true,
|
||||
InsecureSkipTLS: input.InsecureSkipTLS,
|
||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||
return err
|
||||
}
|
||||
if input.Token != "" {
|
||||
fetchOptions.Auth = &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: input.Token,
|
||||
}
|
||||
}
|
||||
|
||||
// Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth
|
||||
// Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone.
|
||||
if isShallow(r) {
|
||||
fetchOptions.Depth = 1
|
||||
if spec, ok := shallowFetchRefSpec(r, input.Ref); ok {
|
||||
fetchOptions.RefSpecs = []config.RefSpec{spec}
|
||||
}
|
||||
}
|
||||
|
||||
// A just-cloned ref is as current as a fetch would make it, and a commit hash never moves.
|
||||
// TODO: revalidate a mutable ref with a conditional archive request instead, once every
|
||||
// supported Gitea sends an ETag for them: https://github.com/go-gitea/gitea/pull/39289
|
||||
_, present := refRevision(r, input.Ref)
|
||||
refresh := !isOfflineMode && (!present || (reused && !plumbing.IsHash(input.Ref)))
|
||||
|
||||
if refresh {
|
||||
err = r.FetchContext(ctx, &fetchOptions)
|
||||
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||
if err := fetch(ctx, input); err != nil {
|
||||
return err
|
||||
}
|
||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||
}
|
||||
|
||||
var hash *plumbing.Hash
|
||||
rev := plumbing.Revision(input.Ref)
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
// ResolveRevision returns a nil hash on error, and a branch ref legitimately fails
|
||||
// here (no local refs/heads/<ref>); the duck-typing below resolves it.
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
} else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) {
|
||||
return &Error{
|
||||
err: ErrShortRef,
|
||||
commit: hash.String(),
|
||||
}
|
||||
}
|
||||
|
||||
rev, _ = refRevision(r, input.Ref)
|
||||
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
return err
|
||||
}
|
||||
|
||||
var w *git.Worktree
|
||||
if w, err = r.Worktree(); err != nil {
|
||||
return err
|
||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||
}
|
||||
|
||||
reusedMsg := ""
|
||||
if isOfflineMode && reused {
|
||||
if input.OfflineMode && reused {
|
||||
reusedMsg = " (reused in offline mode)"
|
||||
}
|
||||
|
||||
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
|
||||
|
||||
if err = w.Checkout(&git.CheckoutOptions{
|
||||
Hash: *hash,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
logger.Errorf("Unable to checkout %s: %v", *hash, err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Checked out %s", input.Ref)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// cloneAtDepth falls back to a full clone when the shallow attempt fails.
|
||||
func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
|
||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||
return nil
|
||||
} else if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
args := []string{"clone", "--no-checkout"}
|
||||
if input.Depth > 0 {
|
||||
if plumbing.IsHash(input.Ref) {
|
||||
r, err := fetchPinnedSHA(ctx, input, opts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow fetch of %s at %s failed: %v", input.URL, input.Ref, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
refNames := []plumbing.ReferenceName{plumbing.NewBranchReferenceName(input.Ref), plumbing.NewTagReferenceName(input.Ref)}
|
||||
if input.Ref == plumbing.HEAD.String() || strings.HasPrefix(input.Ref, "refs/") {
|
||||
refNames = []plumbing.ReferenceName{plumbing.ReferenceName(input.Ref)} // HEAD is the remote's default branch
|
||||
}
|
||||
for _, refName := range refNames {
|
||||
shallowOpts := opts
|
||||
shallowOpts.Depth = input.Depth
|
||||
shallowOpts.SingleBranch = true
|
||||
shallowOpts.ReferenceName = refName
|
||||
shallowOpts.Tags = git.NoTags
|
||||
args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
|
||||
if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
|
||||
args = append(args, "--branch", input.Ref)
|
||||
}
|
||||
}
|
||||
args = append(args, input.URL, input.Dir)
|
||||
if _, err := runGit(ctx, "", &input, args...); err == nil {
|
||||
return nil
|
||||
} else if input.Depth == 0 {
|
||||
return err
|
||||
}
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
|
||||
return err
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
args := []string{"fetch", "--force", "--prune"}
|
||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||
args = append(args, "--depth", "1", "--no-tags")
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
|
||||
return err
|
||||
}
|
||||
|
||||
func cachedRef(ref string) string {
|
||||
return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
|
||||
}
|
||||
|
||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||
if !isFullObjectID(ref) {
|
||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
||||
return sha, nil
|
||||
}
|
||||
}
|
||||
logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
|
||||
}
|
||||
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
|
||||
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
|
||||
return "", &Error{err: ErrShortRef, commit: sha}
|
||||
}
|
||||
return sha, err
|
||||
}
|
||||
|
||||
return git.PlainCloneContext(ctx, input.Dir, false, &opts)
|
||||
func isFullObjectID(ref string) bool {
|
||||
return ObjectFormat(ref) != ""
|
||||
}
|
||||
|
||||
func isFullHash(ref, resolved string) bool {
|
||||
return isFullObjectID(ref) && len(ref) == len(resolved)
|
||||
}
|
||||
|
||||
// ObjectFormat returns the object format of a full commit hash, empty for anything else.
|
||||
func ObjectFormat(sha string) string {
|
||||
switch {
|
||||
case !hexRefRegex.MatchString(sha):
|
||||
return ""
|
||||
case len(sha) == 40:
|
||||
return "sha1"
|
||||
case len(sha) == 64:
|
||||
return "sha256"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func removePartialClone(dir string) error {
|
||||
@@ -472,71 +337,62 @@ func removePartialClone(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchPinnedSHA(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions) (*git.Repository, error) {
|
||||
r, err := git.PlainInit(input.Dir, false)
|
||||
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
|
||||
output, err := runGit(ctx, dir, nil, args...)
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
var config []string
|
||||
if insecureSkipTLS {
|
||||
config = append(config, "http.sslVerify=false")
|
||||
}
|
||||
if token != "" {
|
||||
config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
for i, entry := range config {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
|
||||
env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||
name, _, _ := strings.Cut(entry, "=")
|
||||
return slices.Contains(LocalEnvVars, name)
|
||||
})
|
||||
for key, value := range ConfigEnv(config...) {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{input.URL}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.FetchContext(ctx, &git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{pinnedRefSpec(input.Ref)},
|
||||
Depth: input.Depth,
|
||||
Tags: git.NoTags,
|
||||
Auth: opts.Auth,
|
||||
Progress: opts.Progress,
|
||||
InsecureSkipTLS: opts.InsecureSkipTLS,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// pinnedRefSpec keeps a hash-shaped name out of refs/heads and refs/tags.
|
||||
func pinnedRefSpec(sha string) config.RefSpec {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:refs/pinned/%s", sha, sha))
|
||||
}
|
||||
|
||||
// refRevision picks the revision to check out and reports whether it resolves locally.
|
||||
func refRevision(r *git.Repository, ref string) (plumbing.Revision, bool) {
|
||||
if plumbing.IsHash(ref) {
|
||||
// git ignores a ref named as 40 hex digits, so a full hash always denotes the commit itself.
|
||||
_, err := r.CommitObject(plumbing.NewHash(ref))
|
||||
return plumbing.Revision(ref), err == nil
|
||||
}
|
||||
if _, err := r.Tag(ref); err == nil {
|
||||
return plumbing.Revision(path.Join("refs", "tags", ref)), true
|
||||
}
|
||||
remoteRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", ref))
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
return plumbing.Revision(remoteRef), true
|
||||
}
|
||||
rev := plumbing.Revision(ref)
|
||||
_, err := r.ResolveRevision(rev)
|
||||
return rev, err == nil
|
||||
}
|
||||
|
||||
// isShallow reports whether the local repository was cloned with a limited depth.
|
||||
func isShallow(r *git.Repository) bool {
|
||||
shallows, err := r.Storer.Shallow()
|
||||
return err == nil && len(shallows) > 0
|
||||
}
|
||||
|
||||
// shallowFetchRefSpec limits a shallow update to the requested ref, falling back to the broad refspec when it is not present locally.
|
||||
func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
|
||||
tagRef := plumbing.NewTagReferenceName(ref)
|
||||
if _, err := r.Reference(tagRef, false); err == nil {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true
|
||||
}
|
||||
remoteRef := plumbing.NewRemoteReferenceName("origin", ref)
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
branchRef := plumbing.NewBranchReferenceName(ref)
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
|
||||
}
|
||||
if plumbing.IsHash(ref) {
|
||||
// The broad refspec carries only advertised tips, never a pinned commit.
|
||||
return pinnedRefSpec(ref), true
|
||||
}
|
||||
return "", false
|
||||
return output, nil
|
||||
}
|
||||
+111
-141
@@ -6,6 +6,7 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -15,18 +16,12 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
gogitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||
gogitfile "github.com/go-git/go-git/v5/plumbing/transport/file"
|
||||
log "github.com/sirupsen/logrus"
|
||||
logrustest "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -88,14 +83,27 @@ func cleanGitHooks(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFindGithubRepoUsesOrigin(t *testing.T) {
|
||||
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
const remoteURL = "https://github.com/owner/repo.git"
|
||||
require.NoError(t, gitCmd("init", basedir))
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, cleanGitHooks(basedir))
|
||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||
|
||||
url, err := findGitRemoteURL(context.Background(), basedir, "origin")
|
||||
shortSHA, sha, err := FindGitRevision(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
||||
require.Len(t, sha, 64)
|
||||
require.Equal(t, sha[:7], shortSHA)
|
||||
|
||||
ref, err := FindGitRef(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/main", ref)
|
||||
|
||||
url, err := findGitRemoteURL(context.Background(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, remoteURL, url)
|
||||
|
||||
@@ -105,6 +113,7 @@ func TestFindGithubRepoUsesOrigin(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitFindRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
@@ -191,7 +200,9 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||
@@ -205,6 +216,13 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
fullSha := strings.TrimSpace(string(out))
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Err error
|
||||
Ref string
|
||||
@@ -225,26 +243,42 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
Err: &Error{ErrShortRef, fullSha},
|
||||
Ref: fullSha[:7],
|
||||
},
|
||||
"HEAD": {
|
||||
Ref: "HEAD",
|
||||
},
|
||||
"hex branch named like its own commit": {
|
||||
Ref: fullSha[:4],
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
URL: "https://example.invalid/action",
|
||||
Ref: tt.Ref,
|
||||
Dir: t.TempDir(),
|
||||
Dir: dir,
|
||||
})
|
||||
|
||||
err := clone(context.Background())
|
||||
if tt.Err != nil {
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, err)
|
||||
} else {
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, clone(context.Background()), "a retry on the cache keeps the error")
|
||||
return
|
||||
}
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
marker := filepath.Join(dir, "marker")
|
||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||
require.NoError(t, clone(context.Background()))
|
||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
createRemote := func(message string) string {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -268,6 +302,11 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
nested := filepath.Join(cacheDir, "nested")
|
||||
require.NoError(t, os.Mkdir(nested, 0o755))
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context()))
|
||||
assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository")
|
||||
|
||||
markerPath := filepath.Join(cacheDir, "stale-marker")
|
||||
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
|
||||
|
||||
@@ -277,7 +316,7 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin")
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, newRemoteDir, originURL)
|
||||
|
||||
@@ -289,61 +328,36 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning")
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorNonFastForwardRef(t *testing.T) {
|
||||
// Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes
|
||||
// non-fast-forward between two fetches. Before the fix, the fetch used Force=false,
|
||||
// causing go-git to return ErrForceNeeded and short-circuit the checkout.
|
||||
|
||||
// Create a bare "remote" repo with an initial commit on main and a feature branch.
|
||||
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
// We need a working clone to push commits from.
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Create a feature branch (simulates refs/pull/N/head).
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature"))
|
||||
|
||||
// First clone via the executor — should succeed and cache the repo.
|
||||
cloneDir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: cloneDir,
|
||||
})
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
|
||||
require.NoError(t, clone(context.Background()))
|
||||
|
||||
// Now force-push the feature branch to a non-fast-forward commit (simulates
|
||||
// a PR rebase). This makes refs/heads/feature non-fast-forward.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
|
||||
require.NoError(t, clone(context.Background()), "a non-fast-forward ref must update")
|
||||
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
|
||||
|
||||
// Also advance main so we can verify the clone picks up the new commit.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
|
||||
// Second clone to the same directory — before the fix this returned ErrForceNeeded
|
||||
// and left the working tree at the old commit.
|
||||
err := clone(context.Background())
|
||||
require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true")
|
||||
|
||||
// Verify the working tree was actually updated to the latest main commit.
|
||||
out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit")
|
||||
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: t.TempDir()})
|
||||
require.Error(t, tagClone(context.Background()))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
|
||||
require.NoError(t, tagClone(context.Background()), "a tag published after a failed lookup must be fetched")
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with a single commit on main.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -390,6 +404,7 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
@@ -428,6 +443,7 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
@@ -471,12 +487,11 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
return remote
|
||||
}
|
||||
|
||||
t.Run("commit hash falls back to a full clone when the remote refuses unadvertised objects", func(t *testing.T) {
|
||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.NoFileExists(t, shallowMarker(dir))
|
||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
@@ -503,76 +518,46 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
t.Run("moving branch updates while staying shallow", func(t *testing.T) {
|
||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1,
|
||||
})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
require.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
|
||||
// Advance main on the remote, then reuse the existing shallow clone.
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorTransportSessions(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--initial-branch=main", workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "-a", "v2", "-m", "v2"))
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Ref string
|
||||
Depth int
|
||||
}{
|
||||
"shallow branch": {"main", 1},
|
||||
"full clone branch": {"main", 0},
|
||||
"full lightweight tag": {"v1", 0},
|
||||
"full annotated tag": {"v2", 0},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
counter := installCountingTransport(t)
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: workDir, Ref: tt.Ref, Dir: dir, Depth: tt.Depth,
|
||||
})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Swap(0), "cold clone")
|
||||
assert.Equal(t, gitRevParse(t, workDir, tt.Ref+"^{commit}"), gitRevParse(t, dir, "HEAD"))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Swap(0), "unchanged warm cache")
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.yml"), []byte(name), 0o644))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.yml"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", name))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "-a", "v2", "-m", "v2"))
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("staged"), 0o644))
|
||||
require.NoError(t, gitCmd("-C", dir, "add", "action.yml"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "action.yml"), []byte("unstaged"), 0o644))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Load(), "updated warm cache")
|
||||
assert.Equal(t, gitRevParse(t, workDir, "HEAD"), gitRevParse(t, dir, "HEAD"))
|
||||
status, err := exec.Command("git", "-C", dir, "status", "--porcelain").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, string(status))
|
||||
})
|
||||
}
|
||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
||||
work := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
||||
sha := gitRevParse(t, work, "HEAD")
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir))
|
||||
require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef)
|
||||
require.NoError(t, os.RemoveAll(remote))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
@@ -596,25 +581,6 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type countingTransport struct {
|
||||
transport.Transport
|
||||
sessions atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingTransport) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) {
|
||||
c.sessions.Add(1)
|
||||
return c.Transport.NewUploadPackSession(ep, auth)
|
||||
}
|
||||
|
||||
func installCountingTransport(t *testing.T) *countingTransport {
|
||||
t.Helper()
|
||||
counter := &countingTransport{Transport: gogitfile.DefaultClient}
|
||||
gogitclient.InstallProtocol("file", counter)
|
||||
t.Cleanup(func() { gogitclient.InstallProtocol("file", gogitfile.DefaultClient) })
|
||||
|
||||
return counter
|
||||
}
|
||||
|
||||
func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()
|
||||
@@ -715,12 +681,17 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
}
|
||||
|
||||
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
|
||||
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
block := make(chan struct{})
|
||||
reached := make(chan struct{})
|
||||
var once sync.Once
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
||||
once.Do(func() { close(reached) })
|
||||
var authorization string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
|
||||
once.Do(func() {
|
||||
authorization = request.Header.Get("Authorization")
|
||||
close(reached)
|
||||
})
|
||||
<-block
|
||||
}))
|
||||
t.Cleanup(func() {
|
||||
@@ -729,17 +700,15 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
})
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||
repo, err := gogit.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gitCmd("init", dir))
|
||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx)
|
||||
}()
|
||||
|
||||
select {
|
||||
@@ -747,6 +716,7 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
|
||||
+40
-34
@@ -10,6 +10,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type interfaceIP struct {
|
||||
net.IP
|
||||
net.Interface
|
||||
}
|
||||
|
||||
// GetOutboundIP returns an outbound IP address of this machine.
|
||||
// It tries to access the internet and returns the local IP address of the connection.
|
||||
// If the machine cannot access the internet, it returns a preferred IP address from network interfaces.
|
||||
@@ -26,11 +31,7 @@ func GetOutboundIP() net.IP {
|
||||
|
||||
// So the machine cannot access the internet. Pick an IP address from network interfaces.
|
||||
if ifs, err := net.Interfaces(); err == nil {
|
||||
type IP struct {
|
||||
net.IP
|
||||
net.Interface
|
||||
}
|
||||
var ips []IP
|
||||
var ips []interfaceIP
|
||||
for _, i := range ifs {
|
||||
if addrs, err := i.Addrs(); err == nil {
|
||||
for _, addr := range addrs {
|
||||
@@ -42,40 +43,45 @@ func GetOutboundIP() net.IP {
|
||||
ip = v.IP
|
||||
}
|
||||
if ip.IsGlobalUnicast() {
|
||||
ips = append(ips, IP{ip, i})
|
||||
ips = append(ips, interfaceIP{ip, i})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(ips) > 1 {
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
ifi := ips[i].Interface
|
||||
ifj := ips[j].Interface
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
ipi := ips[i].IP
|
||||
ipj := ips[j].IP
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// en0 is preferred to en1
|
||||
if ifi.Name != ifj.Name {
|
||||
return ifi.Name < ifj.Name
|
||||
}
|
||||
|
||||
// fallback
|
||||
return ipi.String() < ipj.String()
|
||||
})
|
||||
return ips[0].IP
|
||||
}
|
||||
return preferredIP(ips)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func preferredIP(ips []interfaceIP) net.IP {
|
||||
if len(ips) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
ifi := ips[i].Interface
|
||||
ifj := ips[j].Interface
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
ipi := ips[i].IP
|
||||
ipj := ips[j].IP
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// en0 is preferred to en1
|
||||
if ifi.Name != ifj.Name {
|
||||
return ifi.Name < ifj.Name
|
||||
}
|
||||
|
||||
// fallback
|
||||
return ipi.String() < ipj.String()
|
||||
})
|
||||
return ips[0].IP
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestPreferredIP(t *testing.T) {
|
||||
assert.Nil(t, preferredIP(nil))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{{IP: net.ParseIP("2001:db8::1")}}))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{
|
||||
{IP: net.ParseIP("192.0.2.1"), Name: "wlan0"},
|
||||
{IP: net.ParseIP("2001:db8::1"), Name: "eth0"},
|
||||
}))
|
||||
}
|
||||
@@ -75,6 +75,9 @@ const (
|
||||
HealthUnhealthy = string(container.Unhealthy)
|
||||
)
|
||||
|
||||
// marks what a runner creates for its jobs, telling its leftovers from another runner's
|
||||
const runnerUUIDLabel = "com.gitea.runner.uuid"
|
||||
|
||||
// ErrContainerNotFound reports a container the daemon no longer knows. Its text is a
|
||||
// fragment, missingContainerError composes it into the message every operation shares.
|
||||
var ErrContainerNotFound = errors.New("does not exist")
|
||||
|
||||
@@ -46,14 +46,13 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
inspectImage, err := cli.ImageInspect(ctx, imageName)
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
if _, err := cli.ImageInspect(ctx, imageName); cerrdefs.IsNotFound(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if _, err = cli.ImageRemove(ctx, inspectImage.ID, client.ImageRemoveOptions{
|
||||
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
||||
Force: force,
|
||||
PruneChildren: pruneChildren,
|
||||
}); err != nil {
|
||||
|
||||
@@ -24,10 +24,6 @@ import (
|
||||
const (
|
||||
networkCreateAttempts = 3
|
||||
networkCreateRetryDelay = time.Second
|
||||
|
||||
// marks the networks a runner creates for its jobs, so it can tell its own leftovers from
|
||||
// those of another runner sharing the daemon
|
||||
runnerUUIDLabel = "com.gitea.runner.uuid"
|
||||
)
|
||||
|
||||
// RemoveOrphanNetworks removes the networks this runner created for jobs whose teardown did
|
||||
@@ -91,45 +87,54 @@ func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExec
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return createDockerNetwork(ctx, cli, name, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var options map[string]string
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
// the daemon's mtu only reaches its default bridge, Docker's default is left out to keep default-network-opts
|
||||
if mtu := n.Options["com.docker.network.driver.mtu"]; n.Name == network.NetworkBridge && mtu != "" && mtu != "1500" {
|
||||
options = map[string]string{"com.docker.network.driver.mtu": mtu}
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Options: options,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
|
||||
func runnerLabels(runnerUUID string) map[string]string {
|
||||
|
||||
@@ -57,6 +57,27 @@ func TestIsAddressPoolExhausted(t *testing.T) {
|
||||
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
|
||||
}
|
||||
|
||||
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
bridgeOptions, want map[string]string
|
||||
}{
|
||||
{
|
||||
map[string]string{"com.docker.network.bridge.host_binding_ipv4": "127.0.0.1", "com.docker.network.driver.mtu": "1450"},
|
||||
map[string]string{"com.docker.network.driver.mtu": "1450"},
|
||||
},
|
||||
{map[string]string{"com.docker.network.driver.mtu": "1500"}, nil},
|
||||
} {
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).
|
||||
Return(mobyclient.NetworkListResult{Items: []network.Summary{{Name: network.NetworkBridge, Options: tc.bridgeOptions}}}, nil)
|
||||
client.On("NetworkCreate", ctx, "job-network", mobyclient.NetworkCreateOptions{Driver: "bridge", Scope: "local", Options: tc.want}).
|
||||
Return(mobyclient.NetworkCreateResult{}, nil)
|
||||
require.NoError(t, createDockerNetwork(ctx, client, "job-network", NewDockerNetworkCreateExecutorInput{}))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
|
||||
return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
|
||||
}
|
||||
|
||||
+60
-94
@@ -7,8 +7,8 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -25,16 +25,12 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
|
||||
"dario.cat/mergo"
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
"github.com/docker/cli/cli/connhelper"
|
||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
@@ -982,42 +978,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
logger.Error(err)
|
||||
}
|
||||
}(tarFile)
|
||||
tw := tar.NewWriter(tarFile)
|
||||
|
||||
srcPrefix := filepath.Dir(srcPath)
|
||||
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
|
||||
srcPrefix += string(filepath.Separator)
|
||||
}
|
||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||
|
||||
var ignorer gitignore.Matcher
|
||||
if useGitIgnore {
|
||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
||||
if err != nil {
|
||||
logger.Debugf("Error loading .gitignore: %v", err)
|
||||
}
|
||||
|
||||
ignorer = gitignore.NewMatcher(ps)
|
||||
}
|
||||
|
||||
fc := &filecollector.FileCollector{
|
||||
Ignorer: ignorer,
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
Handler: &filecollector.TarCollector{
|
||||
TarWriter: tw,
|
||||
UID: cr.UID,
|
||||
GID: cr.GID,
|
||||
DstDir: dstPath[1:],
|
||||
},
|
||||
}
|
||||
|
||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
if err := writeDirTar(ctx, tarFile, dstPath, srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1042,30 +1003,12 @@ func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) c
|
||||
if cr.id == "" {
|
||||
return cr.missingContainerError("copy to %s", dstPath)
|
||||
}
|
||||
logger := common.Logger(ctx)
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
for _, file := range files {
|
||||
logger.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
|
||||
hdr := &tar.Header{
|
||||
Name: file.Name,
|
||||
Mode: file.Mode,
|
||||
Size: int64(len(file.Body)),
|
||||
Uid: cr.UID,
|
||||
Gid: cr.GID,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(file.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
if err := writeFilesTar(ctx, &buf, cr.UID, cr.GID, files...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Extracting content to '%s'", dstPath)
|
||||
common.Logger(ctx).Debugf("Extracting content to '%s'", dstPath)
|
||||
_, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
|
||||
DestinationPath: dstPath,
|
||||
Content: &buf,
|
||||
@@ -1175,6 +1118,10 @@ func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *c
|
||||
// systempaths=unconfined lands in these two rather than in SecurityOpt
|
||||
resetOption(logger, "--security-opt", &hostConfig.MaskedPaths, trusted.MaskedPaths)
|
||||
resetOption(logger, "--security-opt", &hostConfig.ReadonlyPaths, trusted.ReadonlyPaths)
|
||||
resetOption(logger, "--log-driver", &hostConfig.LogConfig.Type, trusted.LogConfig.Type) // drivers run in the daemon, egressing from its network
|
||||
resetOption(logger, "--log-opt", &hostConfig.LogConfig.Config, trusted.LogConfig.Config)
|
||||
resetOption(logger, "--oom-score-adj", &hostConfig.OomScoreAdj, trusted.OomScoreAdj) // -1000 exempts it from the host OOM killer
|
||||
resetOption(logger, "--oom-kill-disable", &hostConfig.OomKillDisable, trusted.OomKillDisable)
|
||||
|
||||
// a driver mounts what it likes, e.g. local with device= binds any host path, which
|
||||
// valid_volumes never gets to see
|
||||
@@ -1216,9 +1163,9 @@ func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
|
||||
logger := common.Logger(ctx)
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
|
||||
if len(cr.input.ValidVolumes) > 0 {
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
// sanitize binds
|
||||
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
|
||||
for _, bind := range hostConfig.Binds {
|
||||
@@ -1232,33 +1179,37 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
continue
|
||||
}
|
||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type)) {
|
||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type), parsed.ReadOnly) {
|
||||
if matcher.needsNoCopy(parsed.Source, mount.Type(parsed.Type)) && (parsed.Volume == nil || !parsed.Volume.NoCopy) {
|
||||
bind += ",nocopy"
|
||||
}
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", parsed.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Binds = sanitizedBinds
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
if matcher.isValid(mt.Source, mt.Type) {
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
} else {
|
||||
for _, bind := range hostConfig.Binds {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", bind)
|
||||
}
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
hostConfig.Binds = []string{}
|
||||
}
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
readOnly := mt.ReadOnly && (mt.BindOptions == nil || !mt.BindOptions.ReadOnlyNonRecursive) // bind-recursive=writable leaves submounts writable
|
||||
if matcher.isValid(mt.Source, mt.Type, readOnly) {
|
||||
if matcher.needsNoCopy(mt.Source, mt.Type) {
|
||||
mt.VolumeOptions = cmp.Or(mt.VolumeOptions, &mount.VolumeOptions{})
|
||||
mt.VolumeOptions.NoCopy = true
|
||||
}
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
hostConfig.Binds = []string{}
|
||||
hostConfig.Mounts = []mount.Mount{}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
|
||||
return config, hostConfig
|
||||
}
|
||||
@@ -1298,26 +1249,37 @@ func overlayVolumes(dst, src *container.HostConfig) {
|
||||
}
|
||||
|
||||
type validVolumeMatcher struct {
|
||||
allowAll bool
|
||||
named []string
|
||||
host []string
|
||||
allowAll bool
|
||||
allowAllReadOnly bool
|
||||
named []validVolumeRule
|
||||
host []validVolumeRule
|
||||
}
|
||||
|
||||
type validVolumeRule struct {
|
||||
pattern string
|
||||
readOnly bool
|
||||
}
|
||||
|
||||
func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolumeMatcher {
|
||||
logger := common.Logger(ctx)
|
||||
ret := validVolumeMatcher{
|
||||
named: make([]string, 0, len(validVolumes)),
|
||||
host: make([]string, 0, len(validVolumes)),
|
||||
named: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
host: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
}
|
||||
|
||||
for _, v := range validVolumes {
|
||||
v, readOnly := strings.CutSuffix(v, ":ro")
|
||||
if v == "**" {
|
||||
ret.allowAll = true
|
||||
if readOnly {
|
||||
ret.allowAllReadOnly = true
|
||||
} else {
|
||||
ret.allowAll = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !isHostVolumePattern(v) {
|
||||
if doublestar.ValidatePattern(v) {
|
||||
ret.named = append(ret.named, v)
|
||||
ret.named = append(ret.named, validVolumeRule{pattern: v, readOnly: readOnly})
|
||||
} else {
|
||||
logger.Errorf("invalid volume pattern %s", v)
|
||||
}
|
||||
@@ -1329,7 +1291,7 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
|
||||
continue
|
||||
}
|
||||
if doublestar.ValidatePathPattern(normalized) {
|
||||
ret.host = append(ret.host, normalized)
|
||||
ret.host = append(ret.host, validVolumeRule{pattern: normalized, readOnly: readOnly})
|
||||
} else {
|
||||
logger.Errorf("invalid volume pattern %s", normalized)
|
||||
}
|
||||
@@ -1338,30 +1300,34 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
|
||||
return ret
|
||||
}
|
||||
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type) bool {
|
||||
if m.allowAll {
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type, readOnly bool) bool {
|
||||
if m.allowAll || (m.allowAllReadOnly && readOnly) {
|
||||
return true
|
||||
}
|
||||
if sourceType == mount.TypeTmpfs && source == "" { // backed by memory, exposes nothing from the host, same as --tmpfs
|
||||
return true
|
||||
}
|
||||
rules, match := m.named, doublestar.MatchUnvalidated
|
||||
if isHostVolumeSource(source, sourceType) {
|
||||
normalized, err := normalizeHostVolumePath(source)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, pattern := range m.host {
|
||||
if doublestar.PathMatchUnvalidated(pattern, normalized) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
rules, match, source = m.host, doublestar.PathMatchUnvalidated, normalized
|
||||
}
|
||||
for _, pattern := range m.named {
|
||||
if doublestar.MatchUnvalidated(pattern, source) {
|
||||
for _, rule := range rules {
|
||||
if (readOnly || !rule.readOnly) && match(rule.pattern, source) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// needsNoCopy reports a volume only `:ro` rules allow, as dockerd fills an empty volume from the image even when mounted read-only.
|
||||
func (m validVolumeMatcher) needsNoCopy(source string, sourceType mount.Type) bool {
|
||||
return sourceType == mount.TypeVolume && !m.isValid(source, sourceType, false)
|
||||
}
|
||||
|
||||
func isHostVolumePattern(pattern string) bool {
|
||||
return filepath.IsAbs(pattern) ||
|
||||
strings.HasPrefix(pattern, "."+string(filepath.Separator)) ||
|
||||
|
||||
@@ -137,6 +137,11 @@ func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.Netw
|
||||
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkCreate(ctx context.Context, name string, opts mobyclient.NetworkCreateOptions) (mobyclient.NetworkCreateResult, error) {
|
||||
args := m.Called(ctx, name, opts)
|
||||
return args.Get(0).(mobyclient.NetworkCreateResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
|
||||
@@ -519,10 +524,12 @@ var _ ExecutionsEnvironment = &containerReference{}
|
||||
|
||||
func TestCheckVolumes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
mounts []mount.Mount
|
||||
expectedMounts []mount.Mount
|
||||
}{
|
||||
{
|
||||
desc: "match all volumes",
|
||||
@@ -578,6 +585,27 @@ func TestCheckVolumes(t *testing.T) {
|
||||
"/opt/app/cache:/cache",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only rules allow only read-only mounts and copy nothing into volumes",
|
||||
validVolumes: []string{"/etc/ssl:ro", "data:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "/etc/ssl:/ssl", "data:/data:ro", "data:/data", "shared:/shared:ro"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "data:/data:ro,nocopy", "shared:/shared:ro"},
|
||||
mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true, BindOptions: &mount.BindOptions{ReadOnlyNonRecursive: true}},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true},
|
||||
},
|
||||
expectedMounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true, VolumeOptions: &mount.VolumeOptions{NoCopy: true}},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only wildcard keeps writable rules writable",
|
||||
validVolumes: []string{"**:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "data:/data", "shared:/shared"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "shared:/shared"},
|
||||
},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
@@ -588,9 +616,10 @@ func TestCheckVolumes(t *testing.T) {
|
||||
ValidVolumes: tc.validVolumes,
|
||||
},
|
||||
}
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds})
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts})
|
||||
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
||||
assert.Len(t, hook.AllEntries(), len(tc.binds)-len(tc.expectedBinds)) // every drop is warned about
|
||||
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
||||
assert.Len(t, hook.AllEntries(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts)) // every drop is warned about
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -666,6 +695,9 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
DeviceCgroupRules: []string{"a *:* rwm"},
|
||||
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}},
|
||||
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
|
||||
LogConfig: container.LogConfig{Type: "syslog", Config: map[string]string{"syslog-address": "tcp://169.254.169.254:514"}},
|
||||
OomScoreAdj: -1000,
|
||||
OomKillDisable: new(true),
|
||||
}
|
||||
|
||||
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{})
|
||||
@@ -699,7 +731,9 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
|
||||
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
|
||||
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
|
||||
"--security-opt apparmor=unconfined --volumes-from other --isolation process " +
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1 " +
|
||||
"--log-driver=syslog --log-opt syslog-address=tcp://169.254.169.254:514 " +
|
||||
"--oom-score-adj=-1000 --oom-kill-disable"
|
||||
|
||||
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's
|
||||
// own options alone describe it, field for field
|
||||
@@ -762,6 +796,22 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
}
|
||||
|
||||
func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
|
||||
hostConfig, _ := mergeOptions(t, "", "--mount type=tmpfs,dst=/var/lib/mysql", false)
|
||||
require.Len(t, hostConfig.Mounts, 1)
|
||||
|
||||
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
||||
logger, hook := test.NewNullLogger()
|
||||
_, sanitized := (&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{Mounts: []mount.Mount{
|
||||
hostConfig.Mounts[0],
|
||||
{Type: mount.TypeVolume, Target: "/anonymous"},
|
||||
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
||||
}})
|
||||
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
||||
assert.Len(t, hook.AllEntries(), 2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerInfoFromInspect(t *testing.T) {
|
||||
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
|
||||
info := containerInfoFromInspect(container.InspectResponse{
|
||||
|
||||
@@ -0,0 +1,576 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// KubernetesOptions configures the pod a job runs in.
|
||||
type KubernetesOptions struct {
|
||||
Kubeconfig string
|
||||
Namespace string
|
||||
PodTemplates []map[string]any // merged in order, a later one winning
|
||||
MaxLifetime time.Duration
|
||||
ForcePull bool
|
||||
RunnerUUID string
|
||||
}
|
||||
|
||||
// KubernetesPod runs a job container and its service containers as one pod.
|
||||
type KubernetesPod struct {
|
||||
options KubernetesOptions
|
||||
client *kubeClient
|
||||
name, secretName string
|
||||
job *kubernetesContainer
|
||||
services []*kubernetesContainer
|
||||
arch, path string
|
||||
}
|
||||
|
||||
type kubernetesContainer struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
pod *KubernetesPod
|
||||
id string // the workflow's service id, name is its DNS label form
|
||||
name string
|
||||
input *NewContainerInput
|
||||
probe map[string]any
|
||||
}
|
||||
|
||||
type containerStatus struct {
|
||||
Name string
|
||||
Ready bool
|
||||
State struct {
|
||||
Waiting *struct{ Reason, Message string }
|
||||
Running *struct{}
|
||||
Terminated *struct {
|
||||
ExitCode int
|
||||
Message string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type podStatus struct {
|
||||
Spec struct{ Containers []struct{} }
|
||||
Status struct {
|
||||
Phase, Reason, Message string
|
||||
Conditions []struct{ Type, Status, Message string }
|
||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||
}
|
||||
}
|
||||
|
||||
func NewKubernetesPod(options KubernetesOptions) (*KubernetesPod, error) {
|
||||
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to Kubernetes: %w", err)
|
||||
}
|
||||
return &KubernetesPod{options: options, client: client}, nil
|
||||
}
|
||||
|
||||
// JobContainer returns the container of the pod the job's steps run in.
|
||||
func (p *KubernetesPod) JobContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||
p.job = &kubernetesContainer{pod: p, name: "job", input: input}
|
||||
return p.job
|
||||
}
|
||||
|
||||
var invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
||||
|
||||
// KubernetesServiceName returns the container name and hostname of service id, made a DNS label as both must be.
|
||||
func KubernetesServiceName(id string) string {
|
||||
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
|
||||
return strings.TrimRight(name[:min(63, len(name))], "-")
|
||||
}
|
||||
|
||||
// ServiceContainer returns a container of the pod the job reaches at hostname KubernetesServiceName(id).
|
||||
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
|
||||
service := &kubernetesContainer{pod: p, id: id, name: KubernetesServiceName(id), input: input}
|
||||
p.services = append(p.services, service)
|
||||
return service
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) create(ctx context.Context) error {
|
||||
prefix := strings.Trim(strings.ToLower(p.job.input.Name), "-")
|
||||
prefix = cmp.Or(strings.TrimRight(prefix[:min(40, len(prefix))], "-"), "gitea-runner") + "-"
|
||||
name := prefix + strings.ToLower(rand.Text()[:8]) // chosen here, so cleanup knows it even when a create response is lost
|
||||
labels := map[string]any{"app.kubernetes.io/managed-by": "gitea-runner"}
|
||||
if p.options.RunnerUUID != "" {
|
||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||
}
|
||||
metadata := map[string]any{"name": name, "labels": labels}
|
||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||
var runner struct{ Metadata struct{ Name, UID string } }
|
||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||
}
|
||||
}
|
||||
secretData := map[string]string{}
|
||||
manifest, err := p.manifest(name, secretData, metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, service := range p.services {
|
||||
if !strings.EqualFold(service.id, service.name) {
|
||||
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
|
||||
}
|
||||
}
|
||||
if len(secretData) > 0 {
|
||||
p.secretName = name // named like the pod, RemoveOrphanKubernetesResources finds it by that
|
||||
if err := p.client.do(ctx, http.MethodPost, p.client.path("secrets"), map[string]any{"metadata": metadata, "stringData": secretData}, nil); err != nil {
|
||||
return fmt.Errorf("create secret: %w", err)
|
||||
}
|
||||
}
|
||||
p.name = name
|
||||
if err := p.client.do(ctx, http.MethodPost, p.client.path("pods"), manifest, nil); err != nil {
|
||||
return fmt.Errorf("create pod: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) manifest(secret string, secretData map[string]string, metadata map[string]any) (any, error) {
|
||||
template := map[string]any{"spec": map[string]any{
|
||||
"automountServiceAccountToken": false,
|
||||
"enableServiceLinks": false,
|
||||
"terminationGracePeriodSeconds": 0, // the job container's tail ignores SIGTERM
|
||||
}}
|
||||
for _, layer := range p.options.PodTemplates {
|
||||
template, _ = mergeManifest(template, layer, "").(map[string]any)
|
||||
}
|
||||
templateSpec, _ := template["spec"].(map[string]any)
|
||||
templateContainers, _ := templateSpec["containers"].([]any)
|
||||
templateVolumes, _ := templateSpec["volumes"].([]any)
|
||||
var serviceTemplate any
|
||||
if index := slices.IndexFunc(templateContainers, func(container any) bool { return manifestField(container, "name") == "$services" }); index >= 0 {
|
||||
serviceTemplate = templateContainers[index]
|
||||
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
|
||||
}
|
||||
var containers, hostnames, volumes []any
|
||||
names := map[string]bool{}
|
||||
for _, container := range templateContainers {
|
||||
names[manifestField(container, "name")] = true
|
||||
}
|
||||
for _, c := range append([]*kubernetesContainer{p.job}, p.services...) {
|
||||
if c != p.job && (c.name == "" || names[c.name]) {
|
||||
return nil, fmt.Errorf("service %s maps to container %q, which is empty or named like another container of the pod", c.id, c.name)
|
||||
}
|
||||
names[c.name] = true
|
||||
spec, err := c.spec(secret, secretData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == p.job {
|
||||
containers = append(containers, spec)
|
||||
continue
|
||||
}
|
||||
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
|
||||
hostnames = append(hostnames, c.name)
|
||||
}
|
||||
for _, name := range []string{"workspace", "act"} {
|
||||
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
|
||||
volumes = append(volumes, map[string]any{"name": name, "emptyDir": map[string]any{}})
|
||||
}
|
||||
}
|
||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||
if len(hostnames) > 0 {
|
||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||
}
|
||||
if p.options.MaxLifetime > 0 {
|
||||
spec["activeDeadlineSeconds"] = max(1, int64(p.options.MaxLifetime.Seconds()))
|
||||
}
|
||||
return mergeManifest(template, map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}, ""), nil
|
||||
}
|
||||
|
||||
// mergeManifest merges over onto base like a strategic merge patch: maps key by key, map list items by name or volumeMounts by mountPath, env entries, volumes and other values replaced.
|
||||
func mergeManifest(base, over any, field string) any {
|
||||
switch over := over.(type) {
|
||||
case map[string]any:
|
||||
merged := map[string]any{}
|
||||
if base, ok := base.(map[string]any); ok {
|
||||
maps.Copy(merged, base)
|
||||
}
|
||||
for key, value := range over {
|
||||
merged[key] = mergeManifest(merged[key], value, key)
|
||||
}
|
||||
return merged
|
||||
case []any:
|
||||
key := cmp.Or(map[string]string{"volumeMounts": "mountPath"}[field], "name")
|
||||
baseList, _ := base.([]any)
|
||||
merged := slices.Clone(baseList)
|
||||
for _, item := range over {
|
||||
if _, ok := item.(map[string]any); !ok {
|
||||
return over
|
||||
}
|
||||
name := manifestField(item, key)
|
||||
index := slices.IndexFunc(merged, func(existing any) bool { return name != "" && manifestField(existing, key) == name })
|
||||
switch {
|
||||
case index < 0:
|
||||
merged = append(merged, item)
|
||||
case field == "env" || field == "volumes": // each holds one source, merging two makes a pod the API rejects
|
||||
merged[index] = item
|
||||
default:
|
||||
merged[index] = mergeManifest(merged[index], item, "")
|
||||
}
|
||||
}
|
||||
return merged
|
||||
}
|
||||
return over
|
||||
}
|
||||
|
||||
func manifestField(item any, key string) string {
|
||||
fields, _ := item.(map[string]any)
|
||||
value, _ := fields[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) spec(secret string, secretData map[string]string) (map[string]any, error) {
|
||||
env := []any{}
|
||||
for index, entry := range c.input.Env {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
if c == c.pod.job {
|
||||
env = append(env, map[string]any{"name": key, "value": value})
|
||||
continue
|
||||
}
|
||||
secretKey := fmt.Sprintf("%s.%d", c.name, index)
|
||||
secretData[secretKey] = value
|
||||
env = append(env, map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": secret, "key": secretKey}}})
|
||||
}
|
||||
spec := map[string]any{
|
||||
"name": c.name, "image": c.input.Image, "command": escapeExpansion(c.input.Entrypoint), "args": escapeExpansion(c.input.Cmd),
|
||||
"workingDir": c.input.WorkingDir, "env": env,
|
||||
}
|
||||
if c.pod.options.ForcePull {
|
||||
spec["imagePullPolicy"] = "Always"
|
||||
}
|
||||
if c == c.pod.job {
|
||||
spec["volumeMounts"] = []any{
|
||||
map[string]any{"name": "workspace", "mountPath": c.input.WorkingDir},
|
||||
map[string]any{"name": "act", "mountPath": c.GetActPath()},
|
||||
}
|
||||
}
|
||||
if err := c.parseOptions(); err != nil {
|
||||
return nil, fmt.Errorf("service %s: %w", c.name, err)
|
||||
}
|
||||
if c.probe != nil {
|
||||
spec["startupProbe"] = c.probe
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
// escapeExpansion keeps kubelet from expanding $(VAR) and $$ in command and args, which docker passes as is.
|
||||
func escapeExpansion(args []string) []string {
|
||||
args = slices.Clone(args)
|
||||
for index := range args {
|
||||
args[index] = strings.ReplaceAll(args[index], "$", "$$")
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// parseOptions turns docker health options, the only ones a pod supports, into a startup probe, which like docker gives up after the retries.
|
||||
func (c *kubernetesContainer) parseOptions() error {
|
||||
args, err := shellquote.Split(c.input.WorkflowOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flags := pflag.NewFlagSet("options", pflag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
command := flags.String("health-cmd", "", "")
|
||||
interval := flags.Duration("health-interval", 0, "")
|
||||
timeout := flags.Duration("health-timeout", 0, "")
|
||||
retries := flags.Int("health-retries", 0, "")
|
||||
startPeriod := flags.Duration("health-start-period", 0, "")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return fmt.Errorf("options %q: %w, only --health-* options are supported on Kubernetes", c.input.WorkflowOptions, err)
|
||||
}
|
||||
if *command == "" {
|
||||
return nil
|
||||
}
|
||||
period := max(1, int(cmp.Or(*interval, 30*time.Second).Seconds()))
|
||||
c.probe = map[string]any{
|
||||
"exec": map[string]any{"command": []string{"/bin/sh", "-c", *command}},
|
||||
"periodSeconds": period,
|
||||
"timeoutSeconds": max(1, int(cmp.Or(*timeout, 30*time.Second).Seconds())),
|
||||
"failureThreshold": max(1, cmp.Or(*retries, 3)) + int(startPeriod.Seconds())/period, // docker does not count failures in the start period
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) get(ctx context.Context) (*podStatus, error) {
|
||||
var pod podStatus
|
||||
return &pod, p.client.do(ctx, http.MethodGet, p.client.path("pods/"+p.name), nil, &pod)
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
var unscheduled string
|
||||
for delay := 100 * time.Millisecond; ; delay = min(2*delay, time.Second) {
|
||||
pod, err := p.get(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if pod.Status.Phase == "Failed" {
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
||||
}
|
||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
if waiting := status.State.Waiting; waiting != nil {
|
||||
switch waiting.Reason {
|
||||
case "ErrImagePull", "ImagePullBackOff", "InvalidImageName", "ErrImageNeverPull", "CreateContainerConfigError", "CreateContainerError", "RunContainerError":
|
||||
return fmt.Errorf("container %s of pod %s: %s: %s", status.Name, p.name, waiting.Reason, waiting.Message)
|
||||
}
|
||||
started = false
|
||||
}
|
||||
}
|
||||
if started {
|
||||
return nil
|
||||
}
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
if condition.Type == "PodScheduled" && condition.Status == "False" && condition.Message != unscheduled {
|
||||
unscheduled = condition.Message
|
||||
common.Logger(ctx).Infof("Waiting for pod %s to be scheduled: %s", p.name, unscheduled)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Create(_, _ []string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
return c.pod.create(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
if err := c.pod.waitStarted(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard); err != nil {
|
||||
return err
|
||||
}
|
||||
machine, path, _ := strings.Cut(stdout.String(), "\n")
|
||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (*kubernetesContainer) Pull(bool) common.Executor {
|
||||
return func(context.Context) error { return nil }
|
||||
}
|
||||
|
||||
var (
|
||||
shellName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
bashReadonly = []string{"BASHOPTS", "BASH_VERSINFO", "EUID", "PPID", "SHELLOPTS", "UID"} // bash as sh cannot export these
|
||||
)
|
||||
|
||||
// Exec feeds the command to sh on stdin, which exports the env so it stays out of the request URL and, where sh can export a name, out of process arguments.
|
||||
func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
script, args := "", []string{"env", "--"}
|
||||
for name, value := range env {
|
||||
if shellName.MatchString(name) && !slices.Contains(bashReadonly, name) {
|
||||
script += "command export " + shellquote.Join(name+"="+value) + "\n"
|
||||
} else {
|
||||
args = append(args, name+"="+value)
|
||||
}
|
||||
}
|
||||
script += "exec " + shellquote.Join(append(args, command...)...)
|
||||
if workdir != "" {
|
||||
script = "cd " + shellquote.Join(workdir) + " || exit\n" + script
|
||||
}
|
||||
defer common.FlushWriter(c.input.Stdout)
|
||||
defer common.FlushWriter(c.input.Stderr)
|
||||
return c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh"}, strings.NewReader(script), c.input.Stdout, c.input.Stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
go func() { writer.CloseWithError(write(writer)) }()
|
||||
var stderr bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
|
||||
func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
writer.CloseWithError(err)
|
||||
}()
|
||||
buffered := bufio.NewReader(reader)
|
||||
if _, err := buffered.Peek(1); err != nil {
|
||||
reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{buffered, reader}, nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
return parseEnvFile(c, srcPath, env)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
return func(context.Context) error {
|
||||
(*env)["PATH"] = cmp.Or((*env)["PATH"], c.pod.path)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Inspect(ctx context.Context) (*Info, error) {
|
||||
if c.pod.name == "" {
|
||||
return nil, fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
|
||||
}
|
||||
pod, err := c.pod.get(ctx)
|
||||
if isNotFound(err) {
|
||||
return nil, fmt.Errorf("pod %s %w", c.pod.name, ErrContainerNotFound)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info := &Info{ID: c.pod.name + "/" + c.name, State: "created", Health: HealthNone, Ports: map[string]string{}}
|
||||
for port := range c.input.ExposedPorts {
|
||||
info.Ports[port.Port()] = port.Port() // services share the pod's network
|
||||
}
|
||||
for _, status := range pod.Status.ContainerStatuses {
|
||||
if status.Name != c.name {
|
||||
continue
|
||||
}
|
||||
if status.State.Running != nil {
|
||||
info.State = StateRunning
|
||||
} else if terminated := status.State.Terminated; terminated != nil {
|
||||
info.State, info.ExitCode, info.HealthOutput = "exited", terminated.ExitCode, terminated.Message
|
||||
}
|
||||
if c.probe != nil {
|
||||
switch {
|
||||
case status.Ready:
|
||||
info.Health = HealthHealthy
|
||||
case info.State == "exited":
|
||||
info.Health = HealthUnhealthy
|
||||
default:
|
||||
info.Health = HealthStarting
|
||||
}
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) DumpLogs(ctx context.Context) error {
|
||||
if c.pod.name == "" {
|
||||
return fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
|
||||
}
|
||||
defer common.FlushWriter(c.input.Stdout)
|
||||
return c.pod.client.do(ctx, http.MethodGet, c.pod.client.path("pods/"+c.pod.name+"/log?container="+c.name), nil, c.input.Stdout)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
return errors.Join(c.pod.client.delete(ctx, "pods", c.pod.name), c.pod.client.delete(ctx, "secrets", c.pod.secretName))
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveOrphanKubernetesResources removes this runner's job pods created before createdBefore and the Secrets named like them, as listing Secrets would reveal all of the namespace's.
|
||||
func RemoveOrphanKubernetesResources(ctx context.Context, options KubernetesOptions, createdBefore time.Time) error {
|
||||
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect to Kubernetes: %w", err)
|
||||
}
|
||||
defer client.http.CloseIdleConnections()
|
||||
var pods struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string
|
||||
CreationTimestamp time.Time
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := client.do(ctx, http.MethodGet, client.path("pods?labelSelector="+url.QueryEscape(runnerUUIDLabel+"="+options.RunnerUUID)), nil, &pods); err != nil {
|
||||
return fmt.Errorf("list pods: %w", err)
|
||||
}
|
||||
var errs []error
|
||||
for _, item := range pods.Items {
|
||||
if !item.Metadata.CreationTimestamp.Before(createdBefore) {
|
||||
continue
|
||||
}
|
||||
if err := client.delete(ctx, "secrets", item.Metadata.Name); err != nil {
|
||||
errs = append(errs, err) // leave the pod for the next cleanup, it is the only way back to its Secret
|
||||
continue
|
||||
}
|
||||
errs = append(errs, client.delete(ctx, "pods", item.Metadata.Name))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Close() common.Executor {
|
||||
return func(context.Context) error {
|
||||
c.pod.client.http.CloseIdleConnections()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
|
||||
oldStdout, oldStderr := c.input.Stdout, c.input.Stderr
|
||||
c.input.Stdout, c.input.Stderr = stdout, stderr
|
||||
return oldStdout, oldStderr
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) GetRunnerContext(_ context.Context) map[string]any {
|
||||
return map[string]any{"os": "Linux", "arch": c.pod.arch, "temp": "/tmp", "tool_cache": DefaultToolCache}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
serviceAccountDir = "/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
execProtocol = "v5.channel.k8s.io"
|
||||
)
|
||||
|
||||
type kubeClient struct {
|
||||
server, namespace, token string
|
||||
tokenFile string // re-read on every request, service account tokens rotate
|
||||
inCluster bool
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
code int
|
||||
message string
|
||||
}
|
||||
|
||||
func (e *apiError) Error() string { return e.message }
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var apiErr *apiError
|
||||
return errors.As(err, &apiErr) && apiErr.code == http.StatusNotFound
|
||||
}
|
||||
|
||||
func newKubeClient(kubeconfig, namespace string) (*kubeClient, error) {
|
||||
client := &kubeClient{namespace: namespace}
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}
|
||||
if host := os.Getenv("KUBERNETES_SERVICE_HOST"); host != "" && kubeconfig == "" {
|
||||
if err := client.loadInCluster(host, transport.TLSClientConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err := client.loadKubeconfig(kubeconfig, transport); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.http = &http.Client{Transport: transport}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) loadInCluster(host string, tlsConfig *tls.Config) error {
|
||||
c.server = "https://" + net.JoinHostPort(host, os.Getenv("KUBERNETES_SERVICE_PORT"))
|
||||
c.tokenFile = serviceAccountDir + "/token"
|
||||
ca, err := os.ReadFile(serviceAccountDir + "/ca.crt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tlsConfig.RootCAs = x509.NewCertPool()
|
||||
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
|
||||
if c.namespace == "" {
|
||||
namespace, err := os.ReadFile(serviceAccountDir + "/namespace")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.namespace = strings.TrimSpace(string(namespace))
|
||||
}
|
||||
c.inCluster = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) loadKubeconfig(path string, transport *http.Transport) error {
|
||||
if paths := filepath.SplitList(os.Getenv("KUBECONFIG")); path == "" && len(paths) > 0 {
|
||||
path = paths[0]
|
||||
}
|
||||
if path == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path = filepath.Join(home, ".kube", "config")
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("no in-cluster service account and no kubeconfig: %w", err)
|
||||
}
|
||||
var config struct {
|
||||
CurrentContext string `yaml:"current-context"`
|
||||
Contexts []struct {
|
||||
Name string
|
||||
Context struct{ Cluster, User, Namespace string }
|
||||
}
|
||||
Clusters []struct {
|
||||
Name string
|
||||
Cluster struct {
|
||||
Server string
|
||||
TLSServerName string `yaml:"tls-server-name"`
|
||||
InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify"`
|
||||
ProxyURL string `yaml:"proxy-url"`
|
||||
CertificateAuthority string `yaml:"certificate-authority"`
|
||||
CertificateAuthorityData string `yaml:"certificate-authority-data"`
|
||||
}
|
||||
}
|
||||
Users []struct {
|
||||
Name string
|
||||
User struct {
|
||||
Token string
|
||||
TokenFile string `yaml:"tokenFile"`
|
||||
ClientCertificate string `yaml:"client-certificate"`
|
||||
ClientCertificateData string `yaml:"client-certificate-data"`
|
||||
ClientKey string `yaml:"client-key"`
|
||||
ClientKeyData string `yaml:"client-key-data"`
|
||||
Exec any
|
||||
AuthProvider any `yaml:"auth-provider"`
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := yaml.Unmarshal(content, &config); err != nil {
|
||||
return fmt.Errorf("parse kubeconfig %s: %w", path, err)
|
||||
}
|
||||
var current struct{ Cluster, User, Namespace string }
|
||||
for _, entry := range config.Contexts {
|
||||
if entry.Name == config.CurrentContext {
|
||||
current = entry.Context
|
||||
}
|
||||
}
|
||||
if current.Cluster == "" {
|
||||
return fmt.Errorf("kubeconfig %s has no current context", path)
|
||||
}
|
||||
c.namespace = cmp.Or(c.namespace, current.Namespace, "default")
|
||||
|
||||
resolve := func(file string) string {
|
||||
if file == "" || filepath.IsAbs(file) {
|
||||
return file
|
||||
}
|
||||
return filepath.Join(filepath.Dir(path), file)
|
||||
}
|
||||
read := func(data, file string) ([]byte, error) {
|
||||
if data != "" {
|
||||
return base64.StdEncoding.DecodeString(data)
|
||||
}
|
||||
if file == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return os.ReadFile(resolve(file))
|
||||
}
|
||||
|
||||
tlsConfig := transport.TLSClientConfig
|
||||
for _, cluster := range config.Clusters {
|
||||
if cluster.Name != current.Cluster {
|
||||
continue
|
||||
}
|
||||
c.server = strings.TrimSuffix(cluster.Cluster.Server, "/")
|
||||
tlsConfig.ServerName, tlsConfig.InsecureSkipVerify = cluster.Cluster.TLSServerName, cluster.Cluster.InsecureSkipTLSVerify
|
||||
if cluster.Cluster.ProxyURL != "" {
|
||||
proxy, err := url.Parse(cluster.Cluster.ProxyURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("kubeconfig %s: %w", path, err)
|
||||
}
|
||||
transport.Proxy = http.ProxyURL(proxy)
|
||||
}
|
||||
ca, err := read(cluster.Cluster.CertificateAuthorityData, cluster.Cluster.CertificateAuthority)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ca != nil {
|
||||
tlsConfig.RootCAs = x509.NewCertPool()
|
||||
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
|
||||
}
|
||||
}
|
||||
for _, user := range config.Users {
|
||||
if user.Name != current.User {
|
||||
continue
|
||||
}
|
||||
if user.User.Exec != nil || user.User.AuthProvider != nil {
|
||||
return fmt.Errorf("kubeconfig %s: user %q authenticates through a credential plugin, which the runner does not support, use a token or client certificate", path, user.Name)
|
||||
}
|
||||
c.token, c.tokenFile = user.User.Token, resolve(user.User.TokenFile)
|
||||
cert, err := read(user.User.ClientCertificateData, user.User.ClientCertificate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := read(user.User.ClientKeyData, user.User.ClientKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cert != nil {
|
||||
pair, err := tls.X509KeyPair(cert, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tlsConfig.Certificates = []tls.Certificate{pair}
|
||||
}
|
||||
}
|
||||
if c.server == "" {
|
||||
return fmt.Errorf("kubeconfig %s names no server for context %q", path, config.CurrentContext)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) authorize(header http.Header) error {
|
||||
token := c.token
|
||||
if c.tokenFile != "" {
|
||||
content, err := os.ReadFile(c.tokenFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token = strings.TrimSpace(string(content))
|
||||
}
|
||||
if token != "" {
|
||||
header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) path(resource string) string {
|
||||
return "/api/v1/namespaces/" + c.namespace + "/" + resource
|
||||
}
|
||||
|
||||
func (c *kubeClient) do(ctx context.Context, method, path string, in, out any) error {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
content, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(content)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.server+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.authorize(req.Header); err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return statusError(resp)
|
||||
}
|
||||
if out == nil {
|
||||
out = io.Discard // draining lets the connection be reused
|
||||
}
|
||||
if writer, ok := out.(io.Writer); ok {
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return err
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
func (c *kubeClient) delete(ctx context.Context, resource, name string) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if err := c.do(ctx, http.MethodDelete, c.path(resource+"/"+name), nil, nil); err != nil && !isNotFound(err) {
|
||||
return fmt.Errorf("delete %s %s: %w", resource, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func statusError(resp *http.Response) error {
|
||||
content, _ := io.ReadAll(resp.Body)
|
||||
var status struct{ Message string }
|
||||
if json.Unmarshal(content, &status) != nil || status.Message == "" {
|
||||
status.Message = strings.TrimSpace(string(content))
|
||||
}
|
||||
return &apiError{code: resp.StatusCode, message: status.Message}
|
||||
}
|
||||
|
||||
func (c *kubeClient) exec(ctx context.Context, pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
query := url.Values{"container": {containerName}, "command": command, "stdout": {"true"}, "stderr": {"true"}}
|
||||
if stdin != nil {
|
||||
query.Set("stdin", "true")
|
||||
}
|
||||
header := http.Header{}
|
||||
if err := c.authorize(header); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
conn, resp, err := websocket.Dial(ctx, c.server+c.path("pods/"+pod+"/exec?")+query.Encode(), &websocket.DialOptions{
|
||||
HTTPClient: c.http,
|
||||
HTTPHeader: header,
|
||||
Subprotocols: []string{execProtocol},
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode >= http.StatusMultipleChoices {
|
||||
defer resp.Body.Close()
|
||||
err = statusError(resp)
|
||||
}
|
||||
return fmt.Errorf("exec in pod %s: %w", pod, err)
|
||||
}
|
||||
defer func() { _ = conn.CloseNow() }()
|
||||
if conn.Subprotocol() != execProtocol {
|
||||
return fmt.Errorf("the Kubernetes API server does not speak %s, which needs Kubernetes 1.30 or later", execProtocol)
|
||||
}
|
||||
conn.SetReadLimit(-1)
|
||||
|
||||
if stdin != nil {
|
||||
go func() {
|
||||
if err := sendStdin(ctx, conn, stdin); err != nil {
|
||||
cancel(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if conn.Ping(ctx) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
_, frame, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
return cmp.Or(context.Cause(ctx), fmt.Errorf("exec in pod %s ended without an exit status: %w", pod, err))
|
||||
}
|
||||
if len(frame) < 2 {
|
||||
continue
|
||||
}
|
||||
switch frame[0] {
|
||||
case 1:
|
||||
_, err = stdout.Write(frame[1:])
|
||||
case 2:
|
||||
_, err = stderr.Write(frame[1:])
|
||||
case 3:
|
||||
return cmp.Or(context.Cause(ctx), execStatus(frame[1:]))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendStdin(ctx context.Context, conn *websocket.Conn, stdin io.Reader) error {
|
||||
frame := make([]byte, 1+32*1024) // frame[0] is the stdin channel, 0
|
||||
for {
|
||||
n, err := stdin.Read(frame[1:])
|
||||
if n > 0 {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, frame[:1+n]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return conn.Write(ctx, websocket.MessageBinary, []byte{255, 0}) // v5 close signal for stdin
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func execStatus(content []byte) error {
|
||||
var status struct {
|
||||
Status, Message string
|
||||
Details struct {
|
||||
Causes []struct{ Reason, Message string }
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal(content, &status); err != nil {
|
||||
return fmt.Errorf("decode exec status: %w", err)
|
||||
}
|
||||
if status.Status == "Success" {
|
||||
return nil
|
||||
}
|
||||
for _, cause := range status.Details.Causes {
|
||||
if code, err := strconv.Atoi(cause.Message); cause.Reason == "ExitCode" && err == nil {
|
||||
return ExitCodeError(code)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("exec failed: %s", status.Message)
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type podManifest struct {
|
||||
Metadata struct{ Labels map[string]string }
|
||||
Spec struct {
|
||||
AutomountServiceAccountToken bool
|
||||
EnableServiceLinks bool
|
||||
HostAliases []struct{ Hostnames []string }
|
||||
Volumes []map[string]any
|
||||
Containers []struct {
|
||||
Name, ImagePullPolicy string
|
||||
Resources map[string]any
|
||||
StartupProbe *struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeCluster struct {
|
||||
mu sync.Mutex
|
||||
requests map[string][]string
|
||||
}
|
||||
|
||||
func (f *fakeCluster) record(request, content string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.requests[request] = append(f.requests[request], content)
|
||||
}
|
||||
|
||||
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
||||
cluster := &fakeCluster{requests: map[string][]string{}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
cluster.record(r.Method+" "+r.URL.Path, string(body))
|
||||
pods, secrets := "/api/v1/namespaces/ci/pods", "/api/v1/namespaces/ci/secrets"
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == pods:
|
||||
cluster.record("list", r.URL.Query().Get("labelSelector"))
|
||||
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
|
||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
||||
cluster.record("exec", r.URL.RawQuery)
|
||||
serveLocalExec(t, w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/log"):
|
||||
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
|
||||
_, _ = io.WriteString(w, podStatus)
|
||||
case r.Method == http.MethodPost && (r.URL.Path == pods || r.URL.Path == secrets), r.Method == http.MethodDelete:
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(kubeconfig), "token"), []byte("test-token\n"), 0o600))
|
||||
require.NoError(t, os.WriteFile(kubeconfig, fmt.Appendf(nil, `
|
||||
current-context: test
|
||||
contexts: [{name: test, context: {cluster: test, user: test, namespace: ci}}]
|
||||
clusters: [{name: test, cluster: {server: "http://kubernetes.invalid", proxy-url: %q}}]
|
||||
users: [{name: test, user: {tokenFile: token}}]
|
||||
`, server.URL), 0o600))
|
||||
return kubeconfig, cluster
|
||||
}
|
||||
|
||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.CloseNow() }()
|
||||
conn.SetReadLimit(-1)
|
||||
command := r.URL.Query()["command"]
|
||||
var stdin bytes.Buffer
|
||||
for r.URL.Query().Get("stdin") == "true" {
|
||||
_, frame, err := conn.Read(r.Context())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if frame[0] == 255 {
|
||||
break
|
||||
}
|
||||
stdin.Write(frame[1:])
|
||||
}
|
||||
cmd := exec.Command(command[0], command[1:]...)
|
||||
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
||||
status := `{"status":"Success"}`
|
||||
var exitErr *exec.ExitError
|
||||
if err := cmd.Run(); errors.As(err, &exitErr) {
|
||||
status = fmt.Sprintf(`{"status":"Failure","reason":"NonZeroExitCode","details":{"causes":[{"reason":"ExitCode","message":"%d"}]}}`, exitErr.ExitCode())
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
for _, frame := range [][]byte{append([]byte{1}, stdout.Bytes()...), append([]byte{2}, stderr.Bytes()...), append([]byte{3}, status...)} {
|
||||
_ = conn.Write(r.Context(), websocket.MessageBinary, frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}},
|
||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
||||
{"spec": map[string]any{
|
||||
"automountServiceAccountToken": true,
|
||||
"containers": []any{
|
||||
map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"cpu": "2"}}},
|
||||
map[string]any{
|
||||
"name": "$services", "imagePullPolicy": "Always", "resources": map[string]any{"limits": map[string]any{"memory": "256Mi"}},
|
||||
"env": []any{map[string]any{"name": "POSTGRES_PASSWORD", "value": "template"}},
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"spec": map[string]any{
|
||||
"volumes": []any{map[string]any{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}}},
|
||||
"containers": []any{map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}}},
|
||||
}},
|
||||
}}
|
||||
pod, err := NewKubernetesPod(options)
|
||||
require.NoError(t, err)
|
||||
var jobLog, serviceLog bytes.Buffer
|
||||
job := pod.JobContainer(&NewContainerInput{Name: "Job-Name", Image: "node", WorkingDir: workdir, Stdout: &jobLog, Stderr: &jobLog})
|
||||
service := pod.ServiceContainer("My_DB", &NewContainerInput{
|
||||
Image: "postgres",
|
||||
Cmd: []string{"postgres", "-D", "$(PGDATA)"},
|
||||
Env: []string{"POSTGRES_PASSWORD=service-secret"},
|
||||
WorkflowOptions: "--health-cmd pg_isready --health-interval 2s --health-start-period 4s",
|
||||
Stdout: &serviceLog,
|
||||
})
|
||||
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
var manifest podManifest
|
||||
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||
assert.Equal(t, "gitea-runner", manifest.Metadata.Labels["app.kubernetes.io/managed-by"])
|
||||
assert.Equal(t, "runner-uuid", manifest.Metadata.Labels[runnerUUIDLabel])
|
||||
assert.True(t, manifest.Spec.AutomountServiceAccountToken)
|
||||
assert.False(t, manifest.Spec.EnableServiceLinks)
|
||||
assert.Equal(t, []string{"my-db"}, manifest.Spec.HostAliases[0].Hostnames)
|
||||
require.Len(t, manifest.Spec.Containers, 2)
|
||||
assert.Equal(t, map[string]any{"limits": map[string]any{"cpu": "2", "memory": "1Gi"}}, manifest.Spec.Containers[0].Resources)
|
||||
assert.Empty(t, manifest.Spec.Containers[0].ImagePullPolicy)
|
||||
assert.Equal(t, "my-db", manifest.Spec.Containers[1].Name)
|
||||
assert.Equal(t, "Always", manifest.Spec.Containers[1].ImagePullPolicy)
|
||||
assert.Equal(t, map[string]any{"limits": map[string]any{"memory": "256Mi"}}, manifest.Spec.Containers[1].Resources)
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}},
|
||||
{"name": "act", "emptyDir": map[string]any{}},
|
||||
}, manifest.Spec.Volumes)
|
||||
assert.Equal(t, struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }{2, 30, 5}, *manifest.Spec.Containers[1].StartupProbe)
|
||||
assert.NotContains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], "service-secret")
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], "service-secret")
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], `"com.gitea.runner.uuid":"runner-uuid"`)
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `{"name":"POSTGRES_PASSWORD","valueFrom":{"secretKeyRef":{"key":"my-db.0","name":"`+pod.secretName+`"}}}`)
|
||||
assert.Equal(t, []any{"new"}, mergeManifest([]any{"old"}, []any{"new"}, ""))
|
||||
mount := func(path string) any { return map[string]any{"name": "config", "mountPath": path} }
|
||||
assert.Equal(t, []any{mount("/a"), mount("/b")}, mergeManifest([]any{mount("/a")}, []any{mount("/b")}, "volumeMounts"))
|
||||
tmpfs := map[string]any{"name": "workspace", "emptyDir": map[string]any{"medium": "Memory"}}
|
||||
ephemeral := map[string]any{"name": "workspace", "ephemeral": map[string]any{}}
|
||||
assert.Equal(t, []any{ephemeral}, mergeManifest([]any{tmpfs}, []any{ephemeral}, "volumes"))
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
||||
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
|
||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||
envArgs, err := os.ReadFile(filepath.Join(workdir, "env-args"))
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(envArgs), "hunter2")
|
||||
require.NoError(t, job.Exec([]string{"printenv", "INPUT_WHO-TO-GREET"}, stepEnv, "", "")(ctx))
|
||||
require.NoError(t, job.Exec([]string{"printenv", "UID"}, stepEnv, "", "")(ctx))
|
||||
require.NoError(t, job.Exec([]string{"pwd", "-P"}, nil, "", workdir)(ctx))
|
||||
assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
|
||||
assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
|
||||
assert.Contains(t, jobLog.String(), "failing\n")
|
||||
for _, query := range cluster.requests["exec"] {
|
||||
assert.NotContains(t, query, "hunter2")
|
||||
}
|
||||
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||
require.NoError(t, err)
|
||||
reader := tar.NewReader(archive)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "event.json", header.Name)
|
||||
assert.JSONEq(t, `{"ref":"main"}`, string(content))
|
||||
require.NoError(t, archive.Close())
|
||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||
require.Error(t, err)
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
||||
|
||||
info, err := service.Inspect(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, HealthHealthy, info.Health)
|
||||
require.NoError(t, service.DumpLogs(ctx))
|
||||
assert.Equal(t, "log of my-db", serviceLog.String())
|
||||
|
||||
require.NoError(t, job.Remove()(ctx))
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/"+pod.name], 1)
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/"+pod.secretName], 1)
|
||||
|
||||
require.NoError(t, RemoveOrphanKubernetesResources(ctx, options, time.Now()))
|
||||
assert.Equal(t, []string{runnerUUIDLabel + "=runner-uuid"}, cluster.requests["list"])
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/old"], 1)
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/old"], 1)
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/new"])
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||
}
|
||||
|
||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||
template := map[string]any{"spec": map[string]any{"containers": []any{map[string]any{"name": "sidecar"}}}}
|
||||
for _, test := range []struct {
|
||||
services []string
|
||||
options, err string
|
||||
}{
|
||||
{[]string{"db"}, "--hostname db", "only --health-* options are supported"},
|
||||
{[]string{"sidecar"}, "", "named like another container"},
|
||||
{[]string{"my_db", "My-DB"}, "", `service My-DB maps to container "my-db"`},
|
||||
} {
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci", PodTemplates: []map[string]any{template}})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
for _, service := range test.services {
|
||||
pod.ServiceContainer(service, &NewContainerInput{Image: "postgres", WorkflowOptions: test.options})
|
||||
}
|
||||
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
|
||||
}
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||
|
||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||
current-context: eks
|
||||
contexts: [{name: eks, context: {cluster: eks, user: eks}}]
|
||||
clusters: [{name: eks, cluster: {server: "https://eks.example"}}]
|
||||
users: [{name: eks, user: {exec: {command: aws}}}]
|
||||
`), 0o600))
|
||||
_, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: plugin})
|
||||
require.ErrorContains(t, err, "credential plugin")
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
|
||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
)
|
||||
|
||||
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
||||
tw := tar.NewWriter(w)
|
||||
for _, file := range files {
|
||||
common.Logger(ctx).Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: file.Name,
|
||||
Mode: file.Mode,
|
||||
Size: int64(len(file.Body)),
|
||||
Uid: uid,
|
||||
Gid: gid,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(file.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tw.Close()
|
||||
}
|
||||
|
||||
// writeDirTar prefixes entries with dstPath for extraction at root.
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
|
||||
logger := common.Logger(ctx)
|
||||
tw := tar.NewWriter(w)
|
||||
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||
|
||||
var ignorer gitignore.Matcher
|
||||
if useGitIgnore {
|
||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
||||
if err != nil {
|
||||
logger.Debugf("Error loading .gitignore: %v", err)
|
||||
}
|
||||
|
||||
ignorer = gitignore.NewMatcher(ps)
|
||||
}
|
||||
|
||||
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
|
||||
Ignorer: ignorer,
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
||||
}).CollectFiles(ctx, []string{})); err != nil {
|
||||
return err
|
||||
}
|
||||
return tw.Close()
|
||||
}
|
||||
+24
-1
@@ -6,6 +6,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
@@ -287,11 +288,30 @@ func dockerActionImageTag(repository, actionName string, localAction bool) strin
|
||||
return strings.ToLower(image)
|
||||
}
|
||||
|
||||
// One image per step and job: its stages share a build, and concurrent jobs never run each other's.
|
||||
func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, actionName string, localAction bool) string {
|
||||
job := rc.topLevelRunContext()
|
||||
if job.dockerActionImages == nil {
|
||||
job.dockerActionImages = map[*model.Step]string{}
|
||||
}
|
||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||
if !ok {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||
job.dockerActionImages[step.getStepModel()] = image
|
||||
}
|
||||
}
|
||||
return image
|
||||
}
|
||||
|
||||
// TODO: break out parts of function to reduce complexicity
|
||||
func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, basedir string, localAction bool, stage stepStage) error {
|
||||
logger := common.Logger(ctx)
|
||||
rc := step.getRunContext()
|
||||
action := step.getActionModel()
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
}
|
||||
|
||||
var prepImage common.Executor
|
||||
var image string
|
||||
@@ -301,7 +321,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
// Apply forcePull only for prebuild docker images
|
||||
forcePull = rc.Config.ForcePull
|
||||
} else {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction)
|
||||
image = rc.dockerActionImage(ctx, step, actionName, localAction)
|
||||
contextDir, fileName := filepath.Split(filepath.Join(basedir, action.Runs.Image))
|
||||
|
||||
anyArchExists, err := ContainerImageExistsLocally(ctx, image, "any")
|
||||
@@ -442,6 +462,9 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
|
||||
logWriter := rc.commandLogWriter(ctx)
|
||||
envList := make([]string, 0)
|
||||
for k, v := range *step.getEnv() {
|
||||
if rc.IsHostEnv() && strings.EqualFold(k, rc.JobContainer.GetPathVariableName()) && v == rc.JobContainer.DefaultPathVariable() { // the runner's own PATH stays out, as on GitHub
|
||||
continue
|
||||
}
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
|
||||
}
|
||||
}
|
||||
|
||||
ee := parent.NewActionInputsExpressionEvaluator(ctx, step)
|
||||
ee := parent.NewStepExpressionEvaluator(ctx, step)
|
||||
|
||||
for inputID, input := range step.getActionModel().Inputs {
|
||||
envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
@@ -104,6 +104,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action
|
||||
Parent: parent,
|
||||
EventJSON: parent.EventJSON,
|
||||
platformImage: parent.platformImage,
|
||||
kubernetes: parent.kubernetes,
|
||||
jobIndex: parent.jobIndex,
|
||||
jobTotal: parent.jobTotal,
|
||||
}
|
||||
|
||||
@@ -513,6 +513,17 @@ func TestDockerActionImageTag(t *testing.T) {
|
||||
dockerActionImageTag("owner/repo", "./", true),
|
||||
dockerActionImageTag("owner/repo", "./sub", true),
|
||||
)
|
||||
|
||||
workflow := &model.Workflow{Jobs: map[string]*model.Job{"job1": {}}}
|
||||
job := newTestRC(workflow, nil)
|
||||
step := &stepActionRemote{Step: &model.Step{}, RunContext: job}
|
||||
image := job.dockerActionImage(t.Context(), step, "abc123", false)
|
||||
assert.True(t, strings.HasPrefix(image, "act-abc123-dockeraction:latest-"))
|
||||
assert.Equal(t, image, (&RunContext{Parent: job}).dockerActionImage(t.Context(), step, "abc123", false))
|
||||
assert.NotEqual(t, image, job.dockerActionImage(t.Context(), &stepActionRemote{Step: &model.Step{}, RunContext: job}, "abc123", false))
|
||||
assert.NotEqual(t, image, newTestRC(workflow, nil).dockerActionImage(t.Context(), step, "abc123", false))
|
||||
job.dockerActionImage(common.WithDryrun(t.Context(), true), &stepActionRemote{Step: &model.Step{}, RunContext: job}, "abc123", false)
|
||||
assert.Len(t, job.dockerActionImages, 2)
|
||||
}
|
||||
|
||||
func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
||||
|
||||
@@ -246,18 +246,19 @@ func inputsFromEnv(env map[string]string) map[string]any {
|
||||
}
|
||||
|
||||
func getEvaluatorInputs(rc *RunContext, stepInputs map[string]any, ghc *model.GithubContext) map[string]any {
|
||||
if stepInputs != nil { // an action's own inputs, which workflow inputs must not shadow
|
||||
return maps.Clone(stepInputs)
|
||||
}
|
||||
inputs := map[string]any{}
|
||||
|
||||
// only a called workflow's job itself takes the inputs Gitea resolved,
|
||||
// composite actions copy its Config but have their own `inputs`
|
||||
if rc.Config.WorkflowCallInputs != nil && rc.Parent == nil {
|
||||
maps.Copy(inputs, rc.Config.WorkflowCallInputs)
|
||||
maps.Copy(inputs, stepInputs)
|
||||
return inputs
|
||||
}
|
||||
|
||||
maps.Copy(inputs, rc.workflowCallInputs)
|
||||
maps.Copy(inputs, stepInputs)
|
||||
|
||||
switch ghc.EventName {
|
||||
case "workflow_dispatch":
|
||||
|
||||
@@ -414,6 +414,31 @@ on:
|
||||
assert.Equal(t, map[string]any{"arg": "x"}, getEvaluatorInputs(composite, map[string]any{"arg": "x"}, ghc))
|
||||
}
|
||||
|
||||
func TestActionInputsShadowDispatchInputs(t *testing.T) {
|
||||
wf, err := model.ReadWorkflow(strings.NewReader(`
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
message: {type: string}
|
||||
jobs:
|
||||
job1: {}
|
||||
`))
|
||||
require.NoError(t, err)
|
||||
rc := newTestRC(wf, nil)
|
||||
rc.Config.EventName = "workflow_dispatch"
|
||||
rc.EventJSON = `{"inputs":{"message":"workflow"}}`
|
||||
step := &stepActionRemote{RunContext: rc, env: map[string]string{"INPUT_MESSAGE": "step"}}
|
||||
assert.Equal(t, "step", rc.NewActionInputsExpressionEvaluator(t.Context(), step).InterpolateName(t.Context(), "${{ inputs.message }}"))
|
||||
assert.Equal(t, "workflow", rc.NewStepExpressionEvaluator(t.Context(), step).InterpolateName(t.Context(), "${{ inputs.message }}"))
|
||||
enabled, err := isStepEnabled(t.Context(), "inputs.message == 'workflow'", step, stepStageMain)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, enabled)
|
||||
step.Step, step.action = &model.Step{}, &model.Action{Inputs: map[string]model.Input{"fallback": {Default: "${{ inputs.message }}"}}}
|
||||
env, err := evaluateCompositeInputAndEnv(t.Context(), rc, step)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
|
||||
}
|
||||
|
||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) {
|
||||
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
|
||||
jobs:
|
||||
|
||||
@@ -5,6 +5,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"testing"
|
||||
@@ -40,6 +41,16 @@ func requireDocker(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func requireDockerSharingFiles(t *testing.T) {
|
||||
t.Helper()
|
||||
requireDocker(t)
|
||||
for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} {
|
||||
if _, err := os.Stat(marker); err == nil {
|
||||
t.Skip("skipping: inside a container, the docker daemon cannot bind-mount this process's files")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// requireHostTools skips the test unless every named executable is on PATH. Used by the
|
||||
// self-hosted (host environment) suite, which runs steps directly on the host.
|
||||
func requireHostTools(t *testing.T, tools ...string) {
|
||||
|
||||
+33
-10
@@ -13,6 +13,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"path"
|
||||
"slices"
|
||||
@@ -258,6 +259,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
|
||||
logger.Errorf("##[error]%s", EscapeCommandData("Error while stop job container: "+err.Error()))
|
||||
}
|
||||
setJobResult(ctx, info, rc, jobError == nil)
|
||||
rc.removeDockerActionImages(ctx)
|
||||
|
||||
return err
|
||||
})
|
||||
@@ -410,26 +412,38 @@ func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success boo
|
||||
logger.WithField("jobResult", jobResult).Infof("Job %s", jobResultMessage)
|
||||
}
|
||||
|
||||
// evaluateJobEnvAndDefaults resolves the job's env, defaults.run and container env once, as GitHub does at job setup.
|
||||
func evaluateJobEnvAndDefaults(ctx context.Context, rc *RunContext) error {
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
||||
// evaluateJobEnv returns the workflow and job env evaluated, as far as it got on error, leaving rc.Env raw.
|
||||
func evaluateJobEnv(ctx context.Context, rc *RunContext) (map[string]string, error) {
|
||||
env := rc.GetEnv()
|
||||
evaluated := maps.Clone(env)
|
||||
evaluator := rc.NewExpressionEvaluatorWithEnv(ctx, evaluated)
|
||||
var workflowEnv map[string]string
|
||||
if rc.Run.Workflow.Env == nil {
|
||||
if err := model.DecodeEvaluated("workflow env", rc.Run.Workflow.RawEnv, rc.ExprEval.shared(ctx).EvaluateYamlNode, &workflowEnv); err != nil {
|
||||
return err
|
||||
if err := model.DecodeEvaluated("workflow env", rc.Run.Workflow.RawEnv, evaluator.shared(ctx).EvaluateYamlNode, &workflowEnv); err != nil {
|
||||
return evaluated, err
|
||||
}
|
||||
}
|
||||
if workflowEnv != nil {
|
||||
rc.Env = mergeMaps(workflowEnv, env)
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
||||
evaluated = mergeMaps(workflowEnv, env)
|
||||
evaluator = rc.NewExpressionEvaluatorWithEnv(ctx, evaluated)
|
||||
}
|
||||
var err error
|
||||
for k, v := range env {
|
||||
if rc.Env[k], err = rc.ExprEval.Interpolate(ctx, v); err != nil {
|
||||
return fmt.Errorf("unable to interpolate env %s: %w", k, err)
|
||||
if evaluated[k], err = evaluator.Interpolate(ctx, v); err != nil {
|
||||
return evaluated, fmt.Errorf("unable to interpolate env %s: %w", k, err)
|
||||
}
|
||||
}
|
||||
return evaluated, nil
|
||||
}
|
||||
|
||||
// evaluateJobEnvAndDefaults resolves the job's env, defaults.run and container env once, as GitHub does at job setup.
|
||||
func evaluateJobEnvAndDefaults(ctx context.Context, rc *RunContext) error {
|
||||
env, err := evaluateJobEnv(ctx, rc)
|
||||
rc.Env = env
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var defaults model.Defaults
|
||||
if err := model.DecodeEvaluated("defaults", rc.Run.Job().RawDefaults, rc.ExprEval.shared(ctx).EvaluateYamlNode, &defaults); err != nil {
|
||||
return err
|
||||
@@ -518,7 +532,7 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
|
||||
"/jobs/" + strconv.FormatInt(jobID, 10) + "/steps/"
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
// Reuse a single client across all step uploads so connections can be pooled.
|
||||
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout}
|
||||
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout, Transport: rc.Config.instanceTransportFor(base)}
|
||||
for i := range rc.Run.Job().Steps {
|
||||
summaryPath := path.Join(actPath, "workflow", "step-summary-"+strconv.Itoa(i)+".md")
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, rc.JobContainer, summaryPath, maxJobSummaryBytes)
|
||||
@@ -530,6 +544,15 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
|
||||
}
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// instanceTransportFor returns InstanceTransport only for URLs on the registered address, as steps can rewrite the summary URL through GITHUB_ENV.
|
||||
func (c Config) instanceTransportFor(rawURL string) http.RoundTripper {
|
||||
if !strings.HasPrefix(rawURL, c.GitHubInstance+"/") {
|
||||
return nil
|
||||
}
|
||||
return c.InstanceTransport
|
||||
}
|
||||
|
||||
// extractJobIDFromRuntimeToken returns the JobID claim from an ACTIONS_RUNTIME_TOKEN JWT
|
||||
// without verifying its signature. Returns 0 if the token is unparseable or has no JobID.
|
||||
func extractJobIDFromRuntimeToken(token string) int64 {
|
||||
|
||||
@@ -858,7 +858,7 @@ func TestTryUploadJobSummaryStopsAtPhaseTimeout(t *testing.T) {
|
||||
cm.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t *testing.T) {
|
||||
runtimeToken := fakeRuntimeToken(34)
|
||||
|
||||
type upload struct {
|
||||
@@ -866,7 +866,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
body string
|
||||
}
|
||||
var got []upload
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
got = append(got, upload{r.URL.Path, string(body)})
|
||||
@@ -896,6 +896,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
|
||||
"ACTIONS_RUNTIME_TOKEN": runtimeToken,
|
||||
"GITEA_RUN_ID": "12",
|
||||
}, cm, 3)
|
||||
rc.Config.GitHubInstance, rc.Config.InstanceTransport = server.URL, server.Client().Transport
|
||||
|
||||
tryUploadJobSummary(ctx, rc)
|
||||
|
||||
|
||||
@@ -39,11 +39,8 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
|
||||
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(uses))
|
||||
|
||||
// If the repository is private, we need a token to clone it
|
||||
token := rc.Config.GetToken()
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir, token),
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir),
|
||||
newReusableWorkflowExecutor(rc, workflowDir, remoteReusableWorkflow.Path),
|
||||
)
|
||||
}
|
||||
@@ -60,10 +57,8 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Owner, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref)
|
||||
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename))
|
||||
|
||||
token := getGitCloneToken(rc.Config, remoteReusableWorkflow.CloneURL())
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir, token),
|
||||
cloneRemoteReusableWorkflow(rc, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, workflowDir),
|
||||
newReusableWorkflowExecutor(rc, workflowDir, remoteReusableWorkflow.Path),
|
||||
)
|
||||
}
|
||||
@@ -76,20 +71,19 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
// 2. Gitea has already full URL with rc.Config.GitHubInstance when calling newRemoteReusableWorkflow
|
||||
//
|
||||
// remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL
|
||||
func cloneRemoteReusableWorkflow(rc *RunContext, cloneURL, ref, targetDirectory, token string) common.Executor {
|
||||
func cloneRemoteReusableWorkflow(rc *RunContext, cloneURL, ref, targetDirectory string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
interpolatedURL, err := rc.NewExpressionEvaluator(ctx).Interpolate(ctx, cloneURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to interpolate the workflow clone URL: %w", err)
|
||||
}
|
||||
return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
return git.NewGitCloneExecutor(rc.Config.withInstanceAuth(git.NewGitCloneExecutorInput{
|
||||
URL: interpolatedURL,
|
||||
Ref: ref,
|
||||
Dir: targetDirectory,
|
||||
Token: token,
|
||||
OfflineMode: rc.Config.ActionOfflineMode,
|
||||
Depth: rc.Config.ActionCloneDepth,
|
||||
})(ctx)
|
||||
}, rc.Config.GetToken()))(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,13 +202,18 @@ func setReusedWorkflowCallerResult(rc *RunContext, runner *runnerImpl) common.Ex
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// getGitCloneToken returns GITEA_TOKEN when shouldCloneURLUseToken returns true,
|
||||
// otherwise returns an empty string
|
||||
func getGitCloneToken(conf *Config, cloneURL string) string {
|
||||
if !shouldCloneURLUseToken(conf.GitHubInstance, conf.trustedActionInstance(), cloneURL) {
|
||||
return ""
|
||||
// withInstanceAuth passes instance credentials only to clones from this Gitea instance, and runner.insecure only to its registered address.
|
||||
func (c Config) withInstanceAuth(input git.NewGitCloneExecutorInput, token string) git.NewGitCloneExecutorInput {
|
||||
u, err := url.Parse(input.URL)
|
||||
if err != nil || !onInstanceHost(u.Host, c.GitHubInstance, c.trustedActionInstance()) {
|
||||
return input
|
||||
}
|
||||
return conf.GetToken()
|
||||
input.ClientCertFile, input.ClientKeyFile = c.ClientCertFile, c.ClientKeyFile
|
||||
input.InsecureSkipTLS = c.InsecureSkipTLS && onInstanceHost(u.Host, c.GitHubInstance)
|
||||
if u.User == nil {
|
||||
input.Token = token
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
@@ -228,26 +227,16 @@ func (c Config) trustedActionInstance() string {
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// shouldCloneURLUseToken returns true when the following conditions are met:
|
||||
// 1. cloneURL's host matches this Gitea instance: either the registered instance
|
||||
// (instanceURL) or, for DEFAULT_ACTIONS_URL=self on a different hostname, the
|
||||
// self-hosted action instance (trustedActionInstance, "" when not trusted)
|
||||
// 2. the cloneURL does not have basic auth embedded
|
||||
func shouldCloneURLUseToken(instanceURL, trustedActionInstance, cloneURL string) bool {
|
||||
u2, err := url.Parse(cloneURL)
|
||||
if err != nil || u2.User != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, candidate := range []string{instanceURL, trustedActionInstance} {
|
||||
// onInstanceHost reports whether host is that of one of the instance URLs, which may lack a scheme.
|
||||
func onInstanceHost(host string, instanceURLs ...string) bool {
|
||||
for _, candidate := range instanceURLs {
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(candidate, "http://") &&
|
||||
!strings.HasPrefix(candidate, "https://") {
|
||||
if !strings.HasPrefix(candidate, "http://") && !strings.HasPrefix(candidate, "https://") {
|
||||
candidate = "https://" + candidate
|
||||
}
|
||||
if u1, err := url.Parse(candidate); err == nil && u1.Host == u2.Host {
|
||||
if u, err := url.Parse(candidate); err == nil && u.Host == host {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,12 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -59,7 +64,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
}
|
||||
cacheDir := t.TempDir()
|
||||
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir, "")(context.Background()))
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(context.Background()))
|
||||
got, err := os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tmpl("v1"), string(got))
|
||||
@@ -69,7 +74,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
gitMust(t, workDir, "commit", "-am", "v2")
|
||||
gitMust(t, workDir, "push", "origin", "master")
|
||||
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir, "")(context.Background()))
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(context.Background()))
|
||||
got, err = os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tmpl("v2"), string(got), "cached workflow file must reflect the updated branch tip")
|
||||
@@ -123,107 +128,79 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetGitCloneTokenWithSchemalessGiteaInstance(t *testing.T) {
|
||||
conf := &Config{
|
||||
GitHubInstance: "gitea.example.net",
|
||||
Secrets: map[string]string{
|
||||
"GITEA_TOKEN": "token-value",
|
||||
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T) {
|
||||
var gotCert bool
|
||||
var gotToken string
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotCert = len(r.TLS.PeerCertificates) > 0
|
||||
_, gotToken, _ = r.BasicAuth()
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
pair := server.TLS.Certificates[0]
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(pair.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
dir := t.TempDir()
|
||||
certFile, keyFile := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: pair.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
rc := &RunContext{
|
||||
Config: &Config{
|
||||
GitHubInstance: server.URL,
|
||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
},
|
||||
Run: &model.Run{JobID: "j1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"j1": {}}}},
|
||||
}
|
||||
|
||||
token := getGitCloneToken(conf, "https://gitea.example.net/actions/tools")
|
||||
|
||||
require.Equal(t, "token-value", token)
|
||||
require.Error(t, cloneRemoteReusableWorkflow(rc, server.URL+"/owner/repo", "main", t.TempDir())(t.Context()))
|
||||
require.True(t, gotCert)
|
||||
require.Equal(t, "token", gotToken)
|
||||
}
|
||||
|
||||
func TestGetGitCloneTokenSelfHostedActionsDifferentHost(t *testing.T) {
|
||||
// The runner registered with one hostname while DEFAULT_ACTIONS_URL=self resolves
|
||||
// actions against AppURL on a different hostname for the same instance.
|
||||
conf := &Config{
|
||||
GitHubInstance: "gitea.local",
|
||||
DefaultActionInstance: "https://gitea.my-nas.lan",
|
||||
DefaultActionInstanceIsSelfHosted: true,
|
||||
Secrets: map[string]string{
|
||||
"GITEA_TOKEN": "token-value",
|
||||
},
|
||||
}
|
||||
|
||||
token := getGitCloneToken(conf, "https://gitea.my-nas.lan/owner/action")
|
||||
|
||||
require.Equal(t, "token-value", token)
|
||||
}
|
||||
|
||||
func TestShouldCloneURLUseToken(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
instanceURL string
|
||||
trustedActionInstance string
|
||||
cloneURL string
|
||||
want bool
|
||||
func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, instanceURL, trustedActionInstance, cloneURL string
|
||||
token, cert, insecure bool
|
||||
}{
|
||||
{
|
||||
name: "same host with schemaless instance",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://gitea.example.net/actions/tools",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "same host with schemaless instance and port",
|
||||
instanceURL: "gitea.example.net:3000",
|
||||
cloneURL: "https://gitea.example.net:3000/actions/tools",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "different host",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://github.com/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "embedded basic auth",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "https://user:[email protected]/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "invalid clone URL",
|
||||
instanceURL: "gitea.example.net",
|
||||
cloneURL: "://gitea.example.net/actions/tools",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// self-hosted DEFAULT_ACTIONS_URL on a different hostname than the
|
||||
// registered instance: the token must still be attached.
|
||||
name: "self-hosted action instance on different host",
|
||||
instanceURL: "gitea.local",
|
||||
trustedActionInstance: "https://gitea.my-nas.lan",
|
||||
cloneURL: "https://gitea.my-nas.lan/owner/action",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// embedded basic auth must still be rejected even when the host matches
|
||||
// the trusted action instance.
|
||||
name: "self-hosted action instance with embedded basic auth",
|
||||
instanceURL: "gitea.local",
|
||||
trustedActionInstance: "https://gitea.my-nas.lan",
|
||||
cloneURL: "https://user:[email protected]/owner/action",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// github.com / mirror hosts are never trusted: trustedActionInstance is
|
||||
// empty in github mode, so an off-instance clone URL gets no token.
|
||||
name: "github mode does not trust mirror host",
|
||||
instanceURL: "gitea.local",
|
||||
cloneURL: "https://mirror.example.com/owner/action",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
{"same host with schemaless instance", "gitea.example.net", "", "https://gitea.example.net/actions/tools", true, true, true},
|
||||
{"same host with schemaless instance and port", "gitea.example.net:3000", "", "https://gitea.example.net:3000/actions/tools", true, true, true},
|
||||
{"different host", "gitea.example.net", "", "https://github.com/actions/tools", false, false, false},
|
||||
{"embedded basic auth keeps its own credentials", "gitea.example.net", "", "https://user:pass@gitea.example.net/actions/tools", false, true, true},
|
||||
{"invalid clone URL", "gitea.example.net", "", "://gitea.example.net/actions/tools", false, false, false},
|
||||
{"self-hosted action instance on different host skips no verification", "gitea.local", "https://gitea.my-nas.lan", "https://gitea.my-nas.lan/owner/action", true, true, false},
|
||||
{"self-hosted action instance with embedded basic auth", "gitea.local", "https://gitea.my-nas.lan", "https://user:[email protected]/owner/action", false, true, false},
|
||||
{"github mode does not trust mirror host", "gitea.local", "", "https://mirror.example.com/owner/action", false, false, false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, shouldCloneURLUseToken(tt.instanceURL, tt.trustedActionInstance, tt.cloneURL))
|
||||
conf := Config{
|
||||
GitHubInstance: tt.instanceURL,
|
||||
DefaultActionInstance: tt.trustedActionInstance,
|
||||
DefaultActionInstanceIsSelfHosted: tt.trustedActionInstance != "",
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: "/cert",
|
||||
ClientKeyFile: "/key",
|
||||
}
|
||||
want := git.NewGitCloneExecutorInput{URL: tt.cloneURL, InsecureSkipTLS: tt.insecure}
|
||||
if tt.token {
|
||||
want.Token = "token"
|
||||
}
|
||||
if tt.cert {
|
||||
want.ClientCertFile, want.ClientKeyFile = conf.ClientCertFile, conf.ClientKeyFile
|
||||
}
|
||||
require.Equal(t, want, conf.withInstanceAuth(git.NewGitCloneExecutorInput{URL: tt.cloneURL}, "token"))
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{}
|
||||
conf := Config{GitHubInstance: "https://gitea.local", InstanceTransport: transport}
|
||||
require.Same(t, transport, conf.instanceTransportFor("https://gitea.local/api/actions_pipeline/"))
|
||||
require.Nil(t, conf.instanceTransportFor("https://gitea.local.example/api/actions_pipeline/"))
|
||||
require.Nil(t, conf.instanceTransportFor("https://other.example/x#/api/actions_pipeline/"))
|
||||
}
|
||||
|
||||
func gitMust(t *testing.T, dir string, args ...string) {
|
||||
|
||||
+188
-111
@@ -7,6 +7,7 @@ package runner
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -64,6 +66,7 @@ type RunContext struct {
|
||||
ExprEval *expressionEvaluator
|
||||
JobContainer container.ExecutionsEnvironment
|
||||
serviceContainers []*serviceContainer
|
||||
kubernetes bool
|
||||
containerSpec model.ContainerSpec // container:, resolved once with platformImage
|
||||
containerEnv yaml.Node // container.env of the kept containerSpec, decoded once the job env resolves
|
||||
JobName string
|
||||
@@ -79,6 +82,10 @@ type RunContext struct {
|
||||
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
||||
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
||||
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
||||
|
||||
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
||||
dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends
|
||||
|
||||
// summaryFileInitialized tracks which per-step summary files (workflow/step-summary-N.md)
|
||||
// have already been created on the JobContainer. The runner sets up file-command files
|
||||
// via JobContainer.Copy at the start of every phase, which truncates them — fine for
|
||||
@@ -253,6 +260,9 @@ const sharedToolCacheVolume = "act-toolcache" // mounted only when the tool cach
|
||||
func (rc *RunContext) validVolumes() []string {
|
||||
name := rc.jobContainerName()
|
||||
volumes := slices.Clone(rc.Config.ValidVolumes)
|
||||
if hostEnv, ok := rc.JobContainer.(*container.HostEnvironment); ok {
|
||||
volumes = append(volumes, hostEnv.Path, hostEnv.GetActPath())
|
||||
}
|
||||
if rc.Config.SharedToolCache {
|
||||
volumes = append(volumes, sharedToolCacheVolume)
|
||||
}
|
||||
@@ -351,6 +361,9 @@ func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[stri
|
||||
mounts[sharedToolCacheVolume] = toolCache
|
||||
}
|
||||
}
|
||||
if hostEnv, ok := rc.JobContainer.(*container.HostEnvironment); ok {
|
||||
binds = append(binds, hostEnv.Path+":"+hostEnv.Path, hostEnv.GetActPath()+":"+hostEnv.GetActPath())
|
||||
}
|
||||
mounts[name+"-env"] = ext.GetActPath() // runner-internal, never overridable
|
||||
|
||||
if workdir := ext.ToContainerPath(rc.Config.Workdir); !claimed[workdir] {
|
||||
@@ -417,11 +430,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
}
|
||||
rc.cleanUpJobContainer = rc.JobContainer.Remove()
|
||||
for k, v := range rc.getRunnerContext(ctx) {
|
||||
if v, ok := v.(string); ok {
|
||||
rc.Env["RUNNER_"+strings.ToUpper(k)] = v
|
||||
}
|
||||
}
|
||||
rc.setRunnerContextEnv(ctx)
|
||||
for _, env := range os.Environ() {
|
||||
if k, v, ok := strings.Cut(env, "="); ok {
|
||||
// don't override
|
||||
@@ -431,16 +440,70 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
return rc.copyWorkflowFiles()(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) setRunnerContextEnv(ctx context.Context) {
|
||||
for k, v := range rc.getRunnerContext(ctx) {
|
||||
if v, ok := v.(string); ok {
|
||||
rc.Env["RUNNER_"+strings.ToUpper(k)] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var errKubernetesDocker = errors.New("docker actions and docker:// steps are not supported on Kubernetes")
|
||||
|
||||
func (rc *RunContext) startKubernetesPod() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if rc.containerSpec.Options != "" || len(rc.containerSpec.Volumes) > 0 || rc.containerSpec.Credentials != nil {
|
||||
return errors.New("container options, volumes and credentials are not supported on Kubernetes")
|
||||
}
|
||||
var options container.KubernetesOptions
|
||||
if rc.Config.KubernetesPicker != nil {
|
||||
options = rc.Config.KubernetesPicker(rc.runsOnPlatformNames(ctx))
|
||||
}
|
||||
options.MaxLifetime, options.ForcePull = rc.Config.ContainerMaxLifetime, rc.Config.ForcePull
|
||||
pod, err := container.NewKubernetesPod(options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
services, err := rc.jobServices(ctx, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, serviceID := range slices.Sorted(maps0.Keys(services)) {
|
||||
input := services[serviceID]
|
||||
if input.Username != "" || len(input.Binds) > 0 || len(input.Mounts) > 0 {
|
||||
return fmt.Errorf("service %s: volumes and credentials are not supported on Kubernetes", serviceID)
|
||||
}
|
||||
rc.serviceContainers = append(rc.serviceContainers, &serviceContainer{name: serviceID, image: input.Image, container: pod.ServiceContainer(serviceID, input)})
|
||||
}
|
||||
|
||||
logWriter := rc.commandLogWriter(ctx)
|
||||
rc.JobContainer = pod.JobContainer(&container.NewContainerInput{
|
||||
Entrypoint: []string{"tail", "-f", "/dev/null"},
|
||||
WorkingDir: (&container.LinuxContainerEnvironmentExtensions{}).ToContainerPath(rc.Config.Workdir),
|
||||
Image: rc.platformImage,
|
||||
Name: rc.jobContainerName(),
|
||||
Env: []string{"LANG=C.UTF-8"},
|
||||
Stdout: logWriter,
|
||||
Stderr: logWriter,
|
||||
})
|
||||
rc.cleanUpJobContainer = func(ctx context.Context) error {
|
||||
rc.printServiceLogs(ctx)
|
||||
return rc.JobContainer.Remove()(ctx)
|
||||
}
|
||||
defer printStartJobContainerGroup(ctx, rc.platformImage, rc.jobContainerName(), "")()
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||
Name: "workflow/event.json",
|
||||
Mode: 0o644,
|
||||
Body: rc.EventJSON,
|
||||
}, &container.FileEntry{
|
||||
Name: "workflow/envs.txt",
|
||||
Mode: 0o666,
|
||||
Body: "",
|
||||
}),
|
||||
rc.JobContainer.Create(nil, nil),
|
||||
rc.startServiceContainers(),
|
||||
rc.JobContainer.Start(false),
|
||||
func(ctx context.Context) error { rc.setRunnerContextEnv(ctx); return nil },
|
||||
rc.captureJobContainerInfo(),
|
||||
rc.copyWorkflowFiles(),
|
||||
rc.reportUnstartedServices(),
|
||||
rc.waitForServiceContainers(),
|
||||
)(ctx)
|
||||
}
|
||||
}
|
||||
@@ -470,7 +533,6 @@ func WithJobVolumeCleanup(ctx context.Context, deferCleanup func(common.Executor
|
||||
func (rc *RunContext) startJobContainer() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
rc.deferVolumeCleanup, _ = ctx.Value(jobVolumeCleanupKey{}).(func(common.Executor))
|
||||
logger := common.Logger(ctx)
|
||||
image := rc.platformImage
|
||||
logWriter := rc.commandLogWriter(ctx)
|
||||
|
||||
@@ -483,6 +545,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
// For gitea, to support --volumes-from <container_name_or_id> in options.
|
||||
// We need to set the container name to the environment variable.
|
||||
rc.Env["JOB_CONTAINER_NAME"] = name
|
||||
delete(rc.jobEnv, "JOB_CONTAINER_NAME") // the runtime name wins over a workflow env of the same name
|
||||
|
||||
envList := make([]string, 0)
|
||||
|
||||
@@ -495,78 +558,13 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
networkName, createAndDeleteNetwork := rc.networkNameForGitea()
|
||||
rc.cleanUpJobContainer = rc.cleanupJobResources(networkName, createAndDeleteNetwork, false)
|
||||
|
||||
var services map[string]*model.ContainerSpec
|
||||
if err := model.DecodeEvaluated("job services", rc.Run.Job().RawServices, rc.ExprEval.shared(ctx).EvaluateYamlNode, &services); err != nil {
|
||||
services, err := rc.jobServices(ctx, networkName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, serviceID := range slices.Sorted(maps0.Keys(services)) {
|
||||
spec := services[serviceID]
|
||||
if spec == nil {
|
||||
return fmt.Errorf("service %s has no container definition", serviceID)
|
||||
}
|
||||
spec.Image = strings.TrimPrefix(spec.Image, "docker://")
|
||||
// GitHub compatibility: skip services whose image evaluates to an
|
||||
// empty string, enabling conditional services via expressions
|
||||
if spec.Image == "" {
|
||||
logger.Infof("The service '%s' will not be started because the container definition has an empty image.", serviceID)
|
||||
continue
|
||||
}
|
||||
envs := make([]string, 0, len(spec.Env)+len(rc.Config.ProxyEnv))
|
||||
// a service reaches the internet the way the job does; its own env still wins
|
||||
for k, v := range mergeMaps(rc.Config.ProxyEnv, spec.Env) {
|
||||
envs = append(envs, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
// keep these local: reusing username/password would overwrite the
|
||||
// credentials the job container is pulled with further down
|
||||
serviceUsername, servicePassword, err := registryCredentials(spec.Credentials, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to handle service %s credentials: %w", serviceID, err)
|
||||
}
|
||||
cmd := spec.Cmd
|
||||
if len(cmd) == 0 && spec.Command != "" {
|
||||
if cmd, err = shellquote.Split(spec.Command); err != nil {
|
||||
return fmt.Errorf("failed to parse service %s command: %w", serviceID, err)
|
||||
}
|
||||
}
|
||||
var entrypoint []string
|
||||
if spec.Entrypoint != "" {
|
||||
entrypoint = []string{spec.Entrypoint}
|
||||
}
|
||||
|
||||
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes)
|
||||
|
||||
exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse service %s ports: %w", serviceID, err)
|
||||
}
|
||||
|
||||
serviceContainerName := createContainerName(rc.jobContainerName(), serviceID)
|
||||
serviceLog := common.NewLineWriter(rawLogLine(ctx)) // raw, so a service's output cannot set the job's env, outputs or state
|
||||
c := newContainer(&container.NewContainerInput{
|
||||
Name: serviceContainerName,
|
||||
Image: spec.Image,
|
||||
Username: serviceUsername,
|
||||
Password: servicePassword,
|
||||
Entrypoint: entrypoint,
|
||||
Cmd: cmd,
|
||||
Env: envs,
|
||||
Mounts: serviceMounts,
|
||||
Binds: serviceBinds,
|
||||
Stdout: serviceLog,
|
||||
Stderr: serviceLog,
|
||||
Privileged: rc.Config.Privileged,
|
||||
UsernsMode: rc.Config.UsernsMode,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
AutoRemove: false, // so a dead service's log survives, cleanupJobResources removes it
|
||||
WorkflowOptions: spec.Options,
|
||||
NetworkMode: networkName,
|
||||
NetworkAliases: []string{serviceID},
|
||||
ExposedPorts: exposedPorts,
|
||||
PortBindings: portBindings,
|
||||
ValidVolumes: rc.Config.ValidVolumes, // not validVolumes(), a service gets no docker socket
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
})
|
||||
rc.serviceContainers = append(rc.serviceContainers, &serviceContainer{name: serviceID, image: spec.Image, container: c})
|
||||
input := services[serviceID]
|
||||
rc.serviceContainers = append(rc.serviceContainers, &serviceContainer{name: serviceID, image: input.Image, container: newContainer(input)})
|
||||
}
|
||||
|
||||
// For Gitea, `jobContainerNetwork` should be the same as `networkName`
|
||||
@@ -575,7 +573,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
containerInput := &container.NewContainerInput{
|
||||
Cmd: nil,
|
||||
Entrypoint: []string{"sleep", fmt.Sprint(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds())},
|
||||
Entrypoint: []string{"sleep", strconv.FormatInt(int64(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds()), 10)},
|
||||
WorkingDir: ext.ToContainerPath(rc.Config.Workdir),
|
||||
Image: image,
|
||||
Username: username,
|
||||
@@ -623,20 +621,86 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop),
|
||||
rc.JobContainer.Start(false),
|
||||
rc.captureJobContainerInfo(),
|
||||
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||
Name: "workflow/event.json",
|
||||
Mode: 0o644,
|
||||
Body: rc.EventJSON,
|
||||
}, &container.FileEntry{
|
||||
Name: "workflow/envs.txt",
|
||||
Mode: 0o666,
|
||||
Body: "",
|
||||
}),
|
||||
rc.copyWorkflowFiles(),
|
||||
rc.waitForServiceContainers(),
|
||||
)(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) jobServices(ctx context.Context, networkName string) (map[string]*container.NewContainerInput, error) {
|
||||
var services map[string]*model.ContainerSpec
|
||||
if err := model.DecodeEvaluated("job services", rc.Run.Job().RawServices, rc.NewExpressionEvaluatorWithEnv(ctx, mergeMaps(rc.GetEnv(), rc.jobEnv)).shared(ctx).EvaluateYamlNode, &services); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inputs := map[string]*container.NewContainerInput{}
|
||||
for _, serviceID := range slices.Sorted(maps0.Keys(services)) {
|
||||
spec := services[serviceID]
|
||||
if spec == nil {
|
||||
return nil, fmt.Errorf("service %s has no container definition", serviceID)
|
||||
}
|
||||
spec.Image = strings.TrimPrefix(spec.Image, "docker://")
|
||||
// GitHub compatibility: skip services whose image evaluates to an
|
||||
// empty string, enabling conditional services via expressions
|
||||
if spec.Image == "" {
|
||||
common.Logger(ctx).Infof("The service '%s' will not be started because the container definition has an empty image.", serviceID)
|
||||
continue
|
||||
}
|
||||
envs := make([]string, 0, len(spec.Env)+len(rc.Config.ProxyEnv))
|
||||
// a service reaches the internet the way the job does; its own env still wins
|
||||
for k, v := range mergeMaps(rc.Config.ProxyEnv, spec.Env) {
|
||||
envs = append(envs, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
serviceUsername, servicePassword, err := registryCredentials(spec.Credentials, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to handle service %s credentials: %w", serviceID, err)
|
||||
}
|
||||
cmd := spec.Cmd
|
||||
if len(cmd) == 0 && spec.Command != "" {
|
||||
if cmd, err = shellquote.Split(spec.Command); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse service %s command: %w", serviceID, err)
|
||||
}
|
||||
}
|
||||
var entrypoint []string
|
||||
if spec.Entrypoint != "" {
|
||||
entrypoint = []string{spec.Entrypoint}
|
||||
}
|
||||
|
||||
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes)
|
||||
|
||||
exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse service %s ports: %w", serviceID, err)
|
||||
}
|
||||
|
||||
serviceLog := common.NewLineWriter(rawLogLine(ctx)) // raw, so a service's output cannot set the job's env, outputs or state
|
||||
inputs[serviceID] = &container.NewContainerInput{
|
||||
Name: createContainerName(rc.jobContainerName(), serviceID),
|
||||
Image: spec.Image,
|
||||
Username: serviceUsername,
|
||||
Password: servicePassword,
|
||||
Entrypoint: entrypoint,
|
||||
Cmd: cmd,
|
||||
Env: envs,
|
||||
Mounts: serviceMounts,
|
||||
Binds: serviceBinds,
|
||||
Stdout: serviceLog,
|
||||
Stderr: serviceLog,
|
||||
Privileged: rc.Config.Privileged,
|
||||
UsernsMode: rc.Config.UsernsMode,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
AutoRemove: false, // so a dead service's log survives, cleanupJobResources removes it
|
||||
WorkflowOptions: spec.Options,
|
||||
NetworkMode: networkName,
|
||||
NetworkAliases: []string{serviceID},
|
||||
ExposedPorts: exposedPorts,
|
||||
PortBindings: portBindings,
|
||||
ValidVolumes: rc.Config.ValidVolumes, // not validVolumes(), a service gets no docker socket
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
}
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
|
||||
func (rc *RunContext) commandLogWriter(ctx context.Context) io.Writer {
|
||||
return common.NewLineWriter(rc.commandHandler(ctx), rawLogLine(ctx))
|
||||
}
|
||||
@@ -661,7 +725,7 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
||||
logger.Infof("Cleaning up services for job %s", rc.JobName)
|
||||
errs = append(errs, rc.stopServiceContainers()(ctx))
|
||||
}
|
||||
if !common.Dryrun(ctx) && (preclean || rc.hadDockerProxy) {
|
||||
if !common.Dryrun(ctx) && !rc.Config.NoSweep && (preclean || rc.hadDockerProxy) {
|
||||
errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName()))
|
||||
}
|
||||
if preclean || rc.deferVolumeCleanup == nil {
|
||||
@@ -783,6 +847,14 @@ func (rc *RunContext) stopJobContainer() common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) removeDockerActionImages(ctx context.Context) {
|
||||
for _, image := range rc.dockerActionImages {
|
||||
if _, err := container.RemoveImage(ctx, image, true, false); err != nil {
|
||||
common.Logger(ctx).Warnf("failed to remove action image %s: %v", image, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) pullServicesImages(forcePull bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
execs := []common.Executor{}
|
||||
@@ -975,6 +1047,13 @@ func (svc *serviceContainer) healthOutputSuffix() string {
|
||||
return ": " + svc.info.HealthOutput
|
||||
}
|
||||
|
||||
func (rc *RunContext) copyWorkflowFiles() common.Executor {
|
||||
return rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/",
|
||||
&container.FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: rc.EventJSON},
|
||||
&container.FileEntry{Name: "workflow/envs.txt", Mode: 0o666},
|
||||
)
|
||||
}
|
||||
|
||||
// captureJobContainerInfo is a convenience: failing to describe the container must not
|
||||
// fail the job.
|
||||
func (rc *RunContext) captureJobContainerInfo() common.Executor {
|
||||
@@ -1043,9 +1122,12 @@ func (rc *RunContext) interpolateOutputs() common.Executor {
|
||||
func (rc *RunContext) startContainer() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
var err error
|
||||
if rc.IsHostEnv() {
|
||||
switch {
|
||||
case rc.kubernetes:
|
||||
err = rc.startKubernetesPod()(ctx)
|
||||
case rc.IsHostEnv():
|
||||
err = rc.startHostEnvironment()(ctx)
|
||||
} else {
|
||||
default:
|
||||
err = rc.startJobContainer()(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1154,10 +1236,6 @@ func (rc *RunContext) Executor() (common.Executor, error) {
|
||||
}
|
||||
|
||||
func (rc *RunContext) runsOnImage(ctx context.Context) (string, error) {
|
||||
if rc.Run.Job().RunsOn() == nil {
|
||||
common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String())
|
||||
}
|
||||
|
||||
runsOn := rc.Run.Job().RunsOn()
|
||||
for i, v := range runsOn {
|
||||
var err error
|
||||
@@ -1192,22 +1270,23 @@ func (rc *RunContext) runsOnPlatformNames(ctx context.Context) []string {
|
||||
|
||||
// resolvePlatformImage evaluates the job's container once for every consumer, skipping one whose image is empty as GitHub does.
|
||||
func (rc *RunContext) resolvePlatformImage(ctx context.Context) error {
|
||||
rc.jobEnv, _ = evaluateJobEnv(ctx, rc) // a broken env fails the job at setup instead
|
||||
withoutEnv, env := splitContainerEnv(rc.Run.Job().RawContainer)
|
||||
var spec model.ContainerSpec
|
||||
if err := model.DecodeEvaluated("container", withoutEnv, rc.ExprEval.shared(ctx).EvaluateYamlNode, &spec); err != nil {
|
||||
if err := model.DecodeEvaluated("container", withoutEnv, rc.NewExpressionEvaluatorWithEnv(ctx, rc.jobEnv).shared(ctx).EvaluateYamlNode, &spec); err != nil {
|
||||
return err
|
||||
}
|
||||
spec.Image = strings.TrimPrefix(spec.Image, "docker://")
|
||||
if spec.Image != "" || imagelessContainer(withoutEnv) {
|
||||
rc.containerSpec, rc.containerEnv = spec, env
|
||||
}
|
||||
if rc.containerSpec.Image != "" {
|
||||
rc.platformImage = rc.containerSpec.Image
|
||||
return nil
|
||||
}
|
||||
image, err := rc.runsOnImage(ctx)
|
||||
rc.platformImage = image
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
image, rc.kubernetes = strings.CutPrefix(image, "kubernetes://")
|
||||
rc.platformImage = cmp.Or(rc.containerSpec.Image, image)
|
||||
return nil
|
||||
}
|
||||
|
||||
// imagelessContainer reports a container: mapping without an image key, which configures the runs-on image.
|
||||
@@ -1625,9 +1704,7 @@ func parentDir(p string) string {
|
||||
// still names a release when the label is a rolling one such as ubuntu-latest.
|
||||
func (rc *RunContext) imageOS(ctx context.Context) string {
|
||||
if rc.Run.Job().RunsOn() == nil {
|
||||
// A composite action runs on a synthetic job, and resolving its image would only
|
||||
// log that runs-on is missing.
|
||||
return ""
|
||||
return "" // a composite action runs on a synthetic job without runs-on
|
||||
}
|
||||
if imageOS := imageOSFromImage(rc.platformImage); imageOS != "" {
|
||||
return imageOS
|
||||
|
||||
@@ -251,7 +251,6 @@ func startJobContainerInputs(t *testing.T, workflowYAML string, cfg *Config) []*
|
||||
rc := &RunContext{
|
||||
Name: "test",
|
||||
Config: cfg,
|
||||
Env: map[string]string{},
|
||||
Run: &model.Run{
|
||||
JobID: "job",
|
||||
Workflow: workflow,
|
||||
@@ -362,6 +361,29 @@ jobs:
|
||||
require.Equal(t, "sleep", job.Entrypoint[0])
|
||||
require.Equal(t, "--label ${{ github.job }}", job.WorkflowOptions)
|
||||
require.Equal(t, "${{ vars.TITLE }}", job.Password)
|
||||
|
||||
inputs = startJobContainerInputs(t, `
|
||||
env:
|
||||
REG_USER: ${{ vars.REG_USER }}
|
||||
TEXT: ${{ vars.TEXT }}
|
||||
JOB_CONTAINER_NAME: from-workflow
|
||||
jobs:
|
||||
job:
|
||||
container:
|
||||
image: node:20
|
||||
credentials: {username: "${{ env.REG_USER }}", password: pass}
|
||||
services:
|
||||
redis:
|
||||
image: redis:latest
|
||||
credentials: {username: "${{ env.REG_USER }}", password: pass}
|
||||
env: {TEXT: "${{ env.TEXT }}"}
|
||||
options: --volumes-from ${{ env.JOB_CONTAINER_NAME }}
|
||||
`, &Config{Vars: map[string]string{"REG_USER": "user", "TEXT": "${{ 'literal' }}"}})
|
||||
redis, job = inputs[0], inputs[1]
|
||||
require.Equal(t, "user", job.Username)
|
||||
require.Equal(t, "user", redis.Username)
|
||||
require.Equal(t, []string{"TEXT=${{ 'literal' }}"}, redis.Env)
|
||||
require.Equal(t, "--volumes-from "+job.Name, redis.WorkflowOptions)
|
||||
}
|
||||
|
||||
// Only the workflow's options may be stripped later, so the two sources have to reach the
|
||||
@@ -662,6 +684,15 @@ func TestRunContextValidVolumes(t *testing.T) {
|
||||
assert.Contains(t, rc.validVolumes(), rc.Config.Workdir)
|
||||
rc.Config.PresetGitHubContext = &model.GithubContext{}
|
||||
assert.Contains(t, rc.validVolumes(), filepath.FromSlash("/workspace/1"))
|
||||
|
||||
hostEnv := &container.HostEnvironment{Path: t.TempDir(), ActPath: t.TempDir()}
|
||||
rc.JobContainer = hostEnv
|
||||
rc.Config.BindWorkdir = false
|
||||
rc.Config.SharedToolCache = true
|
||||
binds, mounts := rc.GetBindsAndMounts(t.Context())
|
||||
assert.Subset(t, binds, []string{hostEnv.Path + ":" + hostEnv.Path, hostEnv.GetActPath() + ":" + hostEnv.GetActPath()})
|
||||
assert.Contains(t, mounts, sharedToolCacheVolume)
|
||||
assert.Equal(t, binds, container.SanitizeBinds(t.Context(), rc.validVolumes(), binds))
|
||||
}
|
||||
|
||||
func TestCleanupJobResourcesCleansServicesWithoutJobContainer(t *testing.T) {
|
||||
@@ -730,12 +761,27 @@ func TestCleanupJobVolumesReapsAbandonedDeferredCleanup(t *testing.T) {
|
||||
assert.ElementsMatch(t, []string{"became-active", "remove-failed", "foreign", "fresh", "unknown-age"}, slices.Collect(maps.Keys(volumes)))
|
||||
}
|
||||
|
||||
func TestRemoveDockerActionImagesByTag(t *testing.T) {
|
||||
var removed []string
|
||||
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodDelete {
|
||||
removed = append(removed, strings.TrimPrefix(request.URL.Path, "/v1.47/images/"))
|
||||
_, _ = io.WriteString(writer, "[]")
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(writer, `{"Id":"sha256:shared"}`)
|
||||
})
|
||||
(&RunContext{dockerActionImages: map[*model.Step]string{{}: "act-action-dockeraction:latest-job"}}).removeDockerActionImages(t.Context())
|
||||
assert.Equal(t, []string{"act-action-dockeraction:latest-job"}, removed)
|
||||
}
|
||||
|
||||
func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
t.Setenv("TMPDIR", "/tmp")
|
||||
proxyDir := t.TempDir()
|
||||
for _, name := range []string{"synchronous", "proxy", "closed proxy", "deferred", "preclean deferred", "canceled"} {
|
||||
for _, name := range []string{"synchronous", "proxy", "closed proxy", "kept proxy", "deferred", "preclean deferred", "canceled"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
proxy, preclean, deferred := strings.HasSuffix(name, "proxy"), strings.HasPrefix(name, "preclean"), strings.HasSuffix(name, "deferred")
|
||||
removesLabelled := (proxy || preclean) && name != "kept proxy"
|
||||
if proxy && runtime.GOOS == "windows" {
|
||||
t.Skip("Unix socket ownership is unavailable on Windows")
|
||||
}
|
||||
@@ -745,7 +791,7 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
service.On("Remove").Return(func(context.Context) error { return removeError }).Once()
|
||||
service.On("Close").Return(func(context.Context) error { return closeError }).Once()
|
||||
rc := &RunContext{
|
||||
Config: &Config{CacheContainer: "cache-container"},
|
||||
Config: &Config{CacheContainer: "cache-container", NoSweep: name == "kept proxy"},
|
||||
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
|
||||
JobContainer: job,
|
||||
serviceContainers: []*serviceContainer{{name: "svc", container: service}},
|
||||
@@ -801,8 +847,8 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, 2, volumeRemovals)
|
||||
}
|
||||
assert.Equal(t, proxy || preclean, strings.Contains(err.Error(), "labelled GET /containers/json"))
|
||||
if proxy || preclean {
|
||||
assert.Equal(t, removesLabelled, strings.Contains(err.Error(), "labelled GET /containers/json"))
|
||||
if removesLabelled {
|
||||
require.ErrorContains(t, err, "labelled GET /networks")
|
||||
require.ErrorContains(t, err, "labelled GET /volumes")
|
||||
}
|
||||
@@ -960,7 +1006,6 @@ func TestGetGithubContextRef(t *testing.T) {
|
||||
func createIfTestRunContext(jobs map[string]*model.Job) *RunContext {
|
||||
rc := &RunContext{
|
||||
Config: &Config{Workdir: ".", PlatformPicker: func([]string) string { return "ubuntu-latest" }},
|
||||
Env: map[string]string{},
|
||||
Run: &model.Run{
|
||||
JobID: "job1",
|
||||
Workflow: &model.Workflow{
|
||||
@@ -1037,9 +1082,12 @@ func TestRunContextIsEnabled(t *testing.T) {
|
||||
// success()
|
||||
rc := createIfTestRunContext(map[string]*model.Job{
|
||||
"job1": createJob(t, `runs-on: ubuntu-latest
|
||||
env:
|
||||
BROKEN: ${{ fromJSON('invalid') }}
|
||||
if: success()`, ""),
|
||||
})
|
||||
assertObject.True(rc.isEnabled(context.Background()))
|
||||
require.Error(t, evaluateJobEnvAndDefaults(context.Background(), rc))
|
||||
|
||||
rc = createIfTestRunContext(map[string]*model.Job{
|
||||
"job1": createJob(t, `runs-on: ubuntu-latest`, "failure"),
|
||||
@@ -1629,10 +1677,13 @@ func createRunsOnRunContext(t *testing.T, runsOn string) *RunContext {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunContextContainerImageCannotSelectHost(t *testing.T) {
|
||||
func TestRunContextContainerImageCannotSelectBackend(t *testing.T) {
|
||||
rc := createIfTestRunContext(map[string]*model.Job{"job1": createJob(t, "container: -self-hosted", "")})
|
||||
require.NoError(t, rc.resolvePlatformImage(t.Context()))
|
||||
assert.False(t, rc.IsHostEnv())
|
||||
|
||||
rc = createIfTestRunContext(map[string]*model.Job{"job1": createJob(t, "runs-on: ${{ fromJSON('x') }}\ncontainer: node", "")})
|
||||
require.ErrorContains(t, rc.resolvePlatformImage(t.Context()), "runs-on")
|
||||
}
|
||||
|
||||
func TestRunContextImageOS(t *testing.T) {
|
||||
|
||||
@@ -7,8 +7,10 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -75,13 +77,17 @@ type Config struct {
|
||||
// AppURL. It is never set for github.com or a GithubMirror, so the token stays on-instance.
|
||||
DefaultActionInstanceIsSelfHosted bool
|
||||
PlatformPicker func(labels []string) string
|
||||
KubernetesPicker func(labels []string) container.KubernetesOptions
|
||||
InstanceTransport http.RoundTripper
|
||||
JobLoggerLevel *log.Level // the level of job logger
|
||||
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
||||
SharedToolCache bool // one tool cache for all jobs instead of one per job
|
||||
InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance
|
||||
ClientCertFile, ClientKeyFile string // PEM client certificate for the Gitea instance
|
||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
||||
NoSweep bool // keep the containers, networks and volumes a job creates through its docker proxy
|
||||
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
||||
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
||||
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
||||
@@ -172,6 +178,15 @@ func maxParallelFor(strategy *model.Strategy, combinations int) int {
|
||||
|
||||
// NewPlanExecutor ...
|
||||
func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
|
||||
for _, stage := range plan.Stages {
|
||||
for _, run := range stage.Runs {
|
||||
job := run.Job()
|
||||
if runsOn := job.RunsOn(); job.Uses == "" && len(job.Steps) > 0 && (len(runsOn) == 0 || slices.Contains(runsOn, "")) { // Gitea's needs placeholders carry no steps
|
||||
return common.NewErrorExecutor(fmt.Errorf("job %q requires a non-empty 'runs-on'", run.JobID))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
maxJobNameLen := 0
|
||||
|
||||
stagePipeline := make([]common.Executor, 0)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
log "github.com/sirupsen/logrus"
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -151,6 +152,21 @@ func TestGraphWithSomeMissing(t *testing.T) {
|
||||
log.SetOutput(out)
|
||||
}
|
||||
|
||||
func TestNewPlanExecutorRequiresRunsOnBeforeRunningAnyJob(t *testing.T) {
|
||||
for _, runsOn := range []string{"", "if: false", "runs-on: [ubuntu-latest, '']"} {
|
||||
workflow, err := model.ReadWorkflow(strings.NewReader("jobs:\n first:\n runs-on: ubuntu-latest\n steps: [run: echo]\n test:\n needs: first\n " + runsOn + "\n steps: [run: echo]\n"))
|
||||
require.NoError(t, err)
|
||||
plan, err := model.CombineWorkflowPlanner(workflow).PlanJob("test")
|
||||
require.NoError(t, err)
|
||||
require.ErrorContains(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(plan)(t.Context()), `job "test" requires a non-empty 'runs-on'`, runsOn)
|
||||
assert.Empty(t, workflow.Jobs["first"].Result, runsOn)
|
||||
}
|
||||
|
||||
workflow := &model.Workflow{Jobs: map[string]*model.Job{"build": {Result: "success"}}}
|
||||
require.NoError(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "build"}}}}})(t.Context()))
|
||||
assert.Equal(t, "success", workflow.Jobs["build"].Result)
|
||||
}
|
||||
|
||||
func TestGraphEvent(t *testing.T) {
|
||||
planner, err := model.NewWorkflowPlanner("testdata/basic", true)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
@@ -221,7 +237,6 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config
|
||||
ContainerNamePrefix: strings.ReplaceAll(t.Name(), "/", "-") + "-" + testRunID,
|
||||
// 0 would run jobs runtime.NumCPU()-wide, making the network peak machine-dependent
|
||||
MaxParallel: 2,
|
||||
ForceRebuild: true,
|
||||
Env: cfg.Env,
|
||||
Secrets: cfg.Secrets,
|
||||
GitHubInstance: "github.com",
|
||||
@@ -434,6 +449,8 @@ func TestRunEventHostEnvironment(t *testing.T) {
|
||||
|
||||
tables = append(tables, []TestJobFileInfo{
|
||||
{workdir, "nix-prepend-path", "push", "", platforms, secrets},
|
||||
{workdir, "uses-docker-url", "push", "", platforms, secrets},
|
||||
{workdir, "local-action-dockerfile", "push", "", platforms, secrets},
|
||||
{workdir, "builtin-checkout", "push", "", platforms, secrets},
|
||||
{workdir, "inputs-via-env-context", "push", "", platforms, secrets},
|
||||
{workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets},
|
||||
@@ -451,6 +468,8 @@ func TestRunEventHostEnvironment(t *testing.T) {
|
||||
requireHostTools(t, "pwsh")
|
||||
case "nix-prepend-path":
|
||||
requireHostTools(t, "nix")
|
||||
case "uses-docker-url", "local-action-dockerfile":
|
||||
requireDockerSharingFiles(t)
|
||||
}
|
||||
t.Parallel()
|
||||
hostPlanSlots <- struct{}{}
|
||||
|
||||
@@ -38,6 +38,7 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
defer rawLogger.Infof("::endgroup::")
|
||||
|
||||
rc := sab.RunContext
|
||||
rc.ApplyExtraPath(ctx, &sab.env)
|
||||
return sab.run(ctx, &action.Context{
|
||||
Container: rc.JobContainer,
|
||||
Github: rc.getGithubContext(ctx),
|
||||
@@ -48,6 +49,9 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
NoSkipCheckout: rc.Config.NoSkipCheckout,
|
||||
UseGitIgnore: rc.Config.UseGitIgnore,
|
||||
InsecureSkipTLS: rc.Config.InsecureSkipTLS,
|
||||
ClientCertFile: rc.Config.ClientCertFile,
|
||||
ClientKeyFile: rc.Config.ClientKeyFile,
|
||||
Env: sab.env,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -61,11 +61,14 @@ func (sal *stepActionLocal) main() common.Executor {
|
||||
treader := tar.NewReader(tars)
|
||||
header, err := treader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
tars.Close()
|
||||
return nil, nil, os.ErrNotExist
|
||||
} else if err != nil {
|
||||
tars.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
tars.Close()
|
||||
spath, err = symlinkJoin(spath, header.Linkname, containerActionPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
)
|
||||
|
||||
type stepActionRemote struct {
|
||||
@@ -68,36 +67,22 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
actionDir := sar.actionDir()
|
||||
defaultActionURL := sar.RunContext.Config.DefaultActionURL()
|
||||
// For Gitea
|
||||
// A composite RunContext nils Config.Secrets, so getGitCloneToken would yield an
|
||||
// empty token and clone the action anonymously (401 against the authenticated
|
||||
// instance). github.Token survives the composite config copy and matches the
|
||||
// top-level token; keep the shouldCloneURLUseToken host gate to avoid leaking it.
|
||||
cloneURL := sar.remoteAction.CloneURL(defaultActionURL)
|
||||
token := ""
|
||||
if shouldCloneURLUseToken(sar.RunContext.Config.GitHubInstance, sar.RunContext.Config.trustedActionInstance(), cloneURL) {
|
||||
token = github.Token
|
||||
}
|
||||
gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
URL: cloneURL,
|
||||
// github.Token, unlike Config.GetToken, survives the composite config copy, which nils Config.Secrets.
|
||||
gitClone := stepActionRemoteNewCloneExecutor(sar.RunContext.Config.withInstanceAuth(git.NewGitCloneExecutorInput{
|
||||
URL: sar.remoteAction.CloneURL(defaultActionURL),
|
||||
Ref: sar.remoteAction.Ref,
|
||||
Dir: actionDir,
|
||||
Token: token,
|
||||
OfflineMode: sar.RunContext.Config.ActionOfflineMode,
|
||||
Depth: sar.RunContext.Config.ActionCloneDepth,
|
||||
// printPrepareActions reports the download with its resolved commit.
|
||||
Quiet: true,
|
||||
|
||||
InsecureSkipTLS: sar.cloneSkipTLS(), // For Gitea
|
||||
})
|
||||
var ntErr common.Executor
|
||||
}, github.Token))
|
||||
if err := gitClone(ctx); err != nil {
|
||||
var refErr *git.Error
|
||||
switch {
|
||||
case errors.As(err, &refErr) && errors.Is(err, git.ErrShortRef):
|
||||
return fmt.Errorf("unable to resolve action `%s`, the provided ref `%s` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `%s` instead",
|
||||
sar.Step.Uses, sar.remoteAction.Ref, refErr.Commit())
|
||||
case errors.Is(err, gogit.ErrForceNeeded): // TODO: figure out if it will be easy to shadow/alias go-git err's
|
||||
ntErr = common.NewInfoExecutor("Non-terminating error while running 'git clone': %v", err)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
@@ -115,15 +100,10 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
return f, f, err
|
||||
}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
ntErr,
|
||||
func(ctx context.Context) error {
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile)
|
||||
sar.action = actionModel
|
||||
return err
|
||||
},
|
||||
)(ctx)
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
actionModel, err := sar.readAction(ctx, sar.Step, actionDir, sar.remoteAction.Path, remoteReader, os.WriteFile)
|
||||
sar.action = actionModel
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,22 +239,6 @@ func (sar *stepActionRemote) getCompositeSteps() *compositeSteps {
|
||||
return sar.compositeSteps
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// cloneSkipTLS returns true if the runner can clone an action from the Gitea instance
|
||||
func (sar *stepActionRemote) cloneSkipTLS() bool {
|
||||
if !sar.RunContext.Config.InsecureSkipTLS {
|
||||
// Return false if the Gitea instance is not an insecure instance
|
||||
return false
|
||||
}
|
||||
if sar.remoteAction.URL == "" {
|
||||
// Empty URL means the default action instance should be used
|
||||
// Return true if the URL of the Gitea instance is the same as the URL of the default action instance
|
||||
return sar.RunContext.Config.DefaultActionURL() == sar.RunContext.Config.GitHubInstance
|
||||
}
|
||||
// Return true if the URL of the remote action is the same as the URL of the Gitea instance
|
||||
return sar.remoteAction.URL == sar.RunContext.Config.GitHubInstance
|
||||
}
|
||||
|
||||
type remoteAction struct {
|
||||
URL string
|
||||
Org string
|
||||
|
||||
@@ -56,6 +56,9 @@ func (sd *stepDocker) runUsesContainer() common.Executor {
|
||||
step := sd.Step
|
||||
|
||||
return func(ctx context.Context) error {
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
}
|
||||
image := strings.TrimPrefix(step.Uses, "docker://")
|
||||
cmd, err := shellquote.Split(step.With["args"])
|
||||
if err != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -151,22 +152,35 @@ func TestStepDockerNewStepContainerAllocatePTY(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepDockerNewStepContainerNetworkMode(t *testing.T) {
|
||||
func TestStepDockerNewStepContainerNetworkModeAndPath(t *testing.T) {
|
||||
inheritedPath := (&container.LinuxContainerEnvironmentExtensions{}).DefaultPathVariable()
|
||||
cases := []struct {
|
||||
name string
|
||||
platform string
|
||||
path string
|
||||
expectDefault bool
|
||||
keepsPath bool
|
||||
}{
|
||||
{
|
||||
name: "docker mode attaches to job container network",
|
||||
platform: "node:14",
|
||||
path: inheritedPath,
|
||||
expectDefault: false,
|
||||
keepsPath: true,
|
||||
},
|
||||
{
|
||||
name: "host mode uses default network",
|
||||
name: "host mode uses default network and the image's PATH",
|
||||
platform: "-self-hosted",
|
||||
path: inheritedPath,
|
||||
expectDefault: true,
|
||||
},
|
||||
{
|
||||
name: "host mode keeps a PATH that is not the runner's",
|
||||
platform: "-self-hosted",
|
||||
path: "/custom/bin",
|
||||
expectDefault: true,
|
||||
keepsPath: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -208,6 +222,7 @@ func TestStepDockerNewStepContainerNetworkMode(t *testing.T) {
|
||||
ID: "1",
|
||||
Uses: "docker://alpine:3.20",
|
||||
},
|
||||
env: map[string]string{"PATH": tc.path},
|
||||
}
|
||||
sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx)
|
||||
|
||||
@@ -215,6 +230,7 @@ func TestStepDockerNewStepContainerNetworkMode(t *testing.T) {
|
||||
assert.Equal(t, tc.expectDefault, sd.RunContext.IsHostEnv(), "IsHostEnv mismatch for platform %q", tc.platform)
|
||||
|
||||
newStepContainer(ctx, sd, "alpine:3.20", []string{"echo", "hello"}, nil, "")
|
||||
assert.Equal(t, tc.keepsPath, slices.Contains(captured.Env, "PATH="+tc.path))
|
||||
|
||||
if tc.expectDefault {
|
||||
assert.Equal(t, "default", captured.NetworkMode,
|
||||
|
||||
+5
-2
@@ -5,6 +5,7 @@ jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: printf 'workspace=from-host-step\n' > workspace-marker
|
||||
- uses: docker://node:24-bookworm-slim
|
||||
with:
|
||||
somekey: somevalue
|
||||
@@ -16,8 +17,10 @@ jobs:
|
||||
with:
|
||||
entrypoint: /bin/sh
|
||||
args: -c -- uname -a
|
||||
- uses: docker://node:24-bookworm-slim
|
||||
- id: workspace
|
||||
uses: docker://node:24-bookworm-slim
|
||||
with:
|
||||
entrypoint: /bin/sh
|
||||
args: -c 'uname -a'
|
||||
args: -c 'uname -a && cat "$GITHUB_WORKSPACE/workspace-marker" >> "$GITHUB_OUTPUT"'
|
||||
- run: test '${{ steps.workspace.outputs.workspace }}' = 'from-host-step'
|
||||
|
||||
+22
-4
@@ -8,14 +8,32 @@ Built-in actions ship with the runner, so a job needs no action download and no
|
||||
|
||||
## checkout
|
||||
|
||||
Works like `actions/checkout` with these inputs. Any other input fails the step.
|
||||
Works like `actions/checkout` with these inputs. Any other input fails the step. It runs `git` where the job runs, see the [prerequisites](../README.md#prerequisites).
|
||||
|
||||
| Input | Default |
|
||||
| --- | --- |
|
||||
| `repository` | `github.repository` |
|
||||
| `ref` | the triggering commit, or the default branch of another repository |
|
||||
| `ref` | the triggering ref and commit, or the default branch of another repository |
|
||||
| `token` | `github.token` |
|
||||
| `ssh-key` | empty, a private key fetches over SSH instead |
|
||||
| `ssh-known-hosts` | empty, host keys trusted besides `~/.ssh/known_hosts` |
|
||||
| `ssh-strict` | `true`, rejects unknown host keys |
|
||||
| `ssh-user` | the user of the repository's SSH URL, else `git` |
|
||||
| `persist-credentials` | `true`, later steps of the job can fetch and push with `token` or `ssh-key` |
|
||||
| `path` | workspace root |
|
||||
| `fetch-depth` | `1`, `0` for full history |
|
||||
| `clean` | `true`, removes untracked files, also in submodules |
|
||||
| `filter` | empty, a partial clone filter such as `blob:none` |
|
||||
| `sparse-checkout` | empty, the paths to check out, one per line |
|
||||
| `sparse-checkout-cone-mode` | `true`, `false` reads `sparse-checkout` as patterns |
|
||||
| `fetch-depth` | `1`, `0` for the full history of every branch and tag |
|
||||
| `fetch-tags` | `false`, `true` fetches every tag |
|
||||
| `show-progress` | `true`, shows the fetch progress |
|
||||
| `lfs` | `false`, `true` downloads Git LFS files |
|
||||
| `submodules` | `false`, `true` checks out submodules, `recursive` also theirs |
|
||||
| `set-safe-directory` | `true`, lets checkout's `git` work in a directory another user owns |
|
||||
|
||||
With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` is set.
|
||||
A branch is checked out as a local branch tracking `origin`, a tag or commit as a detached `HEAD`. A triggering tag that moved since the event fails the step. The step outputs `ref` and `commit` like `actions/checkout`.
|
||||
|
||||
Persisted credentials end with the job. `token` is only sent to `github.server_url`, while `ssh-key` is offered to any SSH host, as with `actions/checkout`. The runner's `client_cert_file` is only offered to `github.server_url` and only during the step, so later steps do not get it. Without `ssh-key`, submodule URLs of the form `git@<server host>:` are fetched from `github.server_url` with `token`. With `ssh-key`, the repository is fetched from the SSH URL in the event's `repository.ssh_url`, or from the server's host on port 22.
|
||||
|
||||
With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` or `ssh-key` is set.
|
||||
@@ -0,0 +1,58 @@
|
||||
# Jobs as Kubernetes pods
|
||||
|
||||
A `kubernetes` label runs each job as a pod, without a Docker daemon:
|
||||
|
||||
```text
|
||||
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
|
||||
The pod has a `job` container from the label's image or the job's `container.image`, plus one container per service.
|
||||
|
||||
## Setup
|
||||
|
||||
In a cluster, the runner uses its pod's service account and namespace. [`examples/kubernetes/kubernetes-pods.yaml`](../examples/kubernetes/kubernetes-pods.yaml) deploys a runner with these permissions:
|
||||
|
||||
| Resource | Verbs |
|
||||
| --- | --- |
|
||||
| `pods` | `create`, `get`, `list`, `delete` |
|
||||
| `pods/exec` | `create`, `get` |
|
||||
| `pods/log` | `get` |
|
||||
| `secrets` | `create`, `delete` |
|
||||
|
||||
Outside a cluster, it uses the current context of the first file in `$KUBECONFIG`, or of `~/.kube/config`, which must authenticate with a token, token file or client certificate. The cluster must run Kubernetes 1.30 or later, and job images need `sh`, `tar`, `tail` and `env`.
|
||||
|
||||
## Configuration
|
||||
|
||||
```yaml
|
||||
kubernetes:
|
||||
kubeconfig: "" # instead of the service account
|
||||
namespace: "" # defaults to the service account's, the kubeconfig context's, or default
|
||||
pod_template: # merged under each job pod, a container named job merges into the job container, one named $services into each service container
|
||||
spec:
|
||||
containers:
|
||||
- name: job
|
||||
resources:
|
||||
limits:
|
||||
memory: 4Gi
|
||||
- name: $services
|
||||
resources:
|
||||
limits:
|
||||
memory: 1Gi
|
||||
pod_templates: # merged over pod_template for jobs whose runs-on has the label they are keyed by
|
||||
gpu:
|
||||
spec:
|
||||
nodeSelector:
|
||||
nvidia.com/gpu.present: "true"
|
||||
```
|
||||
|
||||
A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. Like every config value, templates expand `${NAME}` from the runner's environment, so a shell command in a template writes `$${NAME}` for the pod's own variable.
|
||||
|
||||
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
||||
|
||||
## Services
|
||||
|
||||
Services share the pod's network, so a step reaches one by its id or `localhost`, and two services cannot listen on the same port. An id is lowercased and has characters other than letters, digits and `-` replaced by `-`, so the service `my_db` is reached as `my-db`. Pass secrets to services through `env`, as `command` is visible in the pod spec. Of `options`, only `--health-*` is supported. They become a startup probe with Docker's defaults, which stops a service that fails its retries, so the job fails as on Docker.
|
||||
|
||||
## Unsupported
|
||||
|
||||
Docker actions, `docker://` steps, and the `options`, `volumes` and `credentials` of the job container fail the job, as do `volumes` and `credentials` of services. Run such workflows on a [Docker in Docker](../examples/kubernetes/README.md) runner. Pull private images with `imagePullSecrets` in `pod_template`.
|
||||
+33
-11
@@ -1,6 +1,6 @@
|
||||
# OpenTelemetry tracing
|
||||
# OpenTelemetry
|
||||
|
||||
The runner can send a trace of every job it runs to an OpenTelemetry collector or any backend that accepts OTLP. Each job becomes a trace with a span per step, so you can see where CI time goes and which steps fail. Tracing stays off until you set an endpoint.
|
||||
The runner can send a trace of every job it runs, and its metrics, to an OpenTelemetry collector or any backend that accepts OTLP. Each job becomes a trace with a span per step, so you can see where CI time goes and which steps fail. Export stays off until you set an endpoint.
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -12,20 +12,26 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
|
||||
| Variable | Purpose |
|
||||
| --- | --- |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Collector base URL, traces go to `/v1/traces` under it |
|
||||
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Full traces URL, takes precedence over the base URL |
|
||||
| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | Request headers such as an API key, as comma-separated `key=value` pairs with URL-encoded values |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Collector base URL, traces go to `/v1/traces` and metrics to `/v1/metrics` under it |
|
||||
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Full URL per signal, takes precedence over the base URL |
|
||||
| `OTEL_EXPORTER_OTLP_HEADERS` | Request headers such as an API key, as comma-separated `key=value` pairs with URL-encoded values |
|
||||
| `OTEL_EXPORTER_OTLP_TIMEOUT` | Export timeout in milliseconds, default 10000, 0 for none |
|
||||
| `OTEL_EXPORTER_OTLP_COMPRESSION` | `gzip` or `none`, default `none` |
|
||||
| `OTEL_EXPORTER_OTLP_CERTIFICATE` | PEM file of the CA to trust instead of the system ones |
|
||||
| `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY` | PEM files of the client certificate and key for mTLS |
|
||||
| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, the service name defaults to `gitea-runner` |
|
||||
| `OTEL_TRACES_SAMPLER`, `OTEL_BSP_*` | Sampling and batching, see the [SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) |
|
||||
| `OTEL_SDK_DISABLED=true`, `OTEL_TRACES_EXPORTER=none` | Turn tracing off |
|
||||
| `OTEL_METRIC_EXPORT_INTERVAL`, `OTEL_METRIC_EXPORT_TIMEOUT` | Metrics export interval and timeout in milliseconds, default 60000 and 30000, a timeout of 0 for none |
|
||||
| `OTEL_SDK_DISABLED=true` | Turn all export off |
|
||||
| `OTEL_TRACES_EXPORTER=none`, `OTEL_METRICS_EXPORTER=none` | Turn traces or metrics off |
|
||||
|
||||
Only the `http/protobuf` protocol is supported.
|
||||
Except for the endpoint, each `OTEL_EXPORTER_OTLP_*` variable has `TRACES` and `METRICS` variants such as `OTEL_EXPORTER_OTLP_METRICS_HEADERS`, which take precedence for that signal. Only the `otlp` exporter over the `http/protobuf` protocol is supported.
|
||||
|
||||
## What is sent
|
||||
|
||||
- A span per job, named `RUN {workflow} {job}`, with the repository, ref, commit, run URL, run number, attempt and result.
|
||||
- A span per step, named as in Gitea's step list, including `Pre` and `Post` stages and the steps of composite actions, with the step's result.
|
||||
- The runner's name, UUID and version.
|
||||
- The runner's name, UUID, version, process ID and start time, and the Go version.
|
||||
|
||||
Logs, step output and secrets are never sent. Step names are taken from the workflow file as written, without expanding `${{ }}` expressions.
|
||||
|
||||
@@ -35,12 +41,28 @@ Jobs receive `TRACEPARENT`, so tools in a job that use OpenTelemetry can add the
|
||||
|
||||
## Metrics
|
||||
|
||||
Metrics stay on the Prometheus endpoint enabled by `metrics.enabled`. To get them into OpenTelemetry, scrape `/metrics` with the collector's `prometheus` receiver.
|
||||
The metrics of the Prometheus endpoint are also exported, whether or not `metrics.enabled` is set, except `gitea_runner_info`, `go_info` and `process_start_time_seconds`, which are sent as resource attributes. They are sent every export interval and once more on shutdown. To turn them off, set `OTEL_METRICS_EXPORTER=none`.
|
||||
|
||||
Metrics are named after the [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) where one exists, and under `gitea.runner.*` otherwise. Other Go and process metrics, such as `go_memstats_heap_alloc_bytes`, keep their Prometheus names.
|
||||
|
||||
| Metric | Attributes |
|
||||
| --- | --- |
|
||||
| `cicd.worker.count` | `cicd.worker.state` |
|
||||
| `cicd.system.errors` | `cicd.system.component`, `rpc.method`, `error.type` |
|
||||
| `gitea.runner.jobs` | `cicd.pipeline.result` |
|
||||
| `gitea.runner.job.active`, `gitea.runner.job.limit`, `gitea.runner.job.utilization`, `gitea.runner.job.duration` | |
|
||||
| `gitea.runner.poll.fetches` | `gitea.runner.poll.result` |
|
||||
| `gitea.runner.uptime`, `gitea.runner.poll.backoff`, `gitea.runner.poll.fetch.duration` | |
|
||||
| `gitea.runner.report.log.requests`, `gitea.runner.report.state.requests` | `gitea.runner.report.result` |
|
||||
| `gitea.runner.report.log.buffer`, `gitea.runner.report.log.duration`, `gitea.runner.report.state.duration` | |
|
||||
| `go.memory.used` | `go.memory.type` |
|
||||
| `go.goroutine.count`, `go.memory.allocated`, `go.memory.gc.goal`, `go.memory.gc.cycles`, `go.memory.limit`, `go.config.gogc`, `go.processor.limit` | |
|
||||
| `process.memory.usage`, `process.memory.virtual`, `process.unix.file_descriptor.count` (`process.windows.handle.count` on Windows) | |
|
||||
|
||||
## Security
|
||||
|
||||
- OTLP has no authentication of its own. Send to a collector on the runner's host that holds the backend credentials, as the [OpenTelemetry security guide](https://opentelemetry.io/docs/security/config-best-practices/) recommends, or use `https` with a token in `OTEL_EXPORTER_OTLP_HEADERS`. Over plain `http`, headers travel unencrypted.
|
||||
- TLS certificates are verified against the system CA bundle, set `SSL_CERT_FILE` to use your own. Client certificates are not supported, use a local collector for mTLS. Redirects are not followed, so headers only reach the configured endpoint.
|
||||
- OTLP has no authentication of its own. Send to a collector on the runner's host that holds the backend credentials, as the [OpenTelemetry security guide](https://opentelemetry.io/docs/security/config-best-practices/) recommends, or use `https` with a token in `OTEL_EXPORTER_OTLP_HEADERS` or a client certificate. Over plain `http`, headers travel unencrypted.
|
||||
- TLS certificates are verified against the system CA bundle, or only against `OTEL_EXPORTER_OTLP_CERTIFICATE` when set. Redirects are not followed, so headers only reach the configured endpoint.
|
||||
- Jobs in containers never see the runner's `OTEL_*` variables. Jobs on the host inherit the runner's whole environment, headers included, and can read its files anyway since they run as the same user.
|
||||
- Every workflow can read `runner.envs`, so never put credentials there. Point jobs at a collector that holds them instead, keeping in mind that any workflow can then send data through it.
|
||||
- Traces carry the repository and step names of every job, private repositories included, so give the collector the same access controls as the runner's logs.
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import "testing"
|
||||
|
||||
func testBuiltinCheckout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
api, repo := newScenario(t)
|
||||
pushWorkflow(t, api, repo, "checkout.yml")
|
||||
branchRun := waitForRun(t, api, repo)
|
||||
if err := api.CreateTag(t.Context(), repo, "v1"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tagRun := waitForRunAfter(t, api, repo, branchRun.ID)
|
||||
requireSuccess(t, api, repo, branchRun.ID)
|
||||
requireSuccess(t, api, repo, tagRun.ID)
|
||||
}
|
||||
@@ -30,7 +30,9 @@ func TestCompatibility(t *testing.T) {
|
||||
t.Run("artifact", testArtifactRoundTrip)
|
||||
t.Run("cancellation_and_log_streaming", testRunCancellation)
|
||||
t.Run("dispatch", testWorkflowDispatch)
|
||||
t.Run("builtin_checkout", testBuiltinCheckout)
|
||||
t.Run("ephemeral", testEphemeralRunner)
|
||||
t.Run("kubernetes", testKubernetes)
|
||||
}
|
||||
|
||||
func testPayloads(t *testing.T) {
|
||||
|
||||
@@ -22,7 +22,7 @@ func testEphemeralRunner(t *testing.T) {
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("ephemeral runner was not deleted")
|
||||
}
|
||||
if !poller.Unregistered() {
|
||||
if !poller.RegistrationRejected() {
|
||||
t.Fatal("ephemeral runner stopped without being deleted")
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,12 @@ func (a *GiteaAPI) CreateFile(ctx context.Context, repo, path, content, message
|
||||
return a.doJSON(ctx, http.MethodPost, url, body, nil)
|
||||
}
|
||||
|
||||
func (a *GiteaAPI) CreateTag(ctx context.Context, repo, name string) error {
|
||||
body := map[string]any{"tag_name": name}
|
||||
url := fmt.Sprintf("/api/v1/repos/%s/%s/tags", giteaAdminUser, repo)
|
||||
return a.doJSON(ctx, http.MethodPost, url, body, nil)
|
||||
}
|
||||
|
||||
func (a *GiteaAPI) CreateSecret(ctx context.Context, repo, name, value string) error {
|
||||
body := map[string]any{"data": value}
|
||||
url := fmt.Sprintf("/api/v1/repos/%s/%s/actions/secrets/%s", giteaAdminUser, repo, name)
|
||||
|
||||
+18
-5
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
apicontainer "github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
@@ -187,6 +188,7 @@ func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, e
|
||||
"GITEA__security__INSTALL_LOCK=true",
|
||||
"GITEA__database__DB_TYPE=sqlite3",
|
||||
"GITEA__actions__ENABLED=true",
|
||||
"GITEA__server__LFS_START_SERVER=true",
|
||||
"GITEA__server__ROOT_URL=" + baseURL + "/",
|
||||
},
|
||||
ExposedPorts: network.PortSet{containerPort: struct{}{}},
|
||||
@@ -306,9 +308,14 @@ func randomToken(n int) string {
|
||||
}
|
||||
|
||||
func (f *GiteaFixture) exec(ctx context.Context, cmd []string) (string, error) {
|
||||
created, err := f.cli.ExecCreate(ctx, f.id, mobyclient.ExecCreateOptions{
|
||||
return containerExec(ctx, f.cli, f.id, "git", nil, cmd...) // gitea refuses root
|
||||
}
|
||||
|
||||
func containerExec(ctx context.Context, cli mobyclient.APIClient, id, user string, stdin io.Reader, cmd ...string) (string, error) {
|
||||
created, err := cli.ExecCreate(ctx, id, mobyclient.ExecCreateOptions{
|
||||
Cmd: cmd,
|
||||
User: "git", // gitea refuses root
|
||||
User: user,
|
||||
AttachStdin: stdin != nil,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
})
|
||||
@@ -316,18 +323,24 @@ func (f *GiteaFixture) exec(ctx context.Context, cmd []string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
attached, err := f.cli.ExecAttach(ctx, created.ID, mobyclient.ExecAttachOptions{})
|
||||
attached, err := cli.ExecAttach(ctx, created.ID, mobyclient.ExecAttachOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer attached.Close()
|
||||
if stdin != nil {
|
||||
go func() {
|
||||
_, _ = io.Copy(attached.Conn, stdin)
|
||||
_ = attached.CloseWrite()
|
||||
}()
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if _, err := io.Copy(&out, attached.Reader); err != nil {
|
||||
if _, err := stdcopy.StdCopy(&out, &out, attached.Reader); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
inspected, err := f.cli.ExecInspect(ctx, created.ID, mobyclient.ExecInspectOptions{})
|
||||
inspected, err := cli.ExecInspect(ctx, created.ID, mobyclient.ExecInspectOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build e2e
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/config"
|
||||
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
apicontainer "github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
func testKubernetes(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := t.Context()
|
||||
k3sID, kubernetes := startK3s(t)
|
||||
|
||||
images := preloadImages(t, k3sID, os.Getenv("E2E_JOB_IMAGE"), os.Getenv("SERVICE_IMAGE"))
|
||||
|
||||
api, repo := newScenario(t)
|
||||
if err := api.CreateVariable(ctx, repo, "E2E_SERVICE_IMAGE", images[1]); err != nil {
|
||||
t.Fatalf("create service image variable: %v", err)
|
||||
}
|
||||
startRunner(t, repo, "e2e-kubernetes", runnerOptions{image: images[0], kubernetes: kubernetes})
|
||||
pushWorkflow(t, api, repo, "kubernetes.yml")
|
||||
|
||||
wfRun := waitForRun(t, api, repo)
|
||||
requireSuccess(t, api, repo, wfRun.ID)
|
||||
|
||||
if out, err := containerExec(ctx, fixture.cli, k3sID, "", nil,
|
||||
"kubectl", "wait", "--for=delete", "pods,secrets", "--selector=app.kubernetes.io/managed-by=gitea-runner", "--timeout=1m",
|
||||
); err != nil {
|
||||
t.Errorf("job pods or secrets left behind: %v: %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func startK3s(t *testing.T) (string, *config.Kubernetes) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
cli := fixture.cli
|
||||
|
||||
name := fmt.Sprintf("gitea-runner-e2e-k3s-%d", time.Now().UnixNano())
|
||||
apiPort := network.MustParsePort("6443/tcp")
|
||||
server := "https://" + name + ":6443"
|
||||
hostConfig := &apicontainer.HostConfig{Privileged: true}
|
||||
var netConfig *network.NetworkingConfig
|
||||
if fixture.network != "" {
|
||||
netConfig = &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{fixture.network: {}}}
|
||||
} else {
|
||||
loopback := netip.MustParseAddr("127.0.0.1")
|
||||
port, err := freeHostPort(loopback.String())
|
||||
if err != nil {
|
||||
t.Fatalf("find a free host port: %v", err)
|
||||
}
|
||||
server = "https://" + net.JoinHostPort(loopback.String(), strconv.Itoa(port))
|
||||
hostConfig.PortBindings = network.PortMap{apiPort: []network.PortBinding{{HostIP: loopback, HostPort: strconv.Itoa(port)}}}
|
||||
}
|
||||
|
||||
created, err := cli.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{
|
||||
Config: &apicontainer.Config{
|
||||
Image: os.Getenv("E2E_K3S_IMAGE"),
|
||||
Cmd: []string{"server", "--disable=traefik,metrics-server,servicelb,local-storage,coredns", "--tls-san=" + name, "--kubelet-arg=fail-cgroupv1=false"},
|
||||
ExposedPorts: network.PortSet{apiPort: struct{}{}},
|
||||
},
|
||||
HostConfig: hostConfig,
|
||||
NetworkingConfig: netConfig,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create k3s container: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = cli.ContainerRemove(context.Background(), created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||
})
|
||||
if _, err := cli.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{}); err != nil {
|
||||
t.Fatalf("start k3s container: %v", err)
|
||||
}
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
for {
|
||||
_, err := containerExec(waitCtx, cli, created.ID, "", nil, "kubectl", "get", "serviceaccount", "default")
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
inspected, inspectErr := cli.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||
if inspectErr != nil || !inspected.Container.State.Running || waitCtx.Err() != nil {
|
||||
var logs bytes.Buffer
|
||||
if reader, logsErr := cli.ContainerLogs(ctx, created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
|
||||
_, _ = stdcopy.StdCopy(&logs, &logs, reader)
|
||||
_ = reader.Close()
|
||||
}
|
||||
t.Fatalf("k3s did not start: %v, state %+v %v, last log lines:\n%s", err, inspected.Container.State, inspectErr, logs.String())
|
||||
}
|
||||
select {
|
||||
case <-waitCtx.Done():
|
||||
case <-time.After(pollInterval):
|
||||
}
|
||||
}
|
||||
|
||||
kubeconfig, err := containerExec(ctx, cli, created.ID, "", nil, "cat", "/etc/rancher/k3s/k3s.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read kubeconfig: %v", err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
if err := os.WriteFile(path, []byte(strings.ReplaceAll(kubeconfig, "https://127.0.0.1:6443", server)), 0o600); err != nil {
|
||||
t.Fatalf("write kubeconfig: %v", err)
|
||||
}
|
||||
kubernetes := &config.Kubernetes{Kubeconfig: path, Namespace: "default"}
|
||||
|
||||
if fixture.network != "" {
|
||||
gitea, err := cli.ContainerInspect(ctx, fixture.id, mobyclient.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("inspect gitea container: %v", err)
|
||||
}
|
||||
giteaURL, err := url.Parse(fixture.baseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse gitea url: %v", err)
|
||||
}
|
||||
kubernetes.PodTemplate = map[string]any{"spec": map[string]any{"hostAliases": []any{map[string]any{
|
||||
"ip": gitea.Container.NetworkSettings.Networks[fixture.network].IPAddress.String(),
|
||||
"hostnames": []any{giteaURL.Hostname()},
|
||||
}}}}
|
||||
}
|
||||
return created.ID, kubernetes
|
||||
}
|
||||
|
||||
func preloadImages(t *testing.T, k3sID string, images ...string) []string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
var aliases []string
|
||||
for index, image := range images {
|
||||
alias := fmt.Sprintf("gitea-runner-e2e/preload-%d:%d", index, time.Now().UnixNano())
|
||||
if _, err := fixture.cli.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
|
||||
t.Fatalf("tag %s: %v", image, err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = fixture.cli.ImageRemove(context.Background(), alias, mobyclient.ImageRemoveOptions{}) })
|
||||
aliases = append(aliases, alias)
|
||||
}
|
||||
archive, err := fixture.cli.ImageSave(ctx, aliases)
|
||||
if err != nil {
|
||||
t.Fatalf("save %v: %v", images, err)
|
||||
}
|
||||
defer archive.Close()
|
||||
if out, err := containerExec(ctx, fixture.cli, k3sID, "", archive, "ctr", "--namespace=k8s.io", "images", "import", "-"); err != nil {
|
||||
t.Fatalf("import %v into k3s: %v: %s", images, err, out)
|
||||
}
|
||||
return aliases
|
||||
}
|
||||
+6
-1
@@ -141,6 +141,11 @@ func dumpRunLogs(t *testing.T, api *GiteaAPI, repo string, runID int64) {
|
||||
}
|
||||
|
||||
func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun {
|
||||
t.Helper()
|
||||
return waitForRunAfter(t, api, repo, 0)
|
||||
}
|
||||
|
||||
func waitForRunAfter(t *testing.T, api *GiteaAPI, repo string, previousID int64) *ActionRun {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
|
||||
defer cancel()
|
||||
@@ -150,7 +155,7 @@ func waitForRun(t *testing.T, api *GiteaAPI, repo string) *ActionRun {
|
||||
if err != nil {
|
||||
t.Fatalf("get latest run: %v", err)
|
||||
}
|
||||
if len(runs) > 0 {
|
||||
if len(runs) > 0 && runs[0].ID > previousID {
|
||||
return &runs[0]
|
||||
}
|
||||
select {
|
||||
|
||||
+14
-7
@@ -6,6 +6,7 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
@@ -24,10 +25,12 @@ import (
|
||||
)
|
||||
|
||||
type runnerOptions struct {
|
||||
capacity int
|
||||
ephemeral bool
|
||||
cacheV2 *bool
|
||||
cacheHost string
|
||||
capacity int
|
||||
ephemeral bool
|
||||
cacheV2 *bool
|
||||
cacheHost string
|
||||
image string
|
||||
kubernetes *config.Kubernetes
|
||||
}
|
||||
|
||||
func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *poll.Poller {
|
||||
@@ -61,14 +64,18 @@ func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *p
|
||||
cfg.Container.Network = fixture.network
|
||||
}
|
||||
|
||||
rawLabel := labelName + ":docker://" + os.Getenv("E2E_JOB_IMAGE")
|
||||
scheme := labels.SchemeDocker
|
||||
if options.kubernetes != nil {
|
||||
scheme, cfg.Kubernetes = labels.SchemeKubernetes, *options.kubernetes
|
||||
}
|
||||
rawLabel := labelName + ":" + scheme + "://" + cmp.Or(options.image, os.Getenv("E2E_JOB_IMAGE"))
|
||||
label, err := labels.Parse(rawLabel)
|
||||
if err != nil {
|
||||
t.Fatalf("parse label %q: %v", rawLabel, err)
|
||||
}
|
||||
labelNames := []string{label.Name}
|
||||
|
||||
pingCli := client.New(fixture.baseURL, cfg.Runner.Insecure, "", "", config.RequestTimeout)
|
||||
pingCli := client.New(fixture.baseURL, cfg.Runner.TLSConfig(), "", "", config.RequestTimeout, cfg.Runner.ExtraHeaders)
|
||||
if _, err := pingCli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{Data: t.Name()})); err != nil {
|
||||
t.Fatalf("ping %s: %v", fixture.baseURL, err)
|
||||
}
|
||||
@@ -98,7 +105,7 @@ func startRunner(t *testing.T, repo, labelName string, options runnerOptions) *p
|
||||
Ephemeral: regResp.Msg.Runner.Ephemeral,
|
||||
}
|
||||
|
||||
cli := client.New(fixture.baseURL, cfg.Runner.Insecure, reg.UUID, reg.Token, config.RequestTimeout)
|
||||
cli := client.New(fixture.baseURL, cfg.Runner.TLSConfig(), reg.UUID, reg.Token, config.RequestTimeout, cfg.Runner.ExtraHeaders)
|
||||
|
||||
runner := run.NewRunner(cfg, reg, cli)
|
||||
declResp, err := runner.Declare(ctx, labelNames)
|
||||
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
name: checkout
|
||||
on: push
|
||||
jobs:
|
||||
checkout:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- id: checkout
|
||||
uses: builtin:checkout
|
||||
- uses: builtin:checkout
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
path: short
|
||||
- run: test "$(git -C short rev-parse HEAD)" = "$GITHUB_SHA"
|
||||
- if: github.ref_type == 'branch'
|
||||
run: |
|
||||
test "$(git symbolic-ref HEAD)" = "$GITHUB_REF"
|
||||
test "${{ steps.checkout.outputs.ref }}" = "$GITHUB_REF"
|
||||
test "${{ steps.checkout.outputs.commit }}" = "$GITHUB_SHA"
|
||||
git -c user.name=e2e -c [email protected] commit --allow-empty -m pushed
|
||||
git push
|
||||
git tag moved
|
||||
git push --force origin moved
|
||||
git switch --orphan child
|
||||
echo child > child.txt
|
||||
git add child.txt
|
||||
git -c user.name=e2e -c [email protected] commit -m child
|
||||
git push origin child
|
||||
git switch -c content main
|
||||
git submodule add -b child "../${{ github.event.repository.name }}.git" sub
|
||||
git lfs install --local
|
||||
git lfs track '*.bin'
|
||||
mkdir keep drop
|
||||
echo big > keep/big.bin
|
||||
echo big > drop/big.bin
|
||||
git add .gitattributes keep drop
|
||||
git -c user.name=e2e -c [email protected] commit -m content
|
||||
git push origin content
|
||||
- if: github.ref_type == 'branch'
|
||||
uses: builtin:checkout
|
||||
with:
|
||||
ref: content
|
||||
path: full
|
||||
submodules: true
|
||||
lfs: true
|
||||
- if: github.ref_type == 'branch'
|
||||
uses: builtin:checkout
|
||||
with:
|
||||
ref: content
|
||||
path: sparse
|
||||
sparse-checkout: keep
|
||||
lfs: true
|
||||
- if: github.ref_type == 'branch'
|
||||
run: |
|
||||
test "$(cat full/sub/child.txt)" = child
|
||||
test "$(cat full/keep/big.bin)" = big
|
||||
test "$(cat sparse/keep/big.bin)" = big
|
||||
test ! -e sparse/drop
|
||||
- if: github.ref_type == 'tag'
|
||||
run: test "$(git describe --tags)" = "$GITHUB_REF_NAME"
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
name: kubernetes
|
||||
on: push
|
||||
jobs:
|
||||
pod:
|
||||
runs-on: e2e-kubernetes
|
||||
services:
|
||||
web:
|
||||
image: ${{ vars.E2E_SERVICE_IMAGE }}
|
||||
env:
|
||||
NGINX_ENTRYPOINT_QUIET_LOGS: 1
|
||||
options: --health-cmd "wget -q -O /dev/null http://localhost/" --health-interval 1s
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: |
|
||||
test -f .gitea/workflows/kubernetes.yml
|
||||
echo "FROM_ENV=exported" >> "$GITHUB_ENV"
|
||||
- run: |
|
||||
test "$FROM_ENV" = exported
|
||||
test -n "$KUBERNETES_SERVICE_HOST" -a -n "${{ runner.arch }}"
|
||||
curl -fsS -o /dev/null http://web/
|
||||
@@ -1,5 +1,7 @@
|
||||
### Running `gitea-runner` using `docker-compose`
|
||||
|
||||
This uses the recommended Docker-in-Docker image, see [image flavours](../../README.md#image-flavours).
|
||||
|
||||
```yml
|
||||
...
|
||||
gitea:
|
||||
@@ -19,8 +21,9 @@
|
||||
# - GITEA_RUNNER_REGISTRATION_TOKEN=<user-defined registration token>
|
||||
|
||||
runner:
|
||||
image: gitea/runner
|
||||
image: gitea/runner:latest-dind
|
||||
restart: always
|
||||
privileged: true
|
||||
depends_on:
|
||||
gitea:
|
||||
# required so runner can attach to gitea, see "healthcheck"
|
||||
@@ -28,7 +31,6 @@
|
||||
restart: true
|
||||
volumes:
|
||||
- ./data/runner:/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
- GITEA_INSTANCE_URL=<instance url>
|
||||
# When using Docker Secrets, it's also possible to use
|
||||
@@ -38,10 +40,9 @@
|
||||
- GITEA_RUNNER_REGISTRATION_TOKEN=<registration token>
|
||||
```
|
||||
|
||||
### Running `gitea-runner` using Docker-in-Docker (DIND)
|
||||
### Running `gitea-runner` using rootless Docker-in-Docker
|
||||
|
||||
- `privileged` has to be set to `true` because in-container Docker daemon requires a lot of kernel capabilities and file system mounts like `procfs` and `sysfs`
|
||||
- `security_opt` sets the `apparmor` profile to `rootlesskit` for hosts running AppArmor (e.g. Ubuntu, Debian), where the kernel might otherwise block user namespace changes that Docker daemon requires for startup. The `rootlesskit` profile is provided by the `docker-ce-rootless-extras` package and is present on hosts where Docker was installed via the official installer or distro packages
|
||||
On AppArmor hosts such as Ubuntu and Debian, `security_opt` is needed for the daemon to start. Docker's `docker-ce-rootless-extras` package provides the `rootlesskit` profile.
|
||||
|
||||
```yml
|
||||
...
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
### Run `gitea-runner` in a Docker Container
|
||||
|
||||
The recommended Docker-in-Docker image runs its own Docker daemon for jobs, so they never get the host's Docker socket:
|
||||
|
||||
```sh
|
||||
docker run -e GITEA_INSTANCE_URL=http://192.168.8.18:3000 -e GITEA_RUNNER_REGISTRATION_TOKEN=<runner_token> -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/data:/data --name my_runner gitea/runner:nightly
|
||||
docker run --privileged \
|
||||
-e GITEA_INSTANCE_URL=http://192.168.8.18:3000 \
|
||||
-e GITEA_RUNNER_REGISTRATION_TOKEN=<runner_token> \
|
||||
-v "$PWD/data:/data" \
|
||||
-v runner-docker:/var/lib/docker \
|
||||
--name my_runner gitea/runner:nightly-dind
|
||||
```
|
||||
|
||||
The `/data` directory inside the docker container contains the runner API keys after registration.
|
||||
It must be persisted, otherwise the runner would try to register again, using the same, now defunct registration token.
|
||||
The volumes keep the runner's registration and the job image cache. See [image flavours](../../README.md#image-flavours) for alternatives.
|
||||
@@ -1,25 +1,8 @@
|
||||
## Kubernetes Docker in Docker Deployment with `gitea-runner`
|
||||
## Kubernetes deployments of `gitea-runner`
|
||||
|
||||
NOTE: Docker in Docker (dind) requires elevated privileges on Kubernetes. The current way to achieve this is to set the pod `SecurityContext` to `privileged`. Keep in mind that this is a potential security issue that has the potential for a malicious application to break out of the container context.
|
||||
Jobs can run on Kubernetes in two ways:
|
||||
|
||||
NOTE: `dind-docker.yaml` uses the native sidecar pattern (init container with `restartPolicy: Always`), which requires Kubernetes 1.29+ (or 1.28 with the `SidecarContainers` feature gate).
|
||||
- **Job pods**: each job runs as its own pod, without privileged containers and under the cluster's quotas and policies. Docker actions and `docker://` steps are not supported. Deploy with [`kubernetes-pods.yaml`](kubernetes-pods.yaml) and see [docs/kubernetes.md](../../docs/kubernetes.md).
|
||||
- **Docker in Docker**: jobs run in a Docker daemon next to the runner. Every workflow feature works, but the daemon is privileged, so a job can break out to the node. Deploy with [`dind-docker.yaml`](dind-docker.yaml), [`statefulset-dind.yaml`](statefulset-dind.yaml) for several replicas, [`rootless-docker.yaml`](rootless-docker.yaml) for a rootless daemon, or the [Helm chart](https://gitea.com/gitea/helm-actions). The sidecars in the first two need Kubernetes 1.29 or later.
|
||||
|
||||
NOTE: A helm chart for `gitea-runner` also exists for easier deployments https://gitea.com/gitea/helm-actions
|
||||
|
||||
Each example persists **two** things, and it is worth knowing which is which:
|
||||
|
||||
- `/data` is the runner's working directory. It holds the `.runner` registration file and, optionally, the config file — so the runner re-attaches to the server instead of registering again.
|
||||
- The Docker daemon's data root holds the images pulled for jobs (`/var/lib/docker` for the dind sidecar, `/home/rootless/.local/share/docker` for `dind-rootless`). It is *not* under `/data`. If you drop this volume, the examples still work, but the image cache is discarded whenever the pod is recreated and every job re-pulls its images.
|
||||
|
||||
- Kubernetes SIGKILLs a pod 30s after SIGTERM by default, long before a job finishes and reports its result, which leaves tasks the server can only reap as zombies. The manifests raise `terminationGracePeriodSeconds` to three hours, matching the systemd example and the `runner.timeout` job ceiling; set `runner.shutdown_timeout` below that so the runner drains jobs within the window rather than being killed mid-cleanup.
|
||||
|
||||
Files in this directory:
|
||||
|
||||
- [`dind-docker.yaml`](dind-docker.yaml)
|
||||
How to create a Deployment and Persistent Volume for Kubernetes to act as a runner. The Docker credentials are re-generated each time the pod connects and does not need to be persisted.
|
||||
|
||||
- [`rootless-docker.yaml`](rootless-docker.yaml)
|
||||
How to create a rootless Deployment and Persistent Volume for Kubernetes to act as a runner. The Docker credentials are re-generated each time the pod connects and does not need to be persisted.
|
||||
|
||||
- [`statefulset-dind.yaml`](statefulset-dind.yaml)
|
||||
StatefulSet variant of the dind example. Each replica gets a stable identity and its own persistent volume via `volumeClaimTemplates`, so the runner keeps its `.runner` registration across restarts and reschedules instead of trying to register again.
|
||||
All examples read the registration token from the `runner-secret` Secret and the Gitea URL from `GITEA_INSTANCE_URL`, and keep the registration in `/data`. They give a stopping pod three hours to finish its jobs, so keep `runner.shutdown_timeout` below `terminationGracePeriodSeconds`.
|
||||
@@ -0,0 +1,83 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
token: << base64 encoded registration token >>
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: runner-secret
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: runner
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["create", "get", "list", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create", "get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/log"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "delete"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: runner
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: runner
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: runner
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: runner
|
||||
spec:
|
||||
serviceName: runner
|
||||
selector:
|
||||
matchLabels:
|
||||
app: runner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: runner
|
||||
spec:
|
||||
serviceAccountName: runner
|
||||
terminationGracePeriodSeconds: 10800 # keep above runner.shutdown_timeout, see README
|
||||
containers:
|
||||
- name: runner
|
||||
image: gitea/runner:nightly
|
||||
env:
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: http://gitea-http.gitea.svc.cluster.local:3000
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: runner-secret
|
||||
key: token
|
||||
- name: GITEA_RUNNER_LABELS
|
||||
value: ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
volumeMounts:
|
||||
- name: runner-data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: runner-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
@@ -7,16 +7,17 @@ toolchain go1.27.1
|
||||
require (
|
||||
connectrpc.com/connect v1.21.0
|
||||
dario.cat/mergo v1.0.2
|
||||
gitea.dev/actionslib v1.2.1
|
||||
gitea.dev/actionslib v1.3.0
|
||||
github.com/avast/retry-go/v5 v5.0.0
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/containerd/errdefs v1.0.0
|
||||
github.com/creack/pty v1.1.24
|
||||
github.com/distribution/reference v0.6.0
|
||||
github.com/docker/cli v29.8.1+incompatible
|
||||
github.com/docker/cli v29.8.2+incompatible
|
||||
github.com/docker/go-connections v0.8.1
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/go-git/go-billy/v5 v5.9.1
|
||||
github.com/go-git/go-billy/v5 v5.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.2
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/johannesboyne/gofakes3 v1.2.0
|
||||
|
||||
@@ -4,8 +4,8 @@ cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik=
|
||||
cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
gitea.dev/actionslib v1.2.1 h1:GL//K/0zIZV6h1OOksv1awvFVg0rg7fug2xWcG7mkG0=
|
||||
gitea.dev/actionslib v1.2.1/go.mod h1:1+gqOKGSEPn2IFHgY8S3GC5Ld+Hn8jF3OxiFHQ/fpx4=
|
||||
gitea.dev/actionslib v1.3.0 h1:xZRoTL1+sK/PaglT9uXkZJ9g0nk1WLMEH4FUXdXpKKw=
|
||||
gitea.dev/actionslib v1.3.0/go.mod h1:svCefEsavx4jmn8vJ4wTG7Q0KBcGfrFmGUjrhxuNUAQ=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -55,6 +55,8 @@ github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZV
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
||||
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
|
||||
github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
@@ -72,8 +74,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/docker/cli v29.8.1+incompatible h1:qYL1bCp6cRw2SB1xmLlIOPyV171dilw9W2Jew38vy9c=
|
||||
github.com/docker/cli v29.8.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ=
|
||||
github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q=
|
||||
github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
|
||||
@@ -90,8 +92,8 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.9.1 h1:8U73XiOTfINdItHVa6z4Gv7ToObcZ6grkqQbLryLCdA=
|
||||
github.com/go-git/go-billy/v5 v5.9.1/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0=
|
||||
github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE=
|
||||
github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY=
|
||||
|
||||
@@ -72,7 +72,7 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
defer func() { _ = releaseLock() }()
|
||||
}
|
||||
|
||||
shutdownTelemetry, err := telemetry.Setup(ctx, reg.UUID, reg.Name)
|
||||
shutdownTelemetry, err := telemetry.Setup(ctx, reg.UUID, reg.Name, metrics.Registry)
|
||||
if err != nil {
|
||||
log.WithError(err).Warn("OpenTelemetry export failed to start")
|
||||
}
|
||||
@@ -170,10 +170,11 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
|
||||
cli := client.New(
|
||||
reg.Address,
|
||||
cfg.Runner.Insecure,
|
||||
cfg.Runner.TLSConfig(),
|
||||
reg.UUID,
|
||||
reg.Token,
|
||||
config.RequestTimeout,
|
||||
cfg.Runner.ExtraHeaders,
|
||||
)
|
||||
|
||||
runner := run.NewRunner(cfg, reg, cli)
|
||||
@@ -191,6 +192,9 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
return err
|
||||
case err != nil:
|
||||
log.WithError(err).Error("fail to invoke Declare")
|
||||
if client.IsRegistrationRejected(err) {
|
||||
removeEphemeralRegistration(reg, cfg.Runner.File)
|
||||
}
|
||||
return err
|
||||
default:
|
||||
log.Infof("runner: %s, with version: %s, with labels: %v, declare successfully",
|
||||
@@ -200,12 +204,21 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
|
||||
poller := poll.New(cfg, cli, runner)
|
||||
|
||||
metrics.Init() // OTLP exports the registry even without the server
|
||||
metrics.RunnerInfo.WithLabelValues(ver.Version(), resp.Msg.Runner.Name).Set(1)
|
||||
metrics.RunnerCapacity.Set(float64(cfg.Runner.Capacity))
|
||||
metrics.RegisterUptimeFunc(time.Now())
|
||||
metrics.RegisterRunningJobsFunc(runner.RunningCount, cfg.Runner.Capacity)
|
||||
metrics.RegisterStateFunc(func() string {
|
||||
if runner.RunningCount() > 0 {
|
||||
return metrics.LabelStateBusy
|
||||
}
|
||||
if ready, _ := poller.Ready(cfg.Metrics.ReadinessGrace); !ready {
|
||||
return metrics.LabelStateUnavailable
|
||||
}
|
||||
return metrics.LabelStateIdle
|
||||
})
|
||||
if cfg.Metrics.Enabled {
|
||||
metrics.Init()
|
||||
metrics.RunnerInfo.WithLabelValues(ver.Version(), resp.Msg.Runner.Name).Set(1)
|
||||
metrics.RunnerCapacity.Set(float64(cfg.Runner.Capacity))
|
||||
metrics.RegisterUptimeFunc(time.Now())
|
||||
metrics.RegisterRunningJobsFunc(runner.RunningCount, cfg.Runner.Capacity)
|
||||
metrics.StartServer(ctx, cfg.Metrics.Addr, func() (bool, string) {
|
||||
return poller.Ready(cfg.Metrics.ReadinessGrace)
|
||||
})
|
||||
@@ -215,7 +228,7 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
poller.PollOnce()
|
||||
poller.PollOnce(func() { removeEphemeralRegistration(reg, cfg.Runner.File) })
|
||||
}()
|
||||
|
||||
// shutdown when we complete a job or cancel is requested
|
||||
@@ -244,6 +257,10 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
log.Warnf("runner: %s cancelled in progress jobs during shutdown", resp.Msg.Runner.Name)
|
||||
}
|
||||
|
||||
if poller.RegistrationRejected() {
|
||||
removeEphemeralRegistration(reg, cfg.Runner.File)
|
||||
}
|
||||
|
||||
if poller.Unregistered() {
|
||||
return errors.New("runner is no longer registered with the server; please register it again")
|
||||
}
|
||||
@@ -252,6 +269,17 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
}
|
||||
}
|
||||
|
||||
func removeEphemeralRegistration(reg *config.Registration, file string) {
|
||||
if !reg.Ephemeral {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(file); err == nil {
|
||||
log.Infof("removed the spent ephemeral registration %q", file)
|
||||
} else if !os.IsNotExist(err) {
|
||||
log.WithError(err).Warn("failed to remove the spent ephemeral registration")
|
||||
}
|
||||
}
|
||||
|
||||
type daemonArgs struct {
|
||||
Once bool
|
||||
Labels string
|
||||
|
||||
@@ -4,13 +4,24 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/config"
|
||||
"gitea.com/gitea/runner/internal/pkg/report"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
"gitea.dev/actionslib/runner/v1/runnerv1connect"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
)
|
||||
|
||||
func TestResolveLabels(t *testing.T) {
|
||||
@@ -70,3 +81,80 @@ func TestInitLoggingSetsLevelAndCaller(t *testing.T) {
|
||||
// act plans a job on this logger, so a live task's secrets have to be masked out of it
|
||||
require.IsType(t, report.MaskingFormatter(nil), log.StandardLogger().Formatter)
|
||||
}
|
||||
|
||||
var errUnregisteredRunner = errors.New("rpc error: code = Unauthenticated desc = unregistered runner")
|
||||
|
||||
type daemonTestService struct {
|
||||
runnerv1connect.UnimplementedRunnerServiceHandler
|
||||
declareErr, fetchErr error
|
||||
beforeTask bool
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (s *daemonTestService) Declare(context.Context, *connect.Request[runnerv1.DeclareRequest]) (*connect.Response[runnerv1.DeclareResponse], error) {
|
||||
if s.declareErr != nil {
|
||||
return nil, s.declareErr
|
||||
}
|
||||
return connect.NewResponse(&runnerv1.DeclareResponse{Runner: &runnerv1.Runner{Name: "test"}}), nil
|
||||
}
|
||||
|
||||
func (s *daemonTestService) FetchTask(context.Context, *connect.Request[runnerv1.FetchTaskRequest]) (*connect.Response[runnerv1.FetchTaskResponse], error) {
|
||||
if s.fetchErr != nil {
|
||||
return nil, s.fetchErr
|
||||
}
|
||||
if s.beforeTask {
|
||||
s.cancel()
|
||||
return nil, context.Canceled
|
||||
}
|
||||
return connect.NewResponse(&runnerv1.FetchTaskResponse{Task: &runnerv1.Task{Id: 1, Context: &structpb.Struct{}, WorkflowPayload: []byte("invalid: [")}}), nil
|
||||
}
|
||||
|
||||
func (*daemonTestService) UpdateTask(_ context.Context, req *connect.Request[runnerv1.UpdateTaskRequest]) (*connect.Response[runnerv1.UpdateTaskResponse], error) {
|
||||
return connect.NewResponse(&runnerv1.UpdateTaskResponse{State: req.Msg.State}), nil
|
||||
}
|
||||
|
||||
func (*daemonTestService) UpdateLog(_ context.Context, req *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error) {
|
||||
return connect.NewResponse(&runnerv1.UpdateLogResponse{AckIndex: req.Msg.Index + int64(len(req.Msg.Rows))}), nil
|
||||
}
|
||||
|
||||
func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ephemeral, beforeTask bool
|
||||
declareErr, fetchErr error
|
||||
removed bool
|
||||
}{
|
||||
{"ephemeral failed task", true, false, nil, nil, true},
|
||||
{"ephemeral interrupted before task", true, true, nil, nil, false},
|
||||
{"persistent once", false, false, nil, nil, false},
|
||||
{"ephemeral rejected on fetch", true, false, nil, errUnregisteredRunner, true},
|
||||
{"ephemeral rejected on fetch with 401", true, false, nil, connect.NewError(connect.CodeUnauthenticated, errors.New("unregistered runner")), true},
|
||||
{"ephemeral proxy 401 on fetch", true, false, nil, connect.NewError(connect.CodeUnauthenticated, errors.New("HTTP status 401 Unauthorized")), false},
|
||||
{"ephemeral rejected on declare", true, false, errUnregisteredRunner, nil, true},
|
||||
{"persistent rejected on declare", false, false, errUnregisteredRunner, nil, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, handler := runnerv1connect.NewRunnerServiceHandler(&daemonTestService{declareErr: tc.declareErr, fetchErr: tc.fetchErr, beforeTask: tc.beforeTask, cancel: cancel})
|
||||
server := httptest.NewServer(http.StripPrefix("/api/actions", handler))
|
||||
defer server.Close()
|
||||
dir := t.TempDir()
|
||||
regFile := filepath.Join(dir, "registration.json")
|
||||
configFile := filepath.Join(dir, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(configFile, []byte("runner:\n file: "+regFile+"\n idle_cleanup_interval: 0s\ncache:\n enabled: false\n"), 0o600))
|
||||
require.NoError(t, config.SaveRegistration(regFile, &config.Registration{Address: server.URL, UUID: "test", Name: "test", Token: "test", Labels: []string{"host:host"}, Ephemeral: tc.ephemeral}))
|
||||
err := runDaemon(ctx, &daemonArgs{Once: true}, &configFile)(nil, nil)
|
||||
if tc.declareErr == nil && tc.fetchErr == nil {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.Error(t, err)
|
||||
}
|
||||
if tc.removed {
|
||||
require.NoFileExists(t, regFile)
|
||||
} else {
|
||||
require.FileExists(t, regFile)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -618,7 +618,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command {
|
||||
execCmd.PersistentFlags().BoolVarP(&execArg.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout")
|
||||
execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")
|
||||
execCmd.PersistentFlags().BoolVarP(&execArg.dryrun, "dryrun", "n", false, "dryrun mode")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"-self-hosted\" to run directly on the host.")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"kubernetes://<image>\" to run it as a pod, \"-self-hosted\" to run directly on the host.")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.toolCacheMode, "tool-cache-mode", "", config.ToolCacheModeNone, "What to mount at RUNNER_TOOL_CACHE: none, or shared to reuse one tool cache across runs")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.network, "network", "", "", "Specify the network to which the container will connect")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.githubInstance, "gitea-instance", "", "", "Gitea instance to use.")
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os/signal"
|
||||
goruntime "runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/app/run"
|
||||
"gitea.com/gitea/runner/internal/pkg/client"
|
||||
@@ -364,34 +363,19 @@ func doRegister(ctx context.Context, cfg *config.Config, inputs *registerInputs)
|
||||
// initial http client
|
||||
cli := client.New(
|
||||
inputs.InstanceAddr,
|
||||
cfg.Runner.Insecure,
|
||||
cfg.Runner.TLSConfig(),
|
||||
"",
|
||||
"",
|
||||
config.RequestTimeout,
|
||||
cfg.Runner.ExtraHeaders,
|
||||
)
|
||||
|
||||
for {
|
||||
_, err := cli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{
|
||||
Data: inputs.RunnerName,
|
||||
}))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
log.WithError(err).
|
||||
Errorln("Cannot ping the Gitea instance server")
|
||||
// TODO: if ping failed, retry or exit
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
log.Debugln("Successfully pinged the Gitea instance server")
|
||||
break
|
||||
}
|
||||
if _, err := cli.Ping(ctx, connect.NewRequest(&pingv1.PingRequest{
|
||||
Data: inputs.RunnerName,
|
||||
})); err != nil {
|
||||
return fmt.Errorf("cannot ping the Gitea instance server: %w", err)
|
||||
}
|
||||
log.Debugln("Successfully pinged the Gitea instance server")
|
||||
|
||||
reg := &config.Registration{
|
||||
Name: inputs.RunnerName,
|
||||
|
||||
@@ -4,7 +4,11 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/config"
|
||||
@@ -246,3 +250,18 @@ func TestInitInputs(t *testing.T) {
|
||||
require.Nil(t, inputs.Labels)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDoRegisterReturnsPingErrorWithoutRetrying(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
requests.Add(1)
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config.Config{}
|
||||
cfg.Runner.File = filepath.Join(t.TempDir(), ".runner")
|
||||
err := doRegister(t.Context(), cfg, ®isterInputs{InstanceAddr: server.URL, RunnerName: "runner"})
|
||||
assert.ErrorContains(t, err, "cannot ping the Gitea instance server")
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
@@ -52,6 +52,8 @@ type Poller struct {
|
||||
|
||||
done chan struct{}
|
||||
|
||||
registrationRejected atomic.Bool
|
||||
|
||||
// unregistered is set when the server rejects the runner with an
|
||||
// Unauthenticated response, meaning the runner is no longer registered.
|
||||
unregistered atomic.Bool
|
||||
@@ -151,7 +153,7 @@ func (p *Poller) Poll() {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) PollOnce() {
|
||||
func (p *Poller) PollOnce(onTaskFetched func()) {
|
||||
defer close(p.done)
|
||||
s := &workerState{}
|
||||
for {
|
||||
@@ -173,6 +175,7 @@ func (p *Poller) PollOnce() {
|
||||
continue
|
||||
}
|
||||
s.resetBackoff()
|
||||
onTaskFetched()
|
||||
p.runTaskWithRecover(p.jobsCtx, task)
|
||||
return
|
||||
}
|
||||
@@ -191,6 +194,11 @@ func (p *Poller) Unregistered() bool {
|
||||
return p.unregistered.Load()
|
||||
}
|
||||
|
||||
// RegistrationRejected reports whether polling stopped on an error matching client.IsRegistrationRejected.
|
||||
func (p *Poller) RegistrationRejected() bool {
|
||||
return p.registrationRejected.Load()
|
||||
}
|
||||
|
||||
// Ready reports whether the daemon can currently communicate with Gitea and
|
||||
// accept work, reusing the availability the poll loop last observed rather than
|
||||
// re-running the check. Transient transport failures are tolerated for grace.
|
||||
@@ -198,6 +206,9 @@ func (p *Poller) Ready(grace time.Duration) (bool, string) {
|
||||
if p.unregistered.Load() {
|
||||
return false, "runner is no longer registered"
|
||||
}
|
||||
if p.pollingCtx.Err() != nil {
|
||||
return false, "runner is shutting down"
|
||||
}
|
||||
p.availabilityMu.Lock()
|
||||
ready, reason := p.availabilityReady, p.availabilityReason
|
||||
p.availabilityMu.Unlock()
|
||||
@@ -368,9 +379,11 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
|
||||
// An Unauthenticated response means the server no longer knows this
|
||||
// runner (e.g. it was deleted). Retrying forever is pointless, so stop
|
||||
// polling and let the daemon exit with an error instead of spinning.
|
||||
if isUnregistered(err) {
|
||||
rejected := client.IsRegistrationRejected(err)
|
||||
if rejected || connect.CodeOf(err) == connect.CodeUnauthenticated {
|
||||
log.WithError(err).Error("server rejected the runner as unregistered, stopping poller")
|
||||
p.unregistered.Store(true)
|
||||
p.registrationRejected.Store(rejected)
|
||||
p.shutdownPolling()
|
||||
return nil, false
|
||||
}
|
||||
@@ -384,7 +397,7 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
|
||||
p.lastPollFailed.Store(true)
|
||||
s.consecutiveErrors++
|
||||
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask, connect.CodeOf(err).String()).Inc()
|
||||
return nil, false
|
||||
}
|
||||
p.markHealthyPoll()
|
||||
@@ -415,12 +428,6 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
|
||||
return resp.Msg.Task, true
|
||||
}
|
||||
|
||||
func isUnregistered(err error) bool {
|
||||
var connectErr *connect.Error
|
||||
return errors.As(err, &connectErr) && (connectErr.Code() == connect.CodeUnauthenticated ||
|
||||
connectErr.Code() == connect.CodeUnknown && connect.IsWireError(err) && connectErr.Message() == "rpc error: code = Unauthenticated desc = unregistered runner")
|
||||
}
|
||||
|
||||
func (p *Poller) markHealthyPoll() {
|
||||
p.lastHealthyPoll.Store(time.Now().UnixNano())
|
||||
p.lastPollFailed.Store(false)
|
||||
|
||||
@@ -106,13 +106,16 @@ func TestPoller_FetchTimeoutIsNoSignal(t *testing.T) {
|
||||
// response marks the runner as unregistered and cancels the polling context so
|
||||
// the daemon can exit instead of retrying forever.
|
||||
func TestPoller_FetchUnauthenticatedStopsPolling(t *testing.T) {
|
||||
for name, fetchErr := range map[string]error{
|
||||
"connect": connect_go.NewError(connect_go.CodeUnauthenticated, errors.New("unregistered runner")),
|
||||
"gitea": connect_go.NewWireError(connect_go.CodeUnknown, errors.New("rpc error: code = Unauthenticated desc = unregistered runner")),
|
||||
for name, tc := range map[string]struct {
|
||||
fetchErr error
|
||||
rejected bool
|
||||
}{
|
||||
"proxy": {connect_go.NewError(connect_go.CodeUnauthenticated, errors.New("401 Unauthorized")), false},
|
||||
"gitea": {connect_go.NewWireError(connect_go.CodeUnknown, errors.New("rpc error: code = Unauthenticated desc = unregistered runner")), true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
client := mocks.NewClient(t)
|
||||
client.On("FetchTask", mock.Anything, mock.Anything).Return(nil, fetchErr)
|
||||
client.On("FetchTask", mock.Anything, mock.Anything).Return(nil, tc.fetchErr)
|
||||
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
@@ -122,6 +125,7 @@ func TestPoller_FetchUnauthenticatedStopsPolling(t *testing.T) {
|
||||
require.False(t, ok)
|
||||
|
||||
assert.True(t, p.Unregistered(), "runner should be marked unregistered")
|
||||
assert.Equal(t, tc.rejected, p.RegistrationRejected())
|
||||
assert.Equal(t, int64(0), s.consecutiveErrors, "unauthenticated must not drive error backoff")
|
||||
select {
|
||||
case <-p.pollingCtx.Done():
|
||||
@@ -223,6 +227,11 @@ func TestPollerReady(t *testing.T) {
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "unable to poll Gitea", reason)
|
||||
|
||||
poller.shutdownPolling()
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "runner is shutting down", reason)
|
||||
|
||||
poller.unregistered.Store(true)
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
@@ -330,7 +339,7 @@ func TestPollerPollOnceCallsOnIdle(t *testing.T) {
|
||||
poller := New(cfg, cli, runner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(poller.PollOnce)
|
||||
wg.Go(func() { poller.PollOnce(func() {}) })
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return runner.idleCalls.Load() > 0
|
||||
@@ -342,6 +351,18 @@ func TestPollerPollOnceCallsOnIdle(t *testing.T) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestPollerPollOnceCallsOnTaskFetchedBeforeRunningIt(t *testing.T) {
|
||||
cli := mocks.NewClient(t)
|
||||
cli.On("FetchTask", mock.Anything, mock.Anything).Return(connect_go.NewResponse(&runnerv1.FetchTaskResponse{Task: &runnerv1.Task{Id: 1}}), nil)
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
runner := &mockRunner{}
|
||||
runsSeenByCallback := int64(-1)
|
||||
New(cfg, cli, runner).PollOnce(func() { runsSeenByCallback = runner.running.Load() + runner.totalCompleted.Load() })
|
||||
assert.Zero(t, runsSeenByCallback)
|
||||
assert.Equal(t, int64(1), runner.totalCompleted.Load())
|
||||
}
|
||||
|
||||
// TestPoller_ConcurrencyLimitedByCapacity verifies that with capacity=3 and
|
||||
// 6 available tasks, at most 3 tasks run concurrently, and FetchTask is
|
||||
// never called concurrently (single poller).
|
||||
|
||||
+79
-10
@@ -7,6 +7,7 @@ import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -73,6 +74,7 @@ type Runner struct {
|
||||
capabilities string
|
||||
|
||||
isolatedCacheContainer func() string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
runningTasks sync.Map
|
||||
runningCount atomic.Int64
|
||||
@@ -139,6 +141,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
runHealthCheck: executeHealthCheck,
|
||||
}
|
||||
runner.isolatedCacheContainer = sync.OnceValue(runner.detectIsolatedCacheContainer)
|
||||
runner.instanceTransport = client.NewTransport(cli.Address(), cfg.Runner.TLSConfig())
|
||||
return runner
|
||||
}
|
||||
|
||||
@@ -152,6 +155,8 @@ var removeOrphanNetworks = container.RemoveOrphanNetworks
|
||||
|
||||
var removeOrphanJobVolumes = container.RemoveOrphanJobVolumes
|
||||
|
||||
var removeOrphanKubernetesResources = container.RemoveOrphanKubernetesResources
|
||||
|
||||
// OnIdle performs lightweight maintenance during polling idle windows.
|
||||
// It runs synchronously on the poller goroutine; shouldRunIdleCleanup
|
||||
// throttles invocations to runner.idle_cleanup_interval so the impact on
|
||||
@@ -174,6 +179,13 @@ func (r *Runner) OnIdle(ctx context.Context) {
|
||||
r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
|
||||
}
|
||||
r.cleanupOrphanDockerResources(ctx)
|
||||
if r.uuid != "" && r.usesKubernetes() {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
if err := removeOrphanKubernetesResources(ctx, r.kubernetesOptions(nil), r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)); err != nil {
|
||||
log.Warnf("failed to clean up job pods left behind by earlier jobs: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
|
||||
@@ -322,7 +334,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
||||
}
|
||||
|
||||
metrics.JobDuration.Observe(time.Since(start).Seconds())
|
||||
metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc()
|
||||
metrics.JobsTotal.WithLabelValues(metrics.JobStatus(ctx, reporter.Result())).Inc()
|
||||
endJob(reporter.Result())
|
||||
}()
|
||||
reporter.RunDaemon()
|
||||
@@ -401,10 +413,36 @@ func (r *Runner) requiresDocker() bool {
|
||||
return r.labels.RequireDocker() || r.cfg.Container.RequireDocker
|
||||
}
|
||||
|
||||
func (r *Runner) usesKubernetes() bool {
|
||||
return slices.ContainsFunc(r.labels, func(label *labels.Label) bool { return label.Schema == labels.SchemeKubernetes })
|
||||
}
|
||||
|
||||
// kubernetesOptions layers the pod templates of the runner labels a job runs on, in runs-on order, over the shared one.
|
||||
func (r *Runner) kubernetesOptions(runsOn []string) container.KubernetesOptions {
|
||||
options := container.KubernetesOptions{
|
||||
Kubeconfig: r.cfg.Kubernetes.Kubeconfig,
|
||||
Namespace: r.cfg.Kubernetes.Namespace,
|
||||
PodTemplates: []map[string]any{r.cfg.Kubernetes.PodTemplate},
|
||||
RunnerUUID: r.uuid,
|
||||
}
|
||||
names := r.labels.Names()
|
||||
for index, name := range runsOn {
|
||||
if template, ok := r.cfg.Kubernetes.PodTemplates[name]; ok && slices.Contains(names, name) && !slices.Contains(runsOn[:index], name) {
|
||||
options.PodTemplates = append(options.PodTemplates, template)
|
||||
}
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
// fallbackPlatform is where a job runs whose runs-on matches no label, as any job without a
|
||||
// runs-on does, since Gitea sends those to every runner.
|
||||
func (r *Runner) fallbackPlatform(ctx context.Context) string {
|
||||
if r.requiresDocker() || dockerReachable(ctx) {
|
||||
switch {
|
||||
case r.requiresDocker():
|
||||
return r.cfg.Runner.DefaultImage
|
||||
case r.usesKubernetes():
|
||||
return labels.SchemeKubernetes + "://" + r.cfg.Runner.DefaultImage
|
||||
case dockerReachable(ctx):
|
||||
return r.cfg.Runner.DefaultImage
|
||||
}
|
||||
return labels.SelfHostedPlatform
|
||||
@@ -424,6 +462,10 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.WithFields(taskLogFields(task, jobID)).
|
||||
WithField("actions_url", r.getDefaultActionsURL(task)).
|
||||
WithField("address", r.client.Address()).
|
||||
Info("task started")
|
||||
|
||||
plan, err := model.CombineWorkflowPlanner(workflow).PlanJob(jobID)
|
||||
if err != nil {
|
||||
@@ -444,7 +486,8 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
}
|
||||
return fallbackPlatform()
|
||||
}
|
||||
cacheURL, cacheContainer := r.cacheForJob(job, platformPicker)
|
||||
platform := platformPicker(job.RunsOn())
|
||||
cacheURL, cacheContainer := r.cacheForJob(job, platform)
|
||||
if cacheURL != "" {
|
||||
envs["ACTIONS_CACHE_URL"] = cacheURL + "/"
|
||||
}
|
||||
@@ -453,7 +496,13 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
// act reaches them by their workflow key.
|
||||
var services map[string]yaml.Node
|
||||
_ = job.RawServices.Decode(&services) // a whole-value expression names no service before evaluation
|
||||
proxyEnv := JobProxyEnv(envs, cacheURL, slices.Sorted(maps.Keys(services)))
|
||||
serviceNames := slices.Sorted(maps.Keys(services))
|
||||
if strings.HasPrefix(platform, labels.SchemeKubernetes+"://") {
|
||||
for index, name := range serviceNames {
|
||||
serviceNames[index] = container.KubernetesServiceName(name)
|
||||
}
|
||||
}
|
||||
proxyEnv := JobProxyEnv(envs, cacheURL, serviceNames)
|
||||
maps.Copy(envs, proxyEnv)
|
||||
|
||||
if r.capabilities != "" {
|
||||
@@ -464,10 +513,6 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
}
|
||||
telemetry.AddJobEnv(ctx, envs)
|
||||
|
||||
log.Infof("task %v repo is %v %v %v", task.Id, taskContext["repository"].GetStringValue(),
|
||||
r.getDefaultActionsURL(task),
|
||||
r.client.Address())
|
||||
|
||||
preset := model.GithubContextFromMap(task.Context.AsMap())
|
||||
if t := task.Secrets["GITEA_TOKEN"]; t != "" {
|
||||
preset.Token = t
|
||||
@@ -572,6 +617,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
},
|
||||
ContainerOptions: r.cfg.Container.Options,
|
||||
ServiceReadyTimeout: r.cfg.Container.ServiceReadyTimeout,
|
||||
NoSweep: r.cfg.Container.Sweep != nil && !*r.cfg.Container.Sweep,
|
||||
ContainerDaemonSocket: r.cfg.Container.DockerHost,
|
||||
Privileged: r.cfg.Container.Privileged,
|
||||
DefaultActionInstance: r.getDefaultActionsURL(task),
|
||||
@@ -583,7 +629,11 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
ValidVolumes: r.cfg.Container.ValidVolumes,
|
||||
SharedToolCache: r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
|
||||
InsecureSkipTLS: r.cfg.Runner.Insecure,
|
||||
ClientCertFile: r.cfg.Runner.ClientCertFile,
|
||||
ClientKeyFile: r.cfg.Runner.ClientKeyFile,
|
||||
InstanceTransport: r.instanceTransport,
|
||||
RunnerName: r.name,
|
||||
KubernetesPicker: r.kubernetesOptions,
|
||||
}
|
||||
|
||||
rr, err := runner.New(runnerConfig)
|
||||
@@ -866,10 +916,10 @@ func warnIgnoredCacheSecret(cfg *config.Config) {
|
||||
}
|
||||
|
||||
// cacheForJob returns the cache address for the job, and the container to attach to its network to reach it there.
|
||||
func (r *Runner) cacheForJob(job *model.Job, pickPlatform func([]string) string) (string, string) {
|
||||
func (r *Runner) cacheForJob(job *model.Job, platform string) (string, string) {
|
||||
jobContainer := job.Container()
|
||||
cacheContainer := ""
|
||||
if jobContainer != nil && jobContainer.Image != "" || pickPlatform(job.RunsOn()) != labels.SelfHostedPlatform {
|
||||
if !strings.HasPrefix(platform, labels.SchemeKubernetes+"://") && (jobContainer != nil && jobContainer.Image != "" || platform != labels.SelfHostedPlatform) {
|
||||
cacheContainer = r.isolatedCacheContainer()
|
||||
}
|
||||
if cacheContainer == "" {
|
||||
@@ -896,3 +946,22 @@ func (r *Runner) detectIsolatedCacheContainer() string {
|
||||
}
|
||||
return cacheContainer
|
||||
}
|
||||
|
||||
func taskLogFields(task *runnerv1.Task, job string) log.Fields {
|
||||
taskContext := task.GetContext().GetFields()
|
||||
fields := log.Fields{
|
||||
"task_id": task.Id,
|
||||
"job": job,
|
||||
"repository": taskContext["repository"].GetStringValue(),
|
||||
"run_id": taskContext["run_id"].GetStringValue(),
|
||||
"run_number": taskContext["run_number"].GetStringValue(),
|
||||
}
|
||||
if parts := strings.Split(taskContext["gitea_runtime_token"].GetStringValue(), "."); len(parts) == 3 {
|
||||
var claims struct{ JobID int64 } // Gitea's actionsClaims fields are untagged, so the claim key is "JobID"
|
||||
payload, err := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
if err == nil && json.Unmarshal(payload, &claims) == nil && claims.JobID > 0 {
|
||||
fields["job_id"] = claims.JobID
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
@@ -11,7 +11,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/config"
|
||||
"gitea.com/gitea/runner/internal/pkg/labels"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -329,8 +331,20 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
|
||||
var sweptPods []string
|
||||
origRemoveOrphanKubernetesResources := removeOrphanKubernetesResources
|
||||
removeOrphanKubernetesResources = func(ctx context.Context, options container.KubernetesOptions, createdBefore time.Time) error {
|
||||
_, hasDeadline := ctx.Deadline()
|
||||
assert.True(t, hasDeadline)
|
||||
sweptPods = append(sweptPods, options.RunnerUUID)
|
||||
assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeOrphanKubernetesResources = origRemoveOrphanKubernetesResources })
|
||||
kubernetesLabel, err := labels.Parse("k8s:kubernetes://node")
|
||||
require.NoError(t, err)
|
||||
|
||||
r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
|
||||
r := &Runner{uuid: "runner-1", cfg: cfg, labels: labels.Labels{kubernetesLabel}, now: func() time.Time { return now }}
|
||||
r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
|
||||
r.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
@@ -353,4 +367,5 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
hostOnly.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1", "runner-2"}, sweptVolumes)
|
||||
assert.Equal(t, []string{"a1b2c3d4e5f6", ""}, detached)
|
||||
assert.Equal(t, []string{"runner-1"}, sweptPods)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
"connectrpc.com/connect"
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -243,6 +245,16 @@ func TestRunnerFallbackPlatform(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerKubernetesOptionsLayerTheLabelPodTemplatesInRunsOnOrder(t *testing.T) {
|
||||
cfg := &config.Config{Kubernetes: config.Kubernetes{
|
||||
PodTemplate: map[string]any{"base": true},
|
||||
PodTemplates: map[string]map[string]any{"gpu": {"gpu": true}, "large": {"large": true}, "other": {"other": true}},
|
||||
}}
|
||||
r := &Runner{cfg: cfg, labels: labels.Labels{{Name: "linux"}, {Name: "gpu"}, {Name: "large"}}}
|
||||
|
||||
require.Equal(t, []map[string]any{{"base": true}, {"large": true}, {"gpu": true}}, r.kubernetesOptions([]string{"linux", "large", "other", "gpu", "large"}).PodTemplates)
|
||||
}
|
||||
|
||||
// Proxy variables are assembled per task, because a job's service containers have to be
|
||||
// reached directly and they are only known once the workflow is parsed.
|
||||
func TestNewRunnerLeavesProxyToTheTask(t *testing.T) {
|
||||
@@ -348,7 +360,7 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
|
||||
r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
|
||||
pickPlatform := func(runsOn []string) string { return map[string]string{"native": labels.SelfHostedPlatform}[runsOn[0]] }
|
||||
for job, isolated := range map[string]bool{"native": false, "linux": true, "containerized": true, "empty": false} {
|
||||
cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform)
|
||||
cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform(workflow.GetJob(job).RunsOn()))
|
||||
if isolated {
|
||||
assert.Equal(t, "a1b2c3d4e5f6", cacheContainer, job)
|
||||
assert.Equal(t, strings.Replace(r.cacheHandler.ExternalURL(), "127.0.0.1", cacheContainer, 1), cacheURL, job)
|
||||
@@ -403,3 +415,15 @@ func TestRestoreCalledWorkflowContext(t *testing.T) {
|
||||
assert.Equal(t, "workflow_call", task.Context.AsMap()["event_name"])
|
||||
})
|
||||
}
|
||||
|
||||
func TestTaskLogFields(t *testing.T) {
|
||||
fields := func(token string) log.Fields {
|
||||
taskContext, err := structpb.NewStruct(map[string]any{"repository": "owner/repo", "run_id": "159", "run_number": "14", "gitea_runtime_token": token})
|
||||
require.NoError(t, err)
|
||||
return taskLogFields(&runnerv1.Task{Id: 72, Context: taskContext}, "build")
|
||||
}
|
||||
want := log.Fields{"task_id": int64(72), "job": "build", "repository": "owner/repo", "run_id": "159", "run_number": "14"}
|
||||
assert.Equal(t, want, fields("malformed"))
|
||||
want["job_id"] = int64(221)
|
||||
assert.Equal(t, want, fields("header."+base64.RawURLEncoding.EncodeToString([]byte(`{"TaskID":72,"JobID":221}`))+".signature"))
|
||||
}
|
||||
@@ -20,4 +20,7 @@ type Context struct {
|
||||
NoSkipCheckout bool
|
||||
UseGitIgnore bool
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile string
|
||||
ClientKeyFile string
|
||||
Env map[string]string
|
||||
}
|
||||
@@ -6,21 +6,37 @@ package checkout
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/kballard/go-shellquote"
|
||||
)
|
||||
|
||||
var inputNames = []string{"repository", "ref", "token", "path", "fetch-depth"}
|
||||
var inputNames = []string{
|
||||
"repository", "ref", "token", "ssh-key", "ssh-known-hosts", "ssh-strict", "ssh-user", "persist-credentials", "path", "clean",
|
||||
"filter", "sparse-checkout", "sparse-checkout-cone-mode", "fetch-depth", "fetch-tags", "show-progress", "lfs", "submodules",
|
||||
"set-safe-directory",
|
||||
}
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) error {
|
||||
const credentialsInclude = "includeIf.gitdir:**.path" // the form actions/checkout removes from a repository it checks out again
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) (err error) {
|
||||
for name := range c.Inputs {
|
||||
if !slices.Contains(inputNames, name) {
|
||||
return fmt.Errorf("checkout: unsupported input %q", name)
|
||||
@@ -37,38 +53,402 @@ func Main(ctx context.Context, c *action.Context) error {
|
||||
|
||||
ghc := c.Github
|
||||
repository := cmp.Or(c.Inputs["repository"], ghc.Repository)
|
||||
ref := shortBranch(c.Inputs["ref"])
|
||||
isWorkflowRepo := repository == ghc.Repository
|
||||
local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, ref) && !c.NoSkipCheckout
|
||||
if local && !c.BindWorkdir {
|
||||
return c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx)
|
||||
}
|
||||
if local && dst == c.Workspace {
|
||||
ref := c.Inputs["ref"]
|
||||
sshKey := strings.TrimSpace(c.Inputs["ssh-key"])
|
||||
isWorkflowRepo := strings.EqualFold(repository, ghc.Repository)
|
||||
local := isWorkflowRepo && slices.Contains([]string{"", shortBranch(ghc.Ref), ghc.Sha}, shortBranch(ref)) && !c.NoSkipCheckout && sshKey == ""
|
||||
if local && (!c.BindWorkdir || dst == c.Workspace) {
|
||||
if !c.BindWorkdir {
|
||||
if err := c.Container.CopyDir(dst, c.HostWorkdir+string(filepath.Separator)+".", c.UseGitIgnore, false)(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if output := c.Env["GITHUB_OUTPUT"]; output != "" {
|
||||
return c.Container.Copy(path.Dir(output), &container.FileEntry{Name: path.Base(output), Mode: 0o666, Body: fmt.Sprintf("ref=%s\ncommit=%s\n", ghc.Ref, ghc.Sha)})(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if ref == "" {
|
||||
ref = "HEAD"
|
||||
if isWorkflowRepo {
|
||||
ref = ghc.Sha
|
||||
co := &checkout{
|
||||
url: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository,
|
||||
dst: dst,
|
||||
ref: ref,
|
||||
depth: depth,
|
||||
fetchTags: boolInput(c.Inputs, "fetch-tags", false),
|
||||
clean: boolInput(c.Inputs, "clean", true),
|
||||
lfs: boolInput(c.Inputs, "lfs", false),
|
||||
progress: boolInput(c.Inputs, "show-progress", true),
|
||||
cone: boolInput(c.Inputs, "sparse-checkout-cone-mode", true),
|
||||
filter: strings.TrimSpace(c.Inputs["filter"]),
|
||||
output: c.Env["GITHUB_OUTPUT"],
|
||||
}
|
||||
for pattern := range strings.Lines(c.Inputs["sparse-checkout"]) {
|
||||
if pattern = strings.TrimSpace(pattern); pattern != "" {
|
||||
co.sparse = append(co.sparse, pattern)
|
||||
}
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "checkout-")
|
||||
if err != nil {
|
||||
if len(co.sparse) > 0 {
|
||||
co.filter = cmp.Or(co.filter, "blob:none")
|
||||
}
|
||||
if submodules := strings.ToLower(strings.TrimSpace(c.Inputs["submodules"])); submodules == "true" || submodules == "recursive" {
|
||||
co.submodules = submodules
|
||||
}
|
||||
if sshKey != "" {
|
||||
if co.url, err = sshURL(ghc, repository, strings.TrimSpace(c.Inputs["ssh-user"])); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
co.objectFormat = git.ObjectFormat(ref)
|
||||
switch {
|
||||
case co.objectFormat != "":
|
||||
co.sha, co.ref = ref, ""
|
||||
case ref == "" && isWorkflowRepo && strings.HasPrefix(ghc.Ref, "refs/"):
|
||||
co.sha, co.ref = ghc.Sha, ghc.Ref
|
||||
case ref == "" && isWorkflowRepo: // a merged pull request's ref is its merge commit
|
||||
co.sha = ghc.Sha
|
||||
case ref == "HEAD":
|
||||
co.ref = ""
|
||||
}
|
||||
if isWorkflowRepo {
|
||||
co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(ghc.Sha))
|
||||
}
|
||||
|
||||
// named like actions/checkout's, which create-pull-request knows to hide, and gone with the job
|
||||
runnerTemp, _ := c.Container.GetRunnerContext(ctx)["temp"].(string)
|
||||
digest := sha256.Sum256([]byte(dst))
|
||||
credentials := fmt.Sprintf("git-credentials-%x.config", digest[:8])
|
||||
co.credentials = path.Join(runnerTemp, credentials)
|
||||
persist := boolInput(c.Inputs, "persist-credentials", true)
|
||||
config, remoteConfig := serverConfig(ghc.ServerURL, cmp.Or(c.Inputs["token"], ghc.Token), c.InsecureSkipTLS, co.submodules != "" && sshKey == "")
|
||||
if boolInput(c.Inputs, "set-safe-directory", true) {
|
||||
remoteConfig = append(remoteConfig, "safe.directory=*") // a bound workdir belongs to the host user
|
||||
}
|
||||
if c.ClientCertFile != "" { // only for this step, later steps never get the runner's key
|
||||
var certPEM, keyPEM []byte
|
||||
if certPEM, err = os.ReadFile(c.ClientCertFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if keyPEM, err = os.ReadFile(c.ClientKeyFile); err != nil {
|
||||
return err
|
||||
}
|
||||
cert, key := fmt.Sprintf("git-client-cert-%x.pem", digest[:8]), fmt.Sprintf("git-client-key-%x.pem", digest[:8])
|
||||
defer func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx))
|
||||
}()
|
||||
if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: cert, Mode: 0o600, Body: string(certPEM)}, &container.FileEntry{Name: key, Mode: 0o600, Body: string(keyPEM)})(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
scope := "http." + strings.TrimSuffix(ghc.ServerURL, "/") + "/."
|
||||
remoteConfig = append(remoteConfig, scope+"sslCert="+path.Join(runnerTemp, cert), scope+"sslKey="+path.Join(runnerTemp, key))
|
||||
}
|
||||
env := gitEnv(c, remoteConfig)
|
||||
env["GIT_LFS_SKIP_SMUDGE"] = strconv.FormatBool(!co.lfs || len(co.sparse) == 0) // lfs pull fetches the objects of a full checkout in one go
|
||||
if sshKey != "" {
|
||||
key := fmt.Sprintf("git-ssh-key-%x", digest[:8])
|
||||
files := []*container.FileEntry{{Name: key, Mode: 0o600, Body: sshKey + "\n"}}
|
||||
ssh := []string{"ssh", "-i", path.Join(runnerTemp, key)}
|
||||
if boolInput(c.Inputs, "ssh-strict", true) {
|
||||
ssh = append(ssh, "-o", "StrictHostKeyChecking=yes", "-o", "CheckHostIP=no")
|
||||
}
|
||||
if knownHosts := strings.TrimSpace(c.Inputs["ssh-known-hosts"]); knownHosts != "" {
|
||||
name := fmt.Sprintf("git-ssh-known-hosts-%x", digest[:8])
|
||||
files = append(files, &container.FileEntry{Name: name, Mode: 0o600, Body: knownHosts + "\n"})
|
||||
ssh = append(ssh, "-o", "UserKnownHostsFile=~/.ssh/known_hosts "+strconv.Quote(path.Join(runnerTemp, name)))
|
||||
}
|
||||
if !persist {
|
||||
defer func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx))
|
||||
}()
|
||||
}
|
||||
if err := c.Container.Copy(runnerTemp, files...)(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
env["GIT_SSH_COMMAND"] = shellquote.Join(ssh...)
|
||||
config += "[core]\n\tsshCommand = " + strconv.Quote(env["GIT_SSH_COMMAND"]) + "\n"
|
||||
}
|
||||
if !persist {
|
||||
config = ""
|
||||
}
|
||||
|
||||
if co.objectFormat == "" || co.sha == "" && !strings.HasPrefix(co.ref, "refs/") {
|
||||
advertised, err := lsRemote(ctx, c, env, co.url, co.ref)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checkout: %s@%s: %w", repository, ref, err)
|
||||
}
|
||||
if co.resolve(advertised); co.ref == "" && co.sha == "" {
|
||||
return fmt.Errorf("checkout: %s has no ref %q", repository, ref)
|
||||
}
|
||||
}
|
||||
co.outputRef = co.ref
|
||||
if ref != co.sha {
|
||||
co.outputRef = cmp.Or(ref, co.ref) // as given, like actions/checkout
|
||||
}
|
||||
if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: credentials, Mode: 0o600, Body: config})(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
if err := git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{
|
||||
URL: strings.TrimSuffix(ghc.ServerURL, "/") + "/" + repository,
|
||||
Ref: ref,
|
||||
Dir: dir,
|
||||
Token: cmp.Or(c.Inputs["token"], ghc.Token),
|
||||
Depth: depth,
|
||||
InsecureSkipTLS: c.InsecureSkipTLS,
|
||||
})(ctx); err != nil {
|
||||
return fmt.Errorf("checkout: clone %s@%s: %w", repository, ref, err)
|
||||
|
||||
clone := func(commands [][]string) error {
|
||||
if err := run(ctx, c, env, commands); err != nil {
|
||||
return fmt.Errorf("checkout: clone %s@%s: %w", repository, cmp.Or(co.ref, co.sha), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.Container.CopyDir(dst, dir+string(filepath.Separator)+".", false, false)(ctx)
|
||||
commands, fetched := co.commands()
|
||||
if co.sha != "" && strings.HasPrefix(co.ref, "refs/tags/") {
|
||||
if err := clone(commands[:fetched]); err != nil {
|
||||
return err
|
||||
}
|
||||
tagged, err := output(ctx, c, env, "git", "-C", dst, "rev-parse", "--verify", "--quiet", co.ref+"^{commit}")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checkout: %s: %w", co.ref, err)
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(tagged), co.sha) {
|
||||
return fmt.Errorf("checkout: %s does not point to the expected commit %s, it may have been moved after the workflow was triggered", co.ref, co.sha)
|
||||
}
|
||||
commands = commands[fetched:]
|
||||
}
|
||||
return clone(commands)
|
||||
}
|
||||
|
||||
func gitEnv(c *action.Context, config []string) map[string]string {
|
||||
env := map[string]string{"GIT_TERMINAL_PROMPT": "0"}
|
||||
maps.Copy(env, c.Env)
|
||||
maps.DeleteFunc(env, func(name, _ string) bool { return slices.Contains(git.LocalEnvVars, name) })
|
||||
maps.Copy(env, git.ConfigEnv(append([]string{"protocol.version=2", "init.defaultBranch=main", "advice.detachedHead=false", "maintenance.auto=false"}, config...)...))
|
||||
return env
|
||||
}
|
||||
|
||||
func run(ctx context.Context, c *action.Context, env map[string]string, commands [][]string) error {
|
||||
if _, host := c.Container.(*container.HostEnvironment); !host || runtime.GOOS != "windows" {
|
||||
script := []string{"set -e", `command -v git >/dev/null || { echo "builtin:checkout needs git on PATH" >&2; exit 127; }`}
|
||||
for _, command := range commands {
|
||||
script = append(script, shellquote.Join(command...))
|
||||
}
|
||||
return c.Container.Exec([]string{"sh", "-c", strings.Join(script, "\n")}, env, "", "")(ctx)
|
||||
}
|
||||
for _, command := range commands {
|
||||
if err := c.Container.Exec(command, env, "", "")(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// output runs where the job runs, so the job's network and git config apply as for the fetch.
|
||||
func output(ctx context.Context, c *action.Context, env map[string]string, command ...string) (string, error) {
|
||||
var out strings.Builder
|
||||
stdout, stderr := c.Container.ReplaceLogWriter(&out, &out)
|
||||
defer c.Container.ReplaceLogWriter(stdout, stderr)
|
||||
if err := c.Container.Exec(command, env, "", "")(ctx); err != nil {
|
||||
return "", fmt.Errorf("%w: %s", err, strings.TrimSpace(out.String()))
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func lsRemote(ctx context.Context, c *action.Context, env map[string]string, url, ref string) (string, error) {
|
||||
patterns := []string{ref}
|
||||
switch {
|
||||
case ref == "":
|
||||
patterns = []string{"HEAD"}
|
||||
case !strings.HasPrefix(ref, "refs/"):
|
||||
patterns = []string{"refs/heads/" + ref, "refs/tags/" + ref}
|
||||
}
|
||||
return output(ctx, c, env, append([]string{"git", "ls-remote", "--symref", url}, patterns...)...)
|
||||
}
|
||||
|
||||
// serverConfig scopes the token to the server, so no other host like a repository's LFS endpoint gets it.
|
||||
func serverConfig(serverURL, token string, insecureSkipTLS, rewriteSSH bool) (file string, env []string) {
|
||||
scope := strings.TrimSuffix(serverURL, "/") + "/"
|
||||
file = fmt.Sprintf("[http %q]\n", scope)
|
||||
for _, entry := range git.RemoteConfig(token, insecureSkipTLS) {
|
||||
setting := strings.TrimPrefix(entry, "http.")
|
||||
key, value, _ := strings.Cut(setting, "=")
|
||||
file += "\t" + key + " = " + value + "\n"
|
||||
env = append(env, "http."+scope+"."+setting)
|
||||
}
|
||||
if server, err := url.Parse(serverURL); rewriteSSH && err == nil && server.Hostname() != "" { // submodules on the server fetch over HTTP with the token
|
||||
file += fmt.Sprintf("[url %q]\n\tinsteadOf = git@%s:\n", scope, server.Hostname())
|
||||
env = append(env, "url."+scope+".insteadOf=git@"+server.Hostname()+":")
|
||||
}
|
||||
return file, env
|
||||
}
|
||||
|
||||
func sshURL(ghc *model.GithubContext, repository, user string) (string, error) {
|
||||
endpoint := &url.URL{Scheme: "ssh"}
|
||||
if advertised, _ := model.NestedMapLookup(ghc.Event, "repository", "ssh_url").(string); advertised != "" {
|
||||
if colon := strings.LastIndex(advertised, ":"); !strings.HasPrefix(advertised, "ssh://") && colon >= 0 {
|
||||
advertised = "ssh://" + advertised[:colon] + "/" + advertised[colon+1:] // scp-like git@host:owner/repo.git
|
||||
}
|
||||
var err error
|
||||
if endpoint, err = url.Parse(advertised); err != nil {
|
||||
return "", fmt.Errorf("checkout: parse the event's ssh_url: %w", err)
|
||||
}
|
||||
} else if server, err := url.Parse(ghc.ServerURL); err == nil {
|
||||
endpoint.Host = server.Hostname()
|
||||
}
|
||||
endpoint.User = url.User(cmp.Or(user, endpoint.User.Username(), "git"))
|
||||
endpoint.Path = "/" + repository + ".git"
|
||||
return endpoint.String(), nil
|
||||
}
|
||||
|
||||
type checkout struct {
|
||||
url, dst, objectFormat, credentials, output, outputRef string
|
||||
ref, sha string // ref is fully qualified or empty, sha pins it when known
|
||||
filter, submodules string
|
||||
sparse []string
|
||||
depth int
|
||||
fetchTags, clean, lfs, progress, cone bool
|
||||
}
|
||||
|
||||
const fullDepth = "2147483647" // git's INFINITE_DEPTH, also deepens a shallow dst
|
||||
|
||||
// commands checks out co.dst, the first `fetched` of them fetch so a fetched tag can be verified before the rest.
|
||||
func (co *checkout) commands() (commands [][]string, fetched int) {
|
||||
repo := func(args ...string) []string { return append([]string{"git", "-C", co.dst}, args...) }
|
||||
commands = [][]string{
|
||||
{"git", "init", "--quiet", "--object-format=" + co.objectFormat, co.dst},
|
||||
repo("config", "remote.origin.url", co.url), // `remote add` fails on a repeated checkout
|
||||
repo("config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"),
|
||||
repo("config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials),
|
||||
}
|
||||
if co.lfs {
|
||||
commands = append(commands, repo("lfs", "install", "--local"))
|
||||
}
|
||||
|
||||
depth := strconv.Itoa(co.depth)
|
||||
var refSpecs []string
|
||||
if co.depth == 0 {
|
||||
depth = fullDepth
|
||||
refSpecs = append(refSpecs, "+refs/heads/*:refs/remotes/origin/*")
|
||||
}
|
||||
fetch := []string{"fetch", "--force", "--no-tags", "--no-recurse-submodules", "--depth", depth}
|
||||
if co.progress {
|
||||
fetch = append(fetch, "--progress")
|
||||
} else {
|
||||
fetch = append(fetch, "--no-progress")
|
||||
}
|
||||
if co.filter != "" {
|
||||
fetch = append(fetch, "--filter="+co.filter)
|
||||
}
|
||||
if co.fetchTags || co.depth == 0 {
|
||||
refSpecs = append(refSpecs, "+refs/tags/*:refs/tags/*")
|
||||
}
|
||||
tag := strings.HasPrefix(co.ref, "refs/tags/") // fetched itself, which keeps an annotated tag's object
|
||||
pinned := co.sha != "" && co.ref != "" && !tag
|
||||
if co.ref != "" && !pinned {
|
||||
refSpecs = append(refSpecs, "+"+co.ref+":"+trackingRef(co.ref))
|
||||
}
|
||||
if co.sha != "" && !tag {
|
||||
refSpecs = append(refSpecs, co.sha)
|
||||
}
|
||||
commands = append(commands, repo(append(append(fetch, "origin"), refSpecs...)...))
|
||||
if pinned {
|
||||
commands = append(commands, repo("update-ref", trackingRef(co.ref), co.sha))
|
||||
}
|
||||
fetched = len(commands)
|
||||
|
||||
if len(co.sparse) > 0 {
|
||||
mode := "--cone"
|
||||
if !co.cone {
|
||||
mode = "--no-cone"
|
||||
}
|
||||
commands = append(commands, repo("sparse-checkout", "init", mode), repo(append([]string{"sparse-checkout", "set", "--"}, co.sparse...)...))
|
||||
}
|
||||
if branch, ok := strings.CutPrefix(co.ref, "refs/heads/"); ok {
|
||||
commands = append(commands, repo("checkout", "--force", "--ignore-other-worktrees", "-B", branch, "--track", trackingRef(co.ref)))
|
||||
} else {
|
||||
commands = append(commands, repo("checkout", "--force", cmp.Or(co.sha, trackingRef(co.ref))))
|
||||
}
|
||||
if len(co.sparse) == 0 { // after the checkout, so a sparse dst of another repository needs nothing of it
|
||||
commands = append(commands, repo("sparse-checkout", "disable"), repo("config", "--local", "--unset-all", "extensions.worktreeConfig"))
|
||||
}
|
||||
if co.lfs && len(co.sparse) == 0 {
|
||||
commands = append(commands, repo("lfs", "pull", "origin"))
|
||||
}
|
||||
if co.clean {
|
||||
commands = append(commands, repo("clean", "-ffdx"))
|
||||
}
|
||||
if co.submodules != "" {
|
||||
commands = append(commands, co.submoduleCommands(repo)...)
|
||||
}
|
||||
if co.output != "" {
|
||||
commands = append(commands, repo("log", "-1", "--format=commit=%H%nref="+strings.ReplaceAll(co.outputRef, "%", "%%"), "--output="+co.output))
|
||||
}
|
||||
return commands, fetched
|
||||
}
|
||||
|
||||
func (co *checkout) submoduleCommands(repo func(args ...string) []string) [][]string {
|
||||
var recursive []string
|
||||
if co.submodules == "recursive" {
|
||||
recursive = []string{"--recursive"}
|
||||
}
|
||||
foreach := func(commands ...[]string) []string {
|
||||
joined := make([]string, len(commands))
|
||||
for i, command := range commands {
|
||||
joined[i] = shellquote.Join(command...)
|
||||
}
|
||||
return repo(append(append([]string{"submodule", "foreach"}, recursive...), strings.Join(joined, " && "))...)
|
||||
}
|
||||
commands := [][]string{repo(append([]string{"submodule", "sync"}, recursive...)...)}
|
||||
if co.clean {
|
||||
commands = append(commands, foreach([]string{"git", "clean", "-ffdx"}))
|
||||
}
|
||||
update := append([]string{"submodule", "update", "--init", "--force"}, recursive...)
|
||||
if co.depth > 0 {
|
||||
update = append(update, "--depth="+strconv.Itoa(co.depth))
|
||||
}
|
||||
include := []string{"git", "config", "--replace-all", "--fixed-value", credentialsInclude, co.credentials, co.credentials}
|
||||
if co.lfs {
|
||||
return append(commands, repo(update...), foreach([]string{"git", "lfs", "install", "--local"}, []string{"git", "lfs", "pull", "origin"}, include))
|
||||
}
|
||||
return append(commands, repo(update...), foreach(include))
|
||||
}
|
||||
|
||||
func (co *checkout) resolve(advertised string) {
|
||||
ref := co.ref
|
||||
if !strings.HasPrefix(ref, "refs/") {
|
||||
co.ref = ""
|
||||
}
|
||||
for line := range strings.Lines(advertised) {
|
||||
value, name, _ := strings.Cut(strings.TrimSpace(line), "\t")
|
||||
if target, ok := strings.CutPrefix(value, "ref: "); ok {
|
||||
if name == "HEAD" {
|
||||
co.ref = target
|
||||
}
|
||||
continue
|
||||
}
|
||||
co.objectFormat = cmp.Or(co.objectFormat, git.ObjectFormat(value))
|
||||
switch name {
|
||||
case "HEAD":
|
||||
if co.ref == "" {
|
||||
co.sha = value
|
||||
}
|
||||
case "refs/heads/" + ref:
|
||||
co.ref = name
|
||||
case "refs/tags/" + ref:
|
||||
co.ref = cmp.Or(co.ref, name)
|
||||
}
|
||||
}
|
||||
co.objectFormat = cmp.Or(co.objectFormat, "sha1")
|
||||
}
|
||||
|
||||
func trackingRef(ref string) string {
|
||||
if branch, ok := strings.CutPrefix(ref, "refs/heads/"); ok {
|
||||
return "refs/remotes/origin/" + branch
|
||||
}
|
||||
if pull, ok := strings.CutPrefix(ref, "refs/pull/"); ok {
|
||||
return "refs/remotes/pull/" + pull
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
func boolInput(inputs map[string]string, name string, fallback bool) bool {
|
||||
value := strings.TrimSpace(inputs[name])
|
||||
return strings.EqualFold(value, "true") || value == "" && fallback
|
||||
}
|
||||
|
||||
func workspaceDestination(workspace, checkoutPath string) (string, error) {
|
||||
@@ -91,7 +471,6 @@ func fetchDepth(input string) (int, error) {
|
||||
return depth, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutor resolves branches by short name, a full tag ref stays exact.
|
||||
func shortBranch(ref string) string {
|
||||
func shortBranch(ref string) string { // full tag refs stay exact
|
||||
return strings.TrimPrefix(ref, "refs/heads/")
|
||||
}
|
||||
@@ -4,14 +4,21 @@
|
||||
package checkout
|
||||
|
||||
import (
|
||||
"context"
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
@@ -20,22 +27,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakeContainer struct {
|
||||
container.ExecutionsEnvironment
|
||||
dst, src string
|
||||
log []string
|
||||
}
|
||||
|
||||
func (f *fakeContainer) CopyDir(dst, src string, _, _ bool) common.Executor {
|
||||
return func(context.Context) error {
|
||||
f.dst, f.src = dst, src
|
||||
if out, err := exec.Command("git", "--git-dir", filepath.Join(src, ".git"), "log", "--format=%s").Output(); err == nil {
|
||||
f.log = strings.Fields(string(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
@@ -49,67 +40,317 @@ func runGit(t *testing.T, args ...string) string {
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func gitLog(dir string) []string {
|
||||
out, err := exec.Command("git", "--git-dir", filepath.Join(dir, ".git"), "log", "--format=%s").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return strings.Fields(string(out))
|
||||
}
|
||||
|
||||
func TestCheckout(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1")
|
||||
sha := runGit(t, "-C", work, "rev-parse", "HEAD")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2")
|
||||
runGit(t, "-C", work, "checkout", "-b", "feature")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1")
|
||||
runGit(t, "-C", work, "checkout", "--orphan", "trunk")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2")
|
||||
runGit(t, "-C", work, "tag", "trunk", "main")
|
||||
server := t.TempDir()
|
||||
runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo"))
|
||||
runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo"))
|
||||
var sha string
|
||||
_, lfsErr := exec.LookPath("git-lfs")
|
||||
require.True(t, t.Run("fixtures", func(t *testing.T) {
|
||||
t.Run("history", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1")
|
||||
sha = runGit(t, "-C", work, "rev-parse", "HEAD")
|
||||
runGit(t, "-C", work, "tag", "v1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c2")
|
||||
runGit(t, "-C", work, "checkout", "-b", "feature")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "f1")
|
||||
runGit(t, "-C", work, "checkout", "--orphan", "trunk")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t1")
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "t2")
|
||||
runGit(t, "-C", work, "tag", "trunk", "main")
|
||||
runGit(t, "clone", "--bare", "--branch", "main", work, filepath.Join(server, "org", "repo"))
|
||||
runGit(t, "clone", "--bare", "--branch", "trunk", work, filepath.Join(server, "other", "repo"))
|
||||
runGit(t, "-C", filepath.Join(server, "other", "repo"), "symbolic-ref", "refs/remotes/origin/HEAD", "refs/heads/main")
|
||||
})
|
||||
t.Run("sha256", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sha256Repo := filepath.Join(server, "org", "sha256")
|
||||
runGit(t, "init", "--object-format=sha256", "--initial-branch=main", sha256Repo)
|
||||
runGit(t, "-C", sha256Repo, "commit", "--allow-empty", "-m", "s1")
|
||||
runGit(t, "-C", sha256Repo, "checkout", "--detach")
|
||||
})
|
||||
t.Run("submodules", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
publish(t, server, "org/nested", map[string]string{"nested.txt": "nested"}, nil)
|
||||
publish(t, server, "org/child", map[string]string{"child.txt": "child"}, map[string]string{"nested": "../nested"})
|
||||
publish(t, server, "org/content", map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b"}, map[string]string{"sub": "../child"})
|
||||
runGit(t, "-C", filepath.Join(server, "org", "content"), "config", "uploadpack.allowFilter", "true")
|
||||
})
|
||||
t.Run("lfs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
oid := fmt.Sprintf("%x", sha256.Sum256([]byte("big")))
|
||||
pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize 3\n"
|
||||
publish(t, server, "org/lfs", map[string]string{".gitattributes": "*.bin filter=lfs diff=lfs merge=lfs -text\n", "keep/big.bin": pointer, "drop/big.bin": pointer}, nil)
|
||||
object := filepath.Join(server, "org", "lfs", "lfs", "objects", oid[:2], oid[2:4], oid)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(object), 0o755))
|
||||
require.NoError(t, os.WriteFile(object, []byte("big"), 0o644))
|
||||
})
|
||||
}))
|
||||
host := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(host, "marker"), []byte("marker"), 0o644))
|
||||
tools := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tools, "ssh"), []byte(`#!/bin/sh
|
||||
test -s "$2" || exit 1
|
||||
printf '%s\n' "$@" > "$SSH_ARGS"
|
||||
for last; do :; done
|
||||
exec sh -c "$(printf '%s' "$last" | sed "s#git-upload-pack '/\(.*\)\.git'#git upload-pack '$SSH_SERVER/\1'#")"
|
||||
`), 0o755))
|
||||
globalConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(globalConfig, []byte("[protocol \"file\"]\n\tallow = always\n"), 0o644))
|
||||
sshEvent := func(url string) map[string]any { return map[string]any{"repository": map[string]any{"ssh_url": url}} }
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
bind, noSkip bool
|
||||
inputs map[string]string
|
||||
dst string
|
||||
log []string
|
||||
hostCopy, err bool
|
||||
name string
|
||||
bind, noSkip bool
|
||||
eventRef string
|
||||
event map[string]any
|
||||
before, inputs map[string]string
|
||||
dst, head, err string
|
||||
log, refs []string
|
||||
files, config map[string]string
|
||||
}{
|
||||
{name: "bound workspace is skipped", bind: true},
|
||||
{name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "/workspace/src", hostCopy: true},
|
||||
{name: "bound path clones event sha", bind: true, inputs: map[string]string{"path": "src"}, dst: "/workspace/src", log: []string{"c1"}},
|
||||
{name: "no-skip clones event sha", noSkip: true, dst: "/workspace", log: []string{"c1"}},
|
||||
{name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, dst: "/workspace", log: []string{"f1"}},
|
||||
{name: "tag ref wins over same-named branch", inputs: map[string]string{"ref": "refs/tags/trunk"}, dst: "/workspace", log: []string{"c2"}},
|
||||
{name: "fetch-depth 0 clones full history", inputs: map[string]string{"ref": "feature", "fetch-depth": "0"}, dst: "/workspace", log: []string{"f1", "c2", "c1"}},
|
||||
{name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, dst: "/workspace", log: []string{"t2"}},
|
||||
{name: "unsupported input", inputs: map[string]string{"submodules": "true"}, err: true},
|
||||
{name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: true},
|
||||
{name: "absolute path", inputs: map[string]string{"path": "/x"}, err: true},
|
||||
{name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: true},
|
||||
{name: "workflow ref copies host workdir", inputs: map[string]string{"ref": "main", "path": "a/../src"}, dst: "src", files: map[string]string{"marker": "marker"}},
|
||||
{name: "bound path checks out the event branch", bind: true, inputs: map[string]string{"path": "src"}, dst: "src", head: "main", log: []string{"c1"}},
|
||||
{name: "no-skip checks out the event branch", noSkip: true, head: "main", log: []string{"c1"}, refs: []string{"refs/remotes/origin/main"}},
|
||||
{name: "event tag is kept", noSkip: true, eventRef: "refs/tags/v1", log: []string{"c1"}, refs: []string{"refs/tags/v1"}},
|
||||
{name: "moved event tag fails", noSkip: true, eventRef: "refs/tags/trunk", err: "does not point to the expected commit " + sha},
|
||||
{name: "deleted event tag fails", noSkip: true, eventRef: "refs/tags/deleted", err: "checkout: clone"},
|
||||
{
|
||||
name: "sha input removes previously persisted credentials without clean", noSkip: true, before: map[string]string{"ref": sha},
|
||||
inputs: map[string]string{"ref": sha, "persist-credentials": "false", "clean": "false"},
|
||||
log: []string{"c1"},
|
||||
},
|
||||
{name: "ref clones shallow", inputs: map[string]string{"ref": "refs/heads/feature"}, head: "feature", log: []string{"f1"}},
|
||||
{name: "tag ref wins over same-named branch and is kept", inputs: map[string]string{"ref": "refs/tags/trunk"}, log: []string{"c2"}, refs: []string{"refs/tags/trunk"}},
|
||||
{name: "branch ref wins over same-named tag", inputs: map[string]string{"ref": "refs/heads/trunk"}, head: "trunk", log: []string{"t2"}},
|
||||
{
|
||||
name: "short branch wins over same-named tag and fetch-tags fetches every tag", inputs: map[string]string{"ref": "trunk", "fetch-tags": "true"},
|
||||
head: "trunk", log: []string{"t2"}, refs: []string{"refs/tags/trunk", "refs/tags/v1"},
|
||||
},
|
||||
{name: "short tag", inputs: map[string]string{"ref": "v1"}, log: []string{"c1"}, refs: []string{"refs/tags/v1"}},
|
||||
{
|
||||
name: "fetch-depth 0 deepens a shallow checkout to every branch and tag", before: map[string]string{"ref": "feature"},
|
||||
inputs: map[string]string{"ref": "refs/heads/feature", "fetch-depth": "0"}, head: "feature", log: []string{"f1", "c2", "c1"},
|
||||
refs: []string{"refs/remotes/origin/feature", "refs/remotes/origin/main", "refs/remotes/origin/trunk", "refs/tags/trunk", "refs/tags/v1"},
|
||||
},
|
||||
{name: "other repository defaults to its default branch", inputs: map[string]string{"repository": "other/repo"}, head: "trunk", log: []string{"t2"}},
|
||||
{name: "explicit HEAD checks out the default branch", inputs: map[string]string{"repository": "other/repo", "ref": "HEAD"}, head: "trunk", log: []string{"t2"}},
|
||||
{name: "detached sha256 HEAD is checked out", inputs: map[string]string{"repository": "org/sha256"}, log: []string{"s1"}},
|
||||
{
|
||||
name: "ssh key with the advertised endpoint persists", event: sshEvent("ssh://[email protected]:2222/org/repo.git"),
|
||||
inputs: map[string]string{"ssh-key": "KEY", "ssh-user": "me", "ssh-known-hosts": "ssh.example ssh-ed25519 AAAA"},
|
||||
head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://[email protected]:2222/org/repo.git"},
|
||||
},
|
||||
{
|
||||
name: "ssh key without persisting is emptied", event: sshEvent("[email protected]:org/repo.git"),
|
||||
inputs: map[string]string{"ssh-key": "KEY", "persist-credentials": "false"},
|
||||
head: "main", log: []string{"c1"}, config: map[string]string{"remote.origin.url": "ssh://[email protected]/org/repo.git"},
|
||||
},
|
||||
{
|
||||
name: "sparse cone checkout filters blobs", inputs: map[string]string{"repository": "org/content", "sparse-checkout": "a\n", "show-progress": "false"},
|
||||
head: "main", log: []string{"content"}, files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": ""},
|
||||
config: map[string]string{"remote.origin.partialclonefilter": "blob:none"},
|
||||
},
|
||||
{
|
||||
name: "non-cone sparse checkout with an explicit filter", head: "main", log: []string{"content"},
|
||||
inputs: map[string]string{"repository": "org/content", "sparse-checkout": "b/b.txt", "sparse-checkout-cone-mode": "false", "filter": "blob:limit=1024"},
|
||||
files: map[string]string{"root.txt": "", "a/a.txt": "", "b/b.txt": "b"}, config: map[string]string{"remote.origin.partialclonefilter": "blob:limit=1024"},
|
||||
},
|
||||
{
|
||||
name: "full checkout after a sparse one restores every file", before: map[string]string{"repository": "org/content", "sparse-checkout": "a"},
|
||||
inputs: map[string]string{"repository": "org/content", "set-safe-directory": "false"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"root.txt": "root", "a/a.txt": "a", "b/b.txt": "b", "sub/child.txt": ""},
|
||||
},
|
||||
{
|
||||
name: "recursive submodules after plain ones", before: map[string]string{"repository": "org/content", "submodules": "true"},
|
||||
inputs: map[string]string{"repository": "org/content", "submodules": "Recursive"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"sub/child.txt": "child", "sub/nested/nested.txt": "nested"},
|
||||
},
|
||||
{
|
||||
name: "lfs content stays when narrowing a full checkout to a sparse one", before: map[string]string{"repository": "org/lfs", "lfs": "true"},
|
||||
inputs: map[string]string{"repository": "org/lfs", "lfs": "true", "sparse-checkout": "keep"}, head: "main", log: []string{"content"},
|
||||
files: map[string]string{"keep/big.bin": "big", "drop/big.bin": ""},
|
||||
},
|
||||
{name: "unknown ref", inputs: map[string]string{"ref": "missing"}, err: "has no ref"},
|
||||
{name: "unsupported input", inputs: map[string]string{"github-server-url": "https://example.com"}, err: "unsupported input"},
|
||||
{name: "relative path escaping workspace", inputs: map[string]string{"path": "../x"}, err: "must be relative"},
|
||||
{name: "absolute path", inputs: map[string]string{"path": "/x"}, err: "must be relative"},
|
||||
{name: "negative fetch-depth", inputs: map[string]string{"fetch-depth": "-1"}, err: "non-negative"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fc := &fakeContainer{}
|
||||
err := Main(t.Context(), &action.Context{
|
||||
Container: fc,
|
||||
Github: &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", Sha: sha, ServerURL: server},
|
||||
Inputs: tc.inputs,
|
||||
HostWorkdir: host,
|
||||
Workspace: "/workspace",
|
||||
BindWorkdir: tc.bind,
|
||||
NoSkipCheckout: tc.noSkip,
|
||||
})
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, fc.dst)
|
||||
t.Parallel()
|
||||
if tc.inputs["lfs"] == "true" && lfsErr != nil {
|
||||
t.Skip("needs git-lfs")
|
||||
}
|
||||
workspace, runnerTemp := t.TempDir(), t.TempDir()
|
||||
output := filepath.Join(runnerTemp, "output")
|
||||
checkout := func(inputs map[string]string) error {
|
||||
return Main(t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard},
|
||||
Github: &model.GithubContext{
|
||||
Repository: "org/repo", Ref: cmp.Or(tc.eventRef, "refs/heads/main"), Sha: sha, ServerURL: "file://" + server, Token: "tok", Event: tc.event,
|
||||
},
|
||||
Inputs: inputs,
|
||||
HostWorkdir: host,
|
||||
Workspace: workspace,
|
||||
BindWorkdir: tc.bind,
|
||||
NoSkipCheckout: tc.noSkip,
|
||||
Env: map[string]string{
|
||||
"PATH": tools + string(os.PathListSeparator) + os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": globalConfig, "GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_DIR": host, "GITHUB_OUTPUT": output, "SSH_SERVER": server, "SSH_ARGS": filepath.Join(runnerTemp, "ssh-args"),
|
||||
},
|
||||
})
|
||||
}
|
||||
dst := filepath.Join(workspace, tc.dst)
|
||||
if tc.err != "" {
|
||||
require.ErrorContains(t, checkout(tc.inputs), tc.err)
|
||||
if tc.eventRef == "" {
|
||||
entries, _ := os.ReadDir(workspace)
|
||||
assert.Empty(t, entries)
|
||||
} else {
|
||||
assert.Nil(t, gitLog(dst))
|
||||
}
|
||||
return
|
||||
}
|
||||
first := tc.before
|
||||
if first == nil {
|
||||
first = tc.inputs
|
||||
}
|
||||
require.NoError(t, checkout(first))
|
||||
repos := []string{dst}
|
||||
if fileExists(filepath.Join(dst, "sub", ".git")) {
|
||||
repos = append(repos, filepath.Join(dst, "sub"))
|
||||
}
|
||||
for _, repo := range repos {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repo, "untracked"), nil, 0o644))
|
||||
}
|
||||
require.NoError(t, checkout(tc.inputs))
|
||||
outputs, err := os.ReadFile(output)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.dst, fc.dst)
|
||||
assert.Equal(t, tc.log, fc.log)
|
||||
assert.Equal(t, tc.hostCopy, strings.HasPrefix(fc.src, host))
|
||||
if tc.log != nil {
|
||||
assert.NoDirExists(t, fc.src)
|
||||
assert.Equal(t, tc.log, gitLog(dst))
|
||||
for name, content := range tc.files {
|
||||
body, err := os.ReadFile(filepath.Join(dst, name))
|
||||
assert.Equal(t, content, strings.TrimSpace(string(body)), name)
|
||||
assert.Equal(t, content == "", os.IsNotExist(err), name)
|
||||
}
|
||||
assert.Equal(t, tc.files["marker"] != "", fileExists(filepath.Join(dst, "marker")))
|
||||
for key, value := range tc.config {
|
||||
assert.Equal(t, value, gitOutput(dst, "config", key), key)
|
||||
}
|
||||
if tc.refs != nil {
|
||||
assert.Subset(t, strings.Fields(runGit(t, "-C", dst, "for-each-ref", "--format=%(refname)")), tc.refs)
|
||||
}
|
||||
if tc.log == nil {
|
||||
assert.Contains(t, string(outputs), "commit="+sha+"\n")
|
||||
assert.Contains(t, string(outputs), "ref="+cmp.Or(tc.eventRef, "refs/heads/main")+"\n")
|
||||
return
|
||||
}
|
||||
if tc.head == "" {
|
||||
assert.Empty(t, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
} else {
|
||||
assert.Equal(t, "refs/heads/"+tc.head, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
assert.Equal(t, "origin/"+tc.head, gitOutput(dst, "rev-parse", "--abbrev-ref", "@{u}"))
|
||||
}
|
||||
assert.Contains(t, string(outputs), "commit="+gitOutput(dst, "rev-parse", "HEAD")+"\n")
|
||||
if inputRef := tc.inputs["ref"]; inputRef != "" && inputRef != sha {
|
||||
assert.Contains(t, string(outputs), "ref="+inputRef+"\n")
|
||||
}
|
||||
assert.Equal(t, tc.inputs["sparse-checkout"] != "", gitOutput(dst, "config", "--local", "extensions.worktreeConfig") == "true")
|
||||
persisted := tc.inputs["persist-credentials"] != "false"
|
||||
for _, repo := range repos {
|
||||
assert.Equal(t, tc.inputs["clean"] == "false", fileExists(filepath.Join(repo, "untracked")), repo)
|
||||
assert.Len(t, strings.Fields(gitOutput(repo, "config", "--local", "--get-all", credentialsInclude)), 1, repo)
|
||||
headers, _ := exec.Command("git", "-C", repo, "config", "--get-all", "http.file://"+server+"/.extraheader").Output()
|
||||
assert.Equal(t, persisted, string(headers) == "\nAuthorization: Basic dG9rZW46dG9r\n", repo)
|
||||
}
|
||||
if tc.inputs["ssh-key"] != "" {
|
||||
keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-ssh-key-*"))
|
||||
require.Len(t, keys, 1)
|
||||
key, _ := os.ReadFile(keys[0])
|
||||
if persisted {
|
||||
assert.Equal(t, "KEY\n", string(key))
|
||||
} else {
|
||||
assert.Empty(t, key)
|
||||
}
|
||||
args, _ := os.ReadFile(filepath.Join(runnerTemp, "ssh-args"))
|
||||
assert.Contains(t, string(args), "StrictHostKeyChecking=yes")
|
||||
assert.Equal(t, tc.inputs["ssh-known-hosts"] != "", strings.Contains(string(args), "UserKnownHostsFile="))
|
||||
assert.Equal(t, persisted, gitOutput(dst, "config", "core.sshCommand") != "")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutOffersClientCertOnlyDuringTheStep(t *testing.T) {
|
||||
var gotCert bool
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotCert = len(r.TLS.PeerCertificates) > 0
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
pair := server.TLS.Certificates[0]
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(pair.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
dir, workspace, runnerTemp := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
certFile, keyFile := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: pair.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
|
||||
require.ErrorContains(t, Main(t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard},
|
||||
Github: &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", ServerURL: server.URL},
|
||||
Inputs: map[string]string{"repository": "other/repo"},
|
||||
Workspace: workspace,
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
Env: map[string]string{"PATH": os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_NOSYSTEM": "1"},
|
||||
}), "checkout: other/repo@")
|
||||
require.True(t, gotCert)
|
||||
keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-client-key-*"))
|
||||
require.Len(t, keys, 1)
|
||||
key, err := os.ReadFile(keys[0])
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, key)
|
||||
}
|
||||
|
||||
func publish(t *testing.T, server, name string, files, submodules map[string]string) {
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
for file, content := range files {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(work, file)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(work, file), []byte(content), 0o644))
|
||||
}
|
||||
runGit(t, "-C", work, "add", ".")
|
||||
for path, url := range submodules {
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".path", path)
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".url", url)
|
||||
commit := runGit(t, "--git-dir", filepath.Join(server, name, url), "rev-parse", "main")
|
||||
runGit(t, "-C", work, "update-index", "--add", "--cacheinfo", "160000,"+commit+","+path)
|
||||
runGit(t, "-C", work, "add", ".gitmodules")
|
||||
}
|
||||
runGit(t, "-C", work, "commit", "-m", "content")
|
||||
runGit(t, "clone", "--bare", "--quiet", work, filepath.Join(server, name))
|
||||
}
|
||||
|
||||
func gitOutput(dir string, args ...string) string {
|
||||
out, _ := exec.Command("git", append([]string{"-C", dir}, args...)...).Output()
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func fileExists(name string) bool {
|
||||
_, err := os.Stat(name)
|
||||
return err == nil
|
||||
}
|
||||
@@ -5,6 +5,8 @@ package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"gitea.dev/actionslib/runner/v1"
|
||||
@@ -18,3 +20,9 @@ type Client interface {
|
||||
UpdateLog(context.Context, *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error)
|
||||
UpdateTask(context.Context, *connect.Request[runnerv1.UpdateTaskRequest]) (*connect.Response[runnerv1.UpdateTaskResponse], error)
|
||||
}
|
||||
|
||||
// IsRegistrationRejected reports whether Gitea itself answered "unregistered runner", unlike a proxy's 401.
|
||||
func IsRegistrationRejected(err error) bool {
|
||||
var connectErr *connect.Error
|
||||
return errors.As(err, &connectErr) && strings.HasSuffix(connectErr.Message(), "unregistered runner")
|
||||
}
|
||||
@@ -6,6 +6,7 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,29 +19,54 @@ import (
|
||||
"gitea.dev/actionslib/runner/v1/runnerv1connect"
|
||||
)
|
||||
|
||||
func getHTTPClient(endpoint string, insecure bool, timeout time.Duration) *http.Client {
|
||||
func NewTransport(endpoint string, tlsConfig *tls.Config) *http.Transport {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
MaxIdleConns: 10,
|
||||
MaxIdleConnsPerHost: 10, // All requests go to one host; default is 2 which causes frequent reconnects.
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "https://") && insecure {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "https://") {
|
||||
transport.TLSClientConfig = tlsConfig
|
||||
}
|
||||
return transport
|
||||
}
|
||||
|
||||
func getHTTPClient(endpoint string, tlsConfig *tls.Config, timeout time.Duration, extraHeaders map[string]string) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: NewTransport(endpoint, tlsConfig),
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 { // a custom CheckRedirect replaces net/http's own limit
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
from, to := via[0].URL, req.URL
|
||||
if from.Hostname() != to.Hostname() || from.Scheme == "https" && to.Scheme != "https" {
|
||||
for name, value := range extraHeaders {
|
||||
if req.Header.Get(name) == value {
|
||||
req.Header.Del(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: timeout}
|
||||
}
|
||||
|
||||
// New returns a new runner client. timeout bounds every RPC: without it a
|
||||
// stalled connection parks the reporter for the whole job context, so logs and
|
||||
// heartbeats stop together and the task is reaped as a zombie.
|
||||
func New(endpoint string, insecure bool, uuid, token string, timeout time.Duration, opts ...connect.ClientOption) *HTTPClient {
|
||||
func New(endpoint string, tlsConfig *tls.Config, uuid, token string, timeout time.Duration, extraHeaders map[string]string, opts ...connect.ClientOption) *HTTPClient {
|
||||
baseURL := strings.TrimRight(endpoint, "/") + "/api/actions"
|
||||
|
||||
opts = append(opts, connect.WithInterceptors(connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc {
|
||||
return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
|
||||
for name, value := range extraHeaders {
|
||||
if req.Header().Get(name) == "" { // keep protocol headers such as Content-Type
|
||||
req.Header().Set(name, value)
|
||||
}
|
||||
}
|
||||
req.Header().Set("User-Agent", "gitea-runner/"+ver.Version())
|
||||
if uuid != "" {
|
||||
req.Header().Set(protocol.UUIDHeader, uuid)
|
||||
@@ -52,7 +78,7 @@ func New(endpoint string, insecure bool, uuid, token string, timeout time.Durati
|
||||
}
|
||||
})))
|
||||
|
||||
httpClient := getHTTPClient(endpoint, insecure, timeout)
|
||||
httpClient := getHTTPClient(endpoint, tlsConfig, timeout, extraHeaders)
|
||||
return &HTTPClient{
|
||||
PingServiceClient: pingv1connect.NewPingServiceClient(
|
||||
httpClient,
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -19,7 +21,7 @@ import (
|
||||
func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
|
||||
t.Setenv("HTTP_PROXY", "http://proxy.example.com:8080")
|
||||
|
||||
client := getHTTPClient("http://gitea.example.com", false, time.Minute)
|
||||
client := getHTTPClient("http://gitea.example.com", nil, time.Minute, nil)
|
||||
require.Equal(t, time.Minute, client.Timeout)
|
||||
transport, ok := client.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
@@ -33,29 +35,10 @@ func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
|
||||
require.Equal(t, "http://proxy.example.com:8080", proxyURL.String())
|
||||
}
|
||||
|
||||
func TestGetHTTPClientInsecureTLS(t *testing.T) {
|
||||
// insecure only takes effect for https endpoints
|
||||
httpsInsecure := getHTTPClient("https://gitea.example.com", true, time.Minute)
|
||||
transport, ok := httpsInsecure.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, transport.TLSClientConfig)
|
||||
require.True(t, transport.TLSClientConfig.InsecureSkipVerify)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
endpoint string
|
||||
insecure bool
|
||||
}{
|
||||
{"https secure", "https://gitea.example.com", false},
|
||||
{"http insecure ignored", "http://gitea.example.com", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := getHTTPClient(tc.endpoint, tc.insecure, time.Minute)
|
||||
tr, ok := c.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.Nil(t, tr.TLSClientConfig)
|
||||
})
|
||||
}
|
||||
func TestGetHTTPClientAppliesTLSConfigOnlyToHTTPS(t *testing.T) {
|
||||
tlsConfig := &tls.Config{}
|
||||
require.Same(t, tlsConfig, getHTTPClient("https://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
|
||||
require.Nil(t, getHTTPClient("http://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
|
||||
}
|
||||
|
||||
func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
@@ -69,7 +52,12 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
// trailing slash must be trimmed before "/api/actions" is appended
|
||||
c := New(server.URL+"/", false, "the-uuid", "the-token", time.Minute)
|
||||
c := New(server.URL+"/", nil, "the-uuid", "the-token", time.Minute, map[string]string{
|
||||
"X-Proxy-Token": "proxy-token",
|
||||
protocol.UUIDHeader: "other-uuid",
|
||||
protocol.TokenHeader: "other-token",
|
||||
"Content-Type": "text/plain",
|
||||
})
|
||||
// Address returns the endpoint as supplied (untrimmed)
|
||||
require.Equal(t, server.URL+"/", c.Address())
|
||||
|
||||
@@ -79,6 +67,8 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
|
||||
require.True(t, strings.HasPrefix(gotPath, "/api/actions/"), "unexpected path %q", gotPath)
|
||||
require.Equal(t, "the-uuid", gotHeaders.Get(protocol.UUIDHeader))
|
||||
require.Equal(t, "the-token", gotHeaders.Get(protocol.TokenHeader))
|
||||
require.Equal(t, "proxy-token", gotHeaders.Get("X-Proxy-Token"))
|
||||
require.Equal(t, "application/proto", gotHeaders.Get("Content-Type"))
|
||||
}
|
||||
|
||||
func TestNewOmitsEmptyHeaders(t *testing.T) {
|
||||
@@ -89,9 +79,52 @@ func TestNewOmitsEmptyHeaders(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := New(server.URL, false, "", "", time.Minute)
|
||||
c := New(server.URL, nil, "", "", time.Minute, nil)
|
||||
_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
|
||||
require.Empty(t, gotHeaders.Get(protocol.UUIDHeader))
|
||||
require.Empty(t, gotHeaders.Get(protocol.TokenHeader))
|
||||
}
|
||||
|
||||
func TestNewStopsRedirectLoop(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.Path, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := New(server.URL, nil, "", "", time.Minute, nil)
|
||||
_, err := c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
require.ErrorContains(t, err, "stopped after 10 redirects")
|
||||
}
|
||||
|
||||
func TestNewRedirectKeepsExtraHeadersOnlyOnSameHost(t *testing.T) {
|
||||
var gotHeaders http.Header
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotHeaders = r.Header.Clone()
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer target.Close()
|
||||
_, port, err := net.SplitHostPort(target.Listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tc := range []struct {
|
||||
newOrigin func(http.Handler) *httptest.Server
|
||||
host string
|
||||
kept bool
|
||||
}{
|
||||
{httptest.NewServer, "127.0.0.1", true},
|
||||
{httptest.NewServer, "localhost", false},
|
||||
{httptest.NewTLSServer, "127.0.0.1", false},
|
||||
} {
|
||||
gotHeaders = nil
|
||||
origin := tc.newOrigin(http.RedirectHandler("http://"+net.JoinHostPort(tc.host, port), http.StatusPermanentRedirect))
|
||||
c := New(origin.URL, &tls.Config{InsecureSkipVerify: true}, "the-uuid", "the-token", time.Minute, map[string]string{"X-Proxy-Token": "proxy-token", "Content-Type": "text/plain"})
|
||||
_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
|
||||
origin.Close()
|
||||
|
||||
require.NotNil(t, gotHeaders, origin.URL)
|
||||
require.Equal(t, tc.kept, gotHeaders.Get("X-Proxy-Token") != "", "%s to %s", origin.URL, tc.host)
|
||||
require.Equal(t, "the-token", gotHeaders.Get(protocol.TokenHeader))
|
||||
require.Equal(t, "application/proto", gotHeaders.Get("Content-Type"))
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
# Every option with its default value, all commented out. Read this file, do not copy it.
|
||||
# `./gitea-runner config init` writes a config file to copy the lines you change into.
|
||||
# Values may read the runner's environment as ${NAME}, an unset NAME fails the load. $${NAME} is a literal ${NAME}.
|
||||
|
||||
# Logging for the runner process itself (messages printed to stderr), plus the copy of
|
||||
# each task's log kept under log.job. Neither controls how workflow step output is streamed
|
||||
@@ -40,6 +41,9 @@ runner:
|
||||
#shutdown_timeout: 0s
|
||||
# Whether skip verifying the TLS certificate of the Gitea instance.
|
||||
#insecure: false
|
||||
# PEM client certificate and key for a Gitea instance that requires mutual TLS.
|
||||
#client_cert_file: /etc/gitea-runner/client.crt
|
||||
#client_key_file: /etc/gitea-runner/client.key
|
||||
# The timeout for fetching the job from the Gitea instance.
|
||||
# Values above the 60s RPC timeout are capped to it.
|
||||
#fetch_timeout: 5s
|
||||
@@ -53,12 +57,13 @@ runner:
|
||||
# scratch directories (left behind when a host cleanup delete stalls) older than
|
||||
# this duration. Setting either workdir_cleanup_age or idle_cleanup_interval to 0
|
||||
# (or any non-positive value) disables stale-directory cleanup entirely, along with
|
||||
# the docker network and volume cleanup below.
|
||||
# the docker and Kubernetes cleanup below.
|
||||
#workdir_cleanup_age: 24h
|
||||
# Cadence for the idle cleanup pass. Besides the directories above, on runners that use
|
||||
# docker it removes the per-job networks and volumes of jobs this runner did not live to
|
||||
# tear down, which would otherwise hold a subnet of the daemon address pool and the job's
|
||||
# workspace contents until the host is rebuilt.
|
||||
# workspace contents until the host is rebuilt. On runners with kubernetes labels it
|
||||
# removes such jobs' pods and Secrets.
|
||||
#idle_cleanup_interval: 10m
|
||||
# The base interval for periodic log flush to the Gitea instance.
|
||||
# Logs may be sent earlier if the buffer reaches log_report_batch_size
|
||||
@@ -82,6 +87,11 @@ runner:
|
||||
# and github_mirror is not empty. In this case,
|
||||
# it replaces https://github.com with the value here, which is useful for some special network environments.
|
||||
#github_mirror: ''
|
||||
# Extra HTTP headers for runner API requests to Gitea, e.g. reverse proxy authentication.
|
||||
# Not used for action clones, artifacts, job summaries or jobs. Runner headers take precedence.
|
||||
# Dropped on redirects to another hostname, or to HTTP if the instance URL uses HTTPS.
|
||||
#extra_headers:
|
||||
# X-Proxy-Token: 'example'
|
||||
# When true (the default), fetch only the requested ref of an action repository (e.g. actions/checkout@v4) at depth 1 instead of cloning every branch's full history.
|
||||
# Set to false to clone the full history.
|
||||
#action_shallow_clone: true
|
||||
@@ -106,8 +116,7 @@ runner:
|
||||
# terminal; tools like `docker build` emit redrawing progress frames into the captured log
|
||||
# when a TTY is present.
|
||||
#allocate_pty: false
|
||||
# Image for a job whose runs-on matches none of the labels above. A runner without a
|
||||
# docker daemon runs such a job on the host instead.
|
||||
# Image for a job whose runs-on matches none of the labels above, see the Labels section of the README for where it runs.
|
||||
#default_image: "docker.gitea.com/runner-images:ubuntu-latest"
|
||||
# What to mount at RUNNER_TOOL_CACHE (/opt/hostedtoolcache), where setup actions install tools:
|
||||
# none: nothing. A docker job sees what its image ships there, a host job an empty dir, and
|
||||
@@ -217,6 +226,7 @@ container:
|
||||
# created by the runner. Omit the entire block to use Docker's defaults. An auto-created
|
||||
# network is labelled com.gitea.runner.uuid=<this runner's uuid>, which is how the idle
|
||||
# cleanup tells its own leftovers apart from those of other runners on the same daemon.
|
||||
# It also takes the `mtu` set in the Docker daemon config.
|
||||
#network_create_options:
|
||||
# enable_ipv4: true # Omit to use Docker's default (IPv4 enabled). Set false to disable IPv4.
|
||||
# enable_ipv6: false # Omit to use Docker's default (IPv6 disabled). Enabling it requires dockerd started with --ipv6.
|
||||
@@ -237,14 +247,15 @@ container:
|
||||
# If it's empty, /workspace will be used.
|
||||
# Purely numeric subdirectories under this path are reserved for task workspaces and may be removed by idle cleanup.
|
||||
#workdir_parent:
|
||||
# Volumes (including bind mounts) can be mounted to containers. Glob syntax is supported, see https://github.com/bmatcuk/doublestar#patterns
|
||||
# `*` matches within a single path segment, `**` also matches across path separators.
|
||||
# You can specify multiple volumes. If the sequence is empty, no volumes can be mounted.
|
||||
# For example, if you only allow containers to mount the `data` volume and all the json files in `/src`, you should change the config to:
|
||||
# Volumes and bind mounts containers may mount, as globs (https://github.com/bmatcuk/doublestar#patterns).
|
||||
# `*` matches within a path segment, `**` across segments. Empty allows none. tmpfs mounts are always allowed.
|
||||
# A `:ro` suffix allows only read-only mounts, unless an entry without it matches too.
|
||||
# For example, to allow the `data` volume and all json files in `/src`, and `/etc/ssl/certs` only read-only:
|
||||
# valid_volumes:
|
||||
# - data
|
||||
# - /src/*.json
|
||||
# If you want to allow any volume, please use the following configuration:
|
||||
# - /etc/ssl/certs:ro
|
||||
# To allow any volume:
|
||||
# valid_volumes:
|
||||
# - '**'
|
||||
#valid_volumes: []
|
||||
@@ -274,12 +285,23 @@ container:
|
||||
# How long a job waits for a service container that declares a healthcheck to become
|
||||
# healthy. A negative value (e.g. -1s) starts the steps without waiting.
|
||||
#service_ready_timeout: 5m
|
||||
# Remove the containers, networks and volumes a job creates through its Docker proxy when
|
||||
# the job ends. Set to false when jobs deploy services to the runner's Docker daemon.
|
||||
#sweep: true
|
||||
|
||||
host:
|
||||
# The parent directory of a job's working directory.
|
||||
# If it's empty, $HOME/.cache/act/ will be used.
|
||||
#workdir_parent:
|
||||
|
||||
# Jobs that run as Kubernetes pods, see docs/kubernetes.md.
|
||||
kubernetes:
|
||||
#kubeconfig: ""
|
||||
#namespace: ""
|
||||
#pod_template: {}
|
||||
# Pod templates merged over pod_template, in runs-on order, for jobs whose runs-on has the label they are keyed by.
|
||||
#pod_templates: {}
|
||||
|
||||
# Optional local task-admission checks. Disabled by default. When enabled, low
|
||||
# disk space or a failing script pauses new task fetching; existing jobs continue.
|
||||
# No health checks run while any job is active; the last result is reused until idle.
|
||||
|
||||
+112
-21
@@ -4,12 +4,13 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.yaml.in/yaml/v4"
|
||||
"golang.org/x/net/http/httpguts"
|
||||
)
|
||||
|
||||
// RequestTimeout bounds every RPC to Gitea, and with it runner.fetch_timeout.
|
||||
@@ -32,6 +34,8 @@ const DefaultImage = "docker.gitea.com/runner-images:ubuntu-latest"
|
||||
// (so a programmatically built config still gets a sane bound).
|
||||
const DefaultPostTaskScriptTimeout = 5 * time.Minute
|
||||
|
||||
var environmentReference = regexp.MustCompile(`\$?\$\{[A-Za-z_][A-Za-z0-9_]*\}`)
|
||||
|
||||
// Minimal is the smallest config file that runs the runner: options it does not
|
||||
// name keep their default, and it names none.
|
||||
const Minimal = `# Minimal config file. Every option it does not set keeps its default.
|
||||
@@ -60,11 +64,14 @@ type Runner struct {
|
||||
Timeout time.Duration `yaml:"timeout"` // Timeout specifies the duration for runner timeout.
|
||||
ShutdownTimeout time.Duration `yaml:"shutdown_timeout"` // ShutdownTimeout specifies the duration to wait for running jobs to complete during a shutdown of the runner.
|
||||
Insecure bool `yaml:"insecure"` // Insecure indicates whether the runner operates in an insecure mode.
|
||||
ClientCertFile string `yaml:"client_cert_file"` // ClientCertFile is the PEM certificate presented to a Gitea instance that requires mutual TLS.
|
||||
ClientKeyFile string `yaml:"client_key_file"` // ClientKeyFile is the PEM private key paired with ClientCertFile.
|
||||
ExtraHeaders map[string]string `yaml:"extra_headers"` // ExtraHeaders adds HTTP headers to runner API requests to Gitea.
|
||||
FetchTimeout time.Duration `yaml:"fetch_timeout"` // FetchTimeout specifies the timeout duration for fetching resources.
|
||||
FetchInterval time.Duration `yaml:"fetch_interval"` // FetchInterval specifies the interval duration for fetching resources.
|
||||
FetchIntervalMax time.Duration `yaml:"fetch_interval_max"` // FetchIntervalMax specifies the maximum backoff interval when idle.
|
||||
WorkdirCleanupAge time.Duration `yaml:"workdir_cleanup_age"` // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker job resources older than this duration during idle cleanup.
|
||||
IdleCleanupInterval time.Duration `yaml:"idle_cleanup_interval"` // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks and volumes) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
|
||||
WorkdirCleanupAge time.Duration `yaml:"workdir_cleanup_age"` // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker and Kubernetes job resources older than this duration during idle cleanup.
|
||||
IdleCleanupInterval time.Duration `yaml:"idle_cleanup_interval"` // IdleCleanupInterval runs the idle cleanup (stale directories, orphaned docker networks and volumes, orphaned job pods) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
|
||||
LogReportInterval time.Duration `yaml:"log_report_interval"` // LogReportInterval specifies the base interval for periodic log flush.
|
||||
LogReportMaxLatency time.Duration `yaml:"log_report_max_latency"` // LogReportMaxLatency specifies the max time a log row can wait before being sent.
|
||||
LogReportBatchSize int `yaml:"log_report_batch_size"` // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
|
||||
@@ -174,6 +181,7 @@ type Container struct {
|
||||
DockerTimeout time.Duration `yaml:"docker_timeout"` // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner
|
||||
BindWorkdir bool `yaml:"bind_workdir"` // BindWorkdir mounts the workspace from a host directory instead of a Docker volume.
|
||||
ServiceReadyTimeout time.Duration `yaml:"service_ready_timeout"` // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting.
|
||||
Sweep *bool `yaml:"sweep"` // Sweep removes the containers, networks and volumes a job creates through its Docker proxy when the job ends. Unset means enabled.
|
||||
}
|
||||
|
||||
// Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses.
|
||||
@@ -194,6 +202,14 @@ type Host struct {
|
||||
WorkdirParent string `yaml:"workdir_parent"` // WorkdirParent specifies the parent directory for the host's working directory.
|
||||
}
|
||||
|
||||
// Kubernetes configures the pods jobs run in.
|
||||
type Kubernetes struct {
|
||||
Kubeconfig string `yaml:"kubeconfig"` // Kubeconfig to use instead of the in-cluster service account.
|
||||
Namespace string `yaml:"namespace"` // Namespace of job pods, empty uses the service account's, else the kubeconfig context's, else default.
|
||||
PodTemplate map[string]any `yaml:"pod_template"` // PodTemplate is a pod manifest the runner merges its own onto.
|
||||
PodTemplates map[string]map[string]any `yaml:"pod_templates"` // PodTemplates are merged over PodTemplate, in runs-on order, for jobs whose runs-on has the label they are keyed by.
|
||||
}
|
||||
|
||||
// Metrics represents the configuration for the Prometheus metrics endpoint.
|
||||
type Metrics struct {
|
||||
Enabled bool `yaml:"enabled"` // Enabled indicates whether the metrics endpoint is exposed.
|
||||
@@ -220,6 +236,7 @@ type Config struct {
|
||||
Host Host `yaml:"host"` // Host represents the configuration for the host.
|
||||
Metrics Metrics `yaml:"metrics"` // Metrics represents the configuration for the Prometheus metrics endpoint.
|
||||
HealthCheck HealthCheck `yaml:"health_check"` // HealthCheck controls opt-in local task-admission checks.
|
||||
Kubernetes Kubernetes `yaml:"kubernetes"`
|
||||
}
|
||||
|
||||
// LoadDefault returns the default configuration.
|
||||
@@ -233,14 +250,18 @@ func LoadDefault(file string) (*Config, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open config file %q: %w", file, err)
|
||||
}
|
||||
if err := yaml.Unmarshal(content, cfg); err != nil {
|
||||
var document yaml.Node
|
||||
if err := yaml.Unmarshal(content, &document); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
warnUnknownKeys(file, content)
|
||||
definedRunnerKeys, err = definedRunnerConfigKeys(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
|
||||
if err := expandEnvironment(&document); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
if err := document.Load(cfg, yaml.WithV3Defaults()); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
warnUnknownKeys(file, &document)
|
||||
definedRunnerKeys = definedRunnerConfigKeys(&document)
|
||||
}
|
||||
|
||||
if cfg.Runner.EnvFile != "" {
|
||||
@@ -255,6 +276,21 @@ func LoadDefault(file string) (*Config, error) {
|
||||
maps.Copy(cfg.Runner.Envs, envs)
|
||||
}
|
||||
}
|
||||
if (cfg.Runner.ClientCertFile == "") != (cfg.Runner.ClientKeyFile == "") {
|
||||
return nil, errors.New("runner.client_cert_file and runner.client_key_file must be set together")
|
||||
}
|
||||
if cfg.Runner.ClientCertFile != "" {
|
||||
if _, err := tls.LoadX509KeyPair(cfg.Runner.ClientCertFile, cfg.Runner.ClientKeyFile); err != nil {
|
||||
return nil, fmt.Errorf("load runner client certificate: %w", err)
|
||||
}
|
||||
for _, file := range []*string{&cfg.Runner.ClientCertFile, &cfg.Runner.ClientKeyFile} {
|
||||
abs, err := filepath.Abs(*file) // git resolves a relative path against the repository it runs in
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
*file = abs
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Log.Level == "" {
|
||||
cfg.Log.Level = "info"
|
||||
@@ -311,6 +347,9 @@ func LoadDefault(file string) (*Config, error) {
|
||||
if !slices.Contains(ToolCacheModes, cfg.Runner.ToolCacheMode) {
|
||||
return nil, fmt.Errorf("invalid runner.tool_cache_mode %q: must be one of %q", cfg.Runner.ToolCacheMode, ToolCacheModes)
|
||||
}
|
||||
if err := validateExtraHeaders(cfg.Runner.ExtraHeaders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Host.WorkdirParent == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
@@ -402,30 +441,67 @@ func LoadDefault(file string) (*Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (r *Runner) TLSConfig() *tls.Config {
|
||||
tlsConfig := &tls.Config{InsecureSkipVerify: r.Insecure}
|
||||
if r.ClientCertFile != "" {
|
||||
tlsConfig.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
pair, err := tls.LoadX509KeyPair(r.ClientCertFile, r.ClientKeyFile) // per handshake, so renewed files apply
|
||||
if err != nil || info.SupportsCertificate(&pair) != nil {
|
||||
return &tls.Certificate{}, err
|
||||
}
|
||||
return &pair, nil
|
||||
}
|
||||
}
|
||||
return tlsConfig
|
||||
}
|
||||
|
||||
// expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset.
|
||||
func expandEnvironment(node *yaml.Node) error {
|
||||
var errs []error
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
expanded := environmentReference.ReplaceAllStringFunc(node.Value, func(match string) string {
|
||||
if strings.HasPrefix(match, "$$") {
|
||||
return match[1:]
|
||||
}
|
||||
value, ok := os.LookupEnv(strings.Trim(match, "${}"))
|
||||
if !ok {
|
||||
errs = append(errs, fmt.Errorf("line %d: %s is not set", node.Line, match))
|
||||
}
|
||||
return value
|
||||
})
|
||||
if expanded != node.Value && node.Style&yaml.TaggedStyle == 0 {
|
||||
node.Tag = ""
|
||||
if expanded == "" && node.Style == 0 {
|
||||
node.Tag = "!!null"
|
||||
}
|
||||
}
|
||||
node.Value = expanded
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
for index, child := range node.Content {
|
||||
if node.Kind != yaml.MappingNode || index%2 == 1 {
|
||||
errs = append(errs, expandEnvironment(child))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// warnUnknownKeys reports keys the config does not define, which are otherwise ignored
|
||||
// without a trace. It only warns, so a config carrying keys from another runner version
|
||||
// still loads.
|
||||
func warnUnknownKeys(file string, content []byte) {
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
decoder.KnownFields(true)
|
||||
|
||||
func warnUnknownKeys(file string, document *yaml.Node) {
|
||||
var loadErrs *yaml.LoadErrors
|
||||
if err := decoder.Decode(&Config{}); errors.As(err, &loadErrs) {
|
||||
if err := document.Load(&Config{}, yaml.WithV3Defaults(), yaml.WithKnownFields()); errors.As(err, &loadErrs) {
|
||||
for _, loadErr := range loadErrs.Errors {
|
||||
log.Warnf("config file %q: %s, it will be ignored", file, loadErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal(content, &root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func definedRunnerConfigKeys(root *yaml.Node) map[string]bool {
|
||||
defined := map[string]bool{}
|
||||
if len(root.Content) == 0 {
|
||||
return defined, nil
|
||||
return defined
|
||||
}
|
||||
|
||||
doc := root.Content[0]
|
||||
@@ -441,7 +517,22 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
||||
break
|
||||
}
|
||||
|
||||
return defined, nil
|
||||
return defined
|
||||
}
|
||||
|
||||
func validateExtraHeaders(headers map[string]string) error {
|
||||
seen := make(map[string]string, len(headers))
|
||||
for name, value := range headers {
|
||||
if !httpguts.ValidHeaderFieldName(name) || !httpguts.ValidHeaderFieldValue(value) {
|
||||
return fmt.Errorf("invalid runner.extra_headers entry %q", name)
|
||||
}
|
||||
key := strings.ToLower(name)
|
||||
if other, ok := seen[key]; ok {
|
||||
return fmt.Errorf("runner.extra_headers sets %q and %q, which name the same header", other, name)
|
||||
}
|
||||
seen[key] = name
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveSecretFile reads key from key_file, so deployments can mount a secret instead of committing it.
|
||||
|
||||
@@ -4,6 +4,13 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -42,6 +49,81 @@ cache:
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClientCertIsValidatedAndAbsoluteOnLoadReadOnUseAndOfferedOnlyWhenAccepted(t *testing.T) {
|
||||
require.True(t, (&Runner{Insecure: true}).TLSConfig().InsecureSkipVerify)
|
||||
|
||||
dir := t.TempDir()
|
||||
certPath, keyPath := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
|
||||
writePair := func() []byte {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(1)}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
return der
|
||||
}
|
||||
load := func(certFile, keyFile string) (*Config, error) {
|
||||
return LoadDefault(write(t, "runner:\n client_cert_file: '"+certFile+"'\n client_key_file: '"+keyFile+"'\n"))
|
||||
}
|
||||
writePair()
|
||||
|
||||
_, err := load(certPath, "")
|
||||
require.ErrorContains(t, err, "must be set together")
|
||||
_, err = load(keyPath, keyPath)
|
||||
require.ErrorContains(t, err, "load runner client certificate")
|
||||
|
||||
t.Chdir(dir)
|
||||
cfg, err := load("client.crt", "client.key")
|
||||
require.NoError(t, err)
|
||||
require.True(t, filepath.IsAbs(cfg.Runner.ClientCertFile))
|
||||
require.True(t, filepath.IsAbs(cfg.Runner.ClientKeyFile))
|
||||
renewed := writePair()
|
||||
info := &tls.CertificateRequestInfo{Version: tls.VersionTLS13, SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}}
|
||||
pair, err := cfg.Runner.TLSConfig().GetClientCertificate(info)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, renewed, pair.Certificate[0])
|
||||
info.AcceptableCAs = [][]byte{[]byte("another CA")}
|
||||
pair, err = cfg.Runner.TLSConfig().GetClientCertificate(info)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, pair.Certificate)
|
||||
}
|
||||
|
||||
func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) {
|
||||
t.Setenv("RUNNER_CAPACITY", "4")
|
||||
t.Setenv("RUNNER_LABEL", "linux:host")
|
||||
t.Setenv("RUNNER_KEY", "fetch_timeout")
|
||||
t.Setenv("RUNNER_EMPTY", "")
|
||||
t.Setenv("RUNNER_MISSING", "")
|
||||
require.NoError(t, os.Unsetenv("RUNNER_MISSING"))
|
||||
path := write(t, `runner:
|
||||
capacity: ${RUNNER_CAPACITY}
|
||||
labels:
|
||||
- ${RUNNER_LABEL}
|
||||
envs:
|
||||
LITERAL: $${RUNNER_LABEL}
|
||||
${RUNNER_KEY}: 1s
|
||||
cache:
|
||||
enabled: ${RUNNER_EMPTY}
|
||||
`)
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 4, cfg.Runner.Capacity)
|
||||
assert.Equal(t, []string{"linux:host"}, cfg.Runner.Labels)
|
||||
assert.Equal(t, map[string]string{"LITERAL": "${RUNNER_LABEL}"}, cfg.Runner.Envs)
|
||||
assert.Equal(t, 5*time.Second, cfg.Runner.FetchTimeout)
|
||||
assert.True(t, *cfg.Cache.Enabled)
|
||||
|
||||
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_MISSING}\n file: ${RUNNER_MISSING}\n"))
|
||||
require.ErrorContains(t, err, "line 3: ${RUNNER_MISSING} is not set\nline 4: ${RUNNER_MISSING} is not set")
|
||||
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_LABEL}\n"))
|
||||
require.ErrorContains(t, err, "line 3:")
|
||||
}
|
||||
|
||||
func TestLoadDefault_CacheS3(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
accessPath := filepath.Join(dir, "access.key")
|
||||
@@ -89,6 +171,25 @@ func TestLoadDefault_ToolCacheMode(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "tool_cache_mode")
|
||||
}
|
||||
|
||||
func TestLoadDefault_LoadsExtraHeaders(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte("runner:\n extra_headers:\n X-Proxy-Token: proxy-token\n"), 0o600))
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]string{"X-Proxy-Token": "proxy-token"}, cfg.Runner.ExtraHeaders)
|
||||
|
||||
for _, headers := range []string{
|
||||
"X-Proxy Token: proxy-token\n",
|
||||
"X-Proxy-Token: |\n proxy-token\n",
|
||||
"X-Proxy-Token: a\n x-proxy-token: b\n",
|
||||
} {
|
||||
require.NoError(t, os.WriteFile(path, []byte("runner:\n extra_headers:\n "+headers), 0o600))
|
||||
_, err = LoadDefault(path)
|
||||
assert.ErrorContains(t, err, "runner.extra_headers", headers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) {
|
||||
cfg, err := LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -9,8 +9,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
SchemeHost = "host"
|
||||
SchemeDocker = "docker"
|
||||
SchemeHost = "host"
|
||||
SchemeDocker = "docker"
|
||||
SchemeKubernetes = "kubernetes"
|
||||
|
||||
// SelfHostedPlatform is the platform marker act treats as "run on the host".
|
||||
SelfHostedPlatform = "-self-hosted"
|
||||
@@ -42,8 +43,9 @@ func Parse(str string) (*Label, error) {
|
||||
if len(splits) >= 3 {
|
||||
label.Arg = splits[2]
|
||||
}
|
||||
if label.Schema != SchemeHost && label.Schema != SchemeDocker {
|
||||
// Not a schema we know: the colon belongs to the label name itself.
|
||||
kubernetes := label.Schema == SchemeKubernetes && strings.HasPrefix(label.Arg, "//") && len(label.Arg) > 2
|
||||
if label.Schema != SchemeHost && label.Schema != SchemeDocker && !kubernetes {
|
||||
// Not a schema we know, or kubernetes without an image: the colon belongs to the label name itself.
|
||||
return &Label{
|
||||
Name: str,
|
||||
Schema: SchemeHost,
|
||||
@@ -72,10 +74,11 @@ func (l Labels) PickPlatform(runsOn []string) string {
|
||||
case SchemeDocker:
|
||||
// "//" will be ignored
|
||||
platforms[label.Name] = strings.TrimPrefix(label.Arg, "//")
|
||||
case SchemeKubernetes:
|
||||
platforms[label.Name] = SchemeKubernetes + ":" + label.Arg
|
||||
case SchemeHost:
|
||||
platforms[label.Name] = SelfHostedPlatform
|
||||
default:
|
||||
// unreachable: Parse only produces host or docker schemas
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,6 +115,7 @@ func TestPickPlatform(t *testing.T) {
|
||||
ls := mustParse(t,
|
||||
"ubuntu:docker://node:18",
|
||||
"self-hosted:host",
|
||||
"k8s:kubernetes://node:24",
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
@@ -123,6 +124,7 @@ func TestPickPlatform(t *testing.T) {
|
||||
want string
|
||||
}{
|
||||
{"docker strips leading slashes", []string{"ubuntu"}, "node:18"},
|
||||
{"kubernetes keeps its scheme", []string{"k8s"}, "kubernetes://node:24"},
|
||||
{"host maps to self-hosted marker", []string{"self-hosted"}, SelfHostedPlatform},
|
||||
{"first match wins", []string{"self-hosted", "ubuntu"}, SelfHostedPlatform},
|
||||
{"unknown label picks nothing", []string{"windows"}, ""},
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +22,7 @@ const Namespace = "gitea_runner"
|
||||
//
|
||||
// LabelResult* values are used on metrics with label key "result" (RPC outcomes).
|
||||
// LabelStatus* values are used on metrics with label key "status" (job outcomes).
|
||||
// LabelState* values are used on metrics with label key "state" (runner state).
|
||||
const (
|
||||
LabelResultTask = "task"
|
||||
LabelResultEmpty = "empty"
|
||||
@@ -35,6 +38,11 @@ const (
|
||||
LabelStatusCancelled = "cancelled"
|
||||
LabelStatusSkipped = "skipped"
|
||||
LabelStatusUnknown = "unknown"
|
||||
LabelStatusTimeout = "timeout"
|
||||
|
||||
LabelStateBusy = "busy"
|
||||
LabelStateIdle = "idle"
|
||||
LabelStateUnavailable = "unavailable"
|
||||
)
|
||||
|
||||
// rpcDurationBuckets covers the expected latency range for short-running
|
||||
@@ -57,6 +65,15 @@ func ResultToStatusLabel(r runnerv1.Result) string {
|
||||
}
|
||||
}
|
||||
|
||||
// JobStatus reports an unsuccessful job past its deadline as timeout, like the job's trace span.
|
||||
func JobStatus(ctx context.Context, r runnerv1.Result) string {
|
||||
status := ResultToStatusLabel(r)
|
||||
if status != LabelStatusSuccess && status != LabelStatusSkipped && errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return LabelStatusTimeout
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
var (
|
||||
RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
@@ -96,7 +113,7 @@ var (
|
||||
Namespace: Namespace,
|
||||
Subsystem: "job",
|
||||
Name: "total",
|
||||
Help: "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
|
||||
Help: "Total jobs processed by status (success, failure, cancelled, skipped, timeout, unknown).",
|
||||
}, []string{"status"})
|
||||
|
||||
JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
@@ -148,8 +165,8 @@ var (
|
||||
Namespace: Namespace,
|
||||
Subsystem: "client",
|
||||
Name: "errors_total",
|
||||
Help: "Total client RPC errors by method.",
|
||||
}, []string{"method"})
|
||||
Help: "Total client RPC errors by method and connect code.",
|
||||
}, []string{"method", "code"})
|
||||
)
|
||||
|
||||
// Registry is the custom Prometheus registry used by the runner.
|
||||
@@ -176,7 +193,7 @@ func Init() {
|
||||
|
||||
// RegisterUptimeFunc registers a GaugeFunc that reports seconds since startTime.
|
||||
func RegisterUptimeFunc(startTime time.Time) {
|
||||
Registry.MustRegister(prometheus.NewGaugeFunc(
|
||||
register(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Name: "uptime_seconds",
|
||||
@@ -190,7 +207,7 @@ func RegisterUptimeFunc(startTime time.Time) {
|
||||
// capacity utilisation ratio, evaluated lazily at Prometheus scrape time.
|
||||
func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
|
||||
capF := float64(capacity)
|
||||
Registry.MustRegister(prometheus.NewGaugeFunc(
|
||||
register(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: "job",
|
||||
@@ -199,7 +216,7 @@ func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
|
||||
},
|
||||
func() float64 { return float64(countFn()) },
|
||||
))
|
||||
Registry.MustRegister(prometheus.NewGaugeFunc(
|
||||
register(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: "job",
|
||||
@@ -214,3 +231,34 @@ func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
// register replaces an earlier registration, as the daemon can run more than once per process.
|
||||
func register(collector prometheus.Collector) {
|
||||
Registry.Unregister(collector)
|
||||
Registry.MustRegister(collector)
|
||||
}
|
||||
|
||||
// RegisterStateFunc registers gitea_runner_state, 1 for the state stateFn reports and 0 for the others.
|
||||
func RegisterStateFunc(stateFn func() string) {
|
||||
register(stateCollector{prometheus.NewDesc(prometheus.BuildFQName(Namespace, "", "state"),
|
||||
"Runner state (busy, idle, unavailable). 1 for the current state.", []string{"state"}, nil), stateFn})
|
||||
}
|
||||
|
||||
// stateCollector reads the state once per collection, so a scrape never reports two states.
|
||||
type stateCollector struct {
|
||||
desc *prometheus.Desc
|
||||
state func() string
|
||||
}
|
||||
|
||||
func (c stateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
|
||||
|
||||
func (c stateCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
current := c.state()
|
||||
for _, state := range []string{LabelStateBusy, LabelStateIdle, LabelStateUnavailable} {
|
||||
value := 0.0
|
||||
if state == current {
|
||||
value = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, value, state)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,11 @@ func TestResultToStatusLabel(t *testing.T) {
|
||||
require.Equal(t, tt.want, ResultToStatusLabel(tt.result))
|
||||
})
|
||||
}
|
||||
expired, cancel := context.WithDeadline(t.Context(), time.Time{})
|
||||
defer cancel()
|
||||
require.Equal(t, LabelStatusTimeout, JobStatus(expired, runnerv1.Result_RESULT_FAILURE))
|
||||
require.Equal(t, LabelStatusSuccess, JobStatus(expired, runnerv1.Result_RESULT_SUCCESS))
|
||||
require.Equal(t, LabelStatusCancelled, JobStatus(t.Context(), runnerv1.Result_RESULT_CANCELLED))
|
||||
}
|
||||
|
||||
func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
@@ -50,6 +55,8 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
RunnerInfo.WithLabelValues("test", "runner").Set(1)
|
||||
RegisterUptimeFunc(time.Now().Add(-time.Second))
|
||||
RegisterRunningJobsFunc(func() int64 { return 2 }, 4)
|
||||
RegisterStateFunc(func() string { return LabelStateIdle })
|
||||
RegisterStateFunc(func() string { return LabelStateBusy })
|
||||
|
||||
metrics, err := Registry.Gather()
|
||||
require.NoError(t, err)
|
||||
@@ -58,6 +65,15 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_uptime_seconds"))
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_job_running"))
|
||||
require.True(t, hasMetric(metrics, "gitea_runner_job_capacity_utilization_ratio"))
|
||||
states := map[string]float64{}
|
||||
for _, mf := range metrics {
|
||||
if mf.GetName() == "gitea_runner_state" {
|
||||
for _, m := range mf.GetMetric() {
|
||||
states[m.GetLabel()[0].GetValue()] = m.GetGauge().GetValue()
|
||||
}
|
||||
}
|
||||
}
|
||||
require.Equal(t, map[string]float64{LabelStateBusy: 1, LabelStateIdle: 0, LabelStateUnavailable: 0}, states)
|
||||
}
|
||||
|
||||
func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {
|
||||
|
||||
@@ -36,9 +36,8 @@ func processAlive(pid int) bool {
|
||||
if b, readErr := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); readErr == nil {
|
||||
// Format: "pid (comm) state ..." — state follows the closing ')' of the
|
||||
// command name (which may itself contain spaces and parens).
|
||||
rest := string(b)
|
||||
if idx := strings.LastIndex(rest, ") "); idx >= 0 {
|
||||
fields := strings.Fields(rest[idx+2:])
|
||||
if _, after, found := strings.CutLast(string(b), ") "); found {
|
||||
fields := strings.Fields(after)
|
||||
if len(fields) > 0 && fields[0] == "Z" {
|
||||
return false // zombie: terminated but not yet reaped
|
||||
}
|
||||
|
||||
@@ -624,7 +624,7 @@ func (r *Reporter) reportLog(noMore bool) (bool, error) {
|
||||
metrics.ReportLogDuration.Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog, connect.CodeOf(err).String()).Inc()
|
||||
return false, err
|
||||
}
|
||||
metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultSuccess).Inc()
|
||||
@@ -701,7 +701,7 @@ func (r *Reporter) ReportState(reportResult bool) error {
|
||||
metrics.ReportStateDuration.Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
metrics.ReportStateTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask).Inc()
|
||||
metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask, connect.CodeOf(err).String()).Inc()
|
||||
r.stateMu.Lock()
|
||||
r.stateChanged = true
|
||||
r.stateMu.Unlock()
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/metrics"
|
||||
|
||||
"gitea.dev/actionslib/runner/v1/runnerv1connect"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
commonpb "go.opentelemetry.io/proto/otlp/common/v1"
|
||||
metricpb "go.opentelemetry.io/proto/otlp/metrics/v1"
|
||||
resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
const cumulative = metricpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE
|
||||
|
||||
// pushMetrics also pushes on shutdown so short-lived runners report.
|
||||
func pushMetrics(exporter *client, res *resource.Resource, gatherer prometheus.Gatherer) func(context.Context) error {
|
||||
attrs := make([]*commonpb.KeyValue, 0, res.Len())
|
||||
for _, kv := range res.Attributes() {
|
||||
attr := keyValue(string(kv.Key), kv.Value.String())
|
||||
if kv.Value.Type() == attribute.INT64 {
|
||||
attr.Value.Value = &commonpb.AnyValue_IntValue{IntValue: kv.Value.AsInt64()}
|
||||
}
|
||||
attrs = append(attrs, attr)
|
||||
}
|
||||
interval, timeout := millis("OTEL_METRIC_EXPORT_INTERVAL", time.Minute, time.Minute), millis("OTEL_METRIC_EXPORT_TIMEOUT", 30*time.Second, noLimit)
|
||||
push := func(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
families, err := gatherer.Gather()
|
||||
return errors.Join(err, exporter.upload(ctx, &metricpb.MetricsData{ResourceMetrics: []*metricpb.ResourceMetrics{{
|
||||
Resource: &resourcepb.Resource{Attributes: attrs},
|
||||
ScopeMetrics: []*metricpb.ScopeMetrics{{Scope: &commonpb.InstrumentationScope{Name: scope}, Metrics: convert(families)}},
|
||||
SchemaUrl: res.SchemaURL(),
|
||||
}}}))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := push(ctx); err != nil && ctx.Err() == nil {
|
||||
log.WithError(err).Warn("OTLP metrics export failed, set OTEL_METRICS_EXPORTER=none to disable it")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func(ctx context.Context) error {
|
||||
cancel()
|
||||
<-done
|
||||
return push(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
var ucum = map[string]string{
|
||||
"days": "d", "hours": "h", "minutes": "min", "seconds": "s", "milliseconds": "ms", "microseconds": "us", "nanoseconds": "ns",
|
||||
"bytes": "By", "kibibytes": "KiBy", "mebibytes": "MiBy", "gibibytes": "GiBy", "tebibytes": "TiBy",
|
||||
"kilobytes": "kBy", "megabytes": "MBy", "gigabytes": "GBy", "terabytes": "TBy",
|
||||
"meters": "m", "volts": "V", "amperes": "A", "joules": "J", "watts": "W", "grams": "g", "celsius": "Cel", "hertz": "Hz", "percent": "%",
|
||||
}
|
||||
|
||||
type otlpMetric struct {
|
||||
name, unit string
|
||||
upDown bool // a gauge semconv defines as an UpDownCounter
|
||||
labels map[string]string // label → attribute
|
||||
values map[string]string // label value → attribute value
|
||||
extra []*commonpb.KeyValue
|
||||
unlimited float64 // gauge value meaning no limit, not sent
|
||||
}
|
||||
|
||||
var rpcMethods = map[string]string{
|
||||
metrics.LabelMethodFetchTask: strings.TrimPrefix(runnerv1connect.RunnerServiceFetchTaskProcedure, "/"),
|
||||
metrics.LabelMethodUpdateLog: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateLogProcedure, "/"),
|
||||
metrics.LabelMethodUpdateTask: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateTaskProcedure, "/"),
|
||||
}
|
||||
|
||||
// openFiles maps process_open_fds, which client_golang fills with the handle count on Windows.
|
||||
func openFiles() otlpMetric {
|
||||
if runtime.GOOS == "windows" {
|
||||
return otlpMetric{name: "process.windows.handle.count", unit: "{handle}", upDown: true}
|
||||
}
|
||||
return otlpMetric{name: "process.unix.file_descriptor.count", unit: "{file_descriptor}", upDown: true}
|
||||
}
|
||||
|
||||
// otlpMetrics maps to semconv names, unlisted metrics keep theirs and an empty name drops one the resource already carries.
|
||||
var otlpMetrics = map[string]otlpMetric{
|
||||
"gitea_runner_info": {},
|
||||
"go_info": {},
|
||||
"process_start_time_seconds": {},
|
||||
"gitea_runner_capacity": {name: "gitea.runner.job.limit", unit: "{job}", upDown: true},
|
||||
"gitea_runner_uptime_seconds": {name: "gitea.runner.uptime", unit: "s"},
|
||||
"gitea_runner_job_running": {name: "gitea.runner.job.active", unit: "{job}", upDown: true},
|
||||
"gitea_runner_job_capacity_utilization_ratio": {name: "gitea.runner.job.utilization", unit: "1"},
|
||||
"gitea_runner_job_duration_seconds": {name: "gitea.runner.job.duration", unit: "s"},
|
||||
"gitea_runner_job_total": {
|
||||
name: "gitea.runner.jobs", unit: "{job}",
|
||||
labels: map[string]string{"status": "cicd.pipeline.result"},
|
||||
values: map[string]string{metrics.LabelStatusCancelled: "cancellation", metrics.LabelStatusSkipped: "skip", metrics.LabelStatusUnknown: "error"},
|
||||
},
|
||||
"gitea_runner_poll_fetch_total": {
|
||||
name: "gitea.runner.poll.fetches", unit: "{fetch}",
|
||||
labels: map[string]string{"result": "gitea.runner.poll.result"},
|
||||
},
|
||||
"gitea_runner_poll_fetch_duration_seconds": {name: "gitea.runner.poll.fetch.duration", unit: "s"},
|
||||
"gitea_runner_poll_backoff_seconds": {name: "gitea.runner.poll.backoff", unit: "s"},
|
||||
"gitea_runner_report_log_duration_seconds": {name: "gitea.runner.report.log.duration", unit: "s"},
|
||||
"gitea_runner_report_state_duration_seconds": {name: "gitea.runner.report.state.duration", unit: "s"},
|
||||
"gitea_runner_report_log_buffer_rows": {name: "gitea.runner.report.log.buffer", unit: "{row}", upDown: true},
|
||||
"gitea_runner_report_log_total": {
|
||||
name: "gitea.runner.report.log.requests", unit: "{request}",
|
||||
labels: map[string]string{"result": "gitea.runner.report.result"},
|
||||
},
|
||||
"gitea_runner_report_state_total": {
|
||||
name: "gitea.runner.report.state.requests", unit: "{request}",
|
||||
labels: map[string]string{"result": "gitea.runner.report.result"},
|
||||
},
|
||||
"gitea_runner_client_errors_total": {
|
||||
name: "cicd.system.errors", unit: "{error}",
|
||||
labels: map[string]string{"method": "rpc.method", "code": "error.type"}, values: rpcMethods,
|
||||
extra: []*commonpb.KeyValue{keyValue("cicd.system.component", "runner")},
|
||||
},
|
||||
"gitea_runner_state": {
|
||||
name: "cicd.worker.count", unit: "{worker}", upDown: true,
|
||||
labels: map[string]string{"state": "cicd.worker.state"},
|
||||
values: map[string]string{metrics.LabelStateIdle: "available", metrics.LabelStateUnavailable: "offline"},
|
||||
},
|
||||
"go_goroutines": {name: "go.goroutine.count", unit: "{goroutine}", upDown: true},
|
||||
"go_memstats_alloc_bytes_total": {name: "go.memory.allocated", unit: "By"},
|
||||
"go_memstats_next_gc_bytes": {name: "go.memory.gc.goal", unit: "By", upDown: true},
|
||||
"go_gc_gomemlimit_bytes": {name: "go.memory.limit", unit: "By", upDown: true, unlimited: math.MaxInt64},
|
||||
"go_gc_gogc_percent": {name: "go.config.gogc", unit: "%", upDown: true},
|
||||
"go_sched_gomaxprocs_threads": {name: "go.processor.limit", unit: "{thread}", upDown: true},
|
||||
"process_resident_memory_bytes": {name: "process.memory.usage", unit: "By", upDown: true},
|
||||
"process_virtual_memory_bytes": {name: "process.memory.virtual", unit: "By", upDown: true},
|
||||
"process_open_fds": openFiles(),
|
||||
}
|
||||
|
||||
// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp except for otlpMetrics.
|
||||
func convert(families []*dto.MetricFamily) []*metricpb.Metric {
|
||||
now := uint64(time.Now().UnixNano())
|
||||
metrics := make([]*metricpb.Metric, 0, len(families))
|
||||
byFamily := make(map[string]*dto.MetricFamily, len(families))
|
||||
for _, family := range families {
|
||||
byFamily[family.GetName()] = family
|
||||
mapping, ok := otlpMetrics[family.GetName()]
|
||||
if !ok {
|
||||
mapping.name = family.GetName()
|
||||
} else if mapping.name == "" {
|
||||
continue
|
||||
}
|
||||
metric := &metricpb.Metric{
|
||||
Name: mapping.name, Description: family.GetHelp(), Unit: cmp.Or(mapping.unit, ucum[family.GetUnit()], family.GetUnit()),
|
||||
Metadata: []*commonpb.KeyValue{keyValue("prometheus.type", strings.ToLower(family.GetType().String()))},
|
||||
}
|
||||
switch family.GetType() {
|
||||
case dto.MetricType_COUNTER:
|
||||
sum := &metricpb.Sum{AggregationTemporality: cumulative, IsMonotonic: true}
|
||||
for _, series := range family.GetMetric() {
|
||||
sum.DataPoints = append(sum.DataPoints, number(labels(series, mapping), unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Sum{Sum: sum}
|
||||
case dto.MetricType_GAUGE:
|
||||
var points []*metricpb.NumberDataPoint
|
||||
for _, series := range family.GetMetric() {
|
||||
if value := series.GetGauge().GetValue(); mapping.unlimited == 0 || value != mapping.unlimited {
|
||||
points = append(points, number(labels(series, mapping), 0, now, value))
|
||||
}
|
||||
}
|
||||
if len(points) == 0 {
|
||||
continue
|
||||
}
|
||||
if mapping.upDown {
|
||||
metric.Data = &metricpb.Metric_Sum{Sum: &metricpb.Sum{AggregationTemporality: cumulative, DataPoints: points}}
|
||||
} else {
|
||||
metric.Data = &metricpb.Metric_Gauge{Gauge: &metricpb.Gauge{DataPoints: points}}
|
||||
}
|
||||
case dto.MetricType_HISTOGRAM:
|
||||
histogram := &metricpb.Histogram{AggregationTemporality: cumulative}
|
||||
for _, series := range family.GetMetric() {
|
||||
histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, labels(series, mapping), now))
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Histogram{Histogram: histogram}
|
||||
case dto.MetricType_SUMMARY:
|
||||
summary := &metricpb.Summary{}
|
||||
for _, series := range family.GetMetric() {
|
||||
point := &metricpb.SummaryDataPoint{
|
||||
Attributes: labels(series, mapping), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Count: series.GetSummary().GetSampleCount(), Sum: series.GetSummary().GetSampleSum(),
|
||||
}
|
||||
for _, quantile := range series.GetSummary().GetQuantile() {
|
||||
point.QuantileValues = append(point.QuantileValues, &metricpb.SummaryDataPoint_ValueAtQuantile{Quantile: quantile.GetQuantile(), Value: quantile.GetValue()})
|
||||
}
|
||||
summary.DataPoints = append(summary.DataPoints, point)
|
||||
}
|
||||
metric.Data = &metricpb.Metric_Summary{Summary: summary}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
return append(metrics, derived(byFamily, now)...)
|
||||
}
|
||||
|
||||
// derived adds the semconv Go metrics that are computed from other metrics rather than renamed.
|
||||
func derived(families map[string]*dto.MetricFamily, now uint64) []*metricpb.Metric {
|
||||
first := func(name string) *dto.Metric {
|
||||
if series := families[name].GetMetric(); len(series) > 0 {
|
||||
return series[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
gauge := func(name string) (float64, bool) {
|
||||
series := first(name)
|
||||
return series.GetGauge().GetValue(), series != nil
|
||||
}
|
||||
sum := func(name, unit, description string, monotonic bool, points ...*metricpb.NumberDataPoint) *metricpb.Metric {
|
||||
return &metricpb.Metric{Name: name, Unit: unit, Description: description, Data: &metricpb.Metric_Sum{Sum: &metricpb.Sum{
|
||||
AggregationTemporality: cumulative, IsMonotonic: monotonic, DataPoints: points,
|
||||
}}}
|
||||
}
|
||||
var metrics []*metricpb.Metric
|
||||
total, ok1 := gauge("go_memstats_sys_bytes")
|
||||
released, ok2 := gauge("go_memstats_heap_released_bytes")
|
||||
stack, ok3 := gauge("go_memstats_stack_inuse_bytes")
|
||||
if ok1 && ok2 && ok3 {
|
||||
metrics = append(metrics, sum("go.memory.used", "By", "Memory used by the Go runtime.", false,
|
||||
number([]*commonpb.KeyValue{keyValue("go.memory.type", "stack")}, 0, now, stack),
|
||||
number([]*commonpb.KeyValue{keyValue("go.memory.type", "other")}, 0, now, total-released-stack)))
|
||||
}
|
||||
if gc := first("go_gc_duration_seconds"); gc != nil {
|
||||
metrics = append(metrics, sum("go.memory.gc.cycles", "{gc_cycle}", "Number of completed GC cycles.", true,
|
||||
number(nil, unixNano(gc.GetSummary().GetCreatedTimestamp()), now, float64(gc.GetSummary().GetSampleCount()))))
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func number(attrs []*commonpb.KeyValue, start, now uint64, value float64) *metricpb.NumberDataPoint {
|
||||
return &metricpb.NumberDataPoint{Attributes: attrs, StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
|
||||
}
|
||||
|
||||
// unixNano maps a nil timestamp to 0, leaving the start time unset as the spec asks.
|
||||
func unixNano(timestamp *timestamppb.Timestamp) uint64 {
|
||||
return uint64(timestamp.AsTime().UnixNano())
|
||||
}
|
||||
|
||||
func histogramPoint(series *dto.Metric, attrs []*commonpb.KeyValue, now uint64) *metricpb.HistogramDataPoint {
|
||||
histogram := series.GetHistogram()
|
||||
point := &metricpb.HistogramDataPoint{
|
||||
Attributes: attrs, StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
|
||||
Count: histogram.GetSampleCount(), Sum: histogram.SampleSum,
|
||||
BucketCounts: make([]uint64, len(histogram.GetBucket())+1),
|
||||
}
|
||||
var below uint64
|
||||
for i, bucket := range histogram.GetBucket() {
|
||||
point.ExplicitBounds = append(point.ExplicitBounds, bucket.GetUpperBound())
|
||||
point.BucketCounts[i], below = bucket.GetCumulativeCount()-below, bucket.GetCumulativeCount()
|
||||
}
|
||||
point.BucketCounts[len(histogram.GetBucket())] = histogram.GetSampleCount() - below
|
||||
return point
|
||||
}
|
||||
|
||||
func labels(series *dto.Metric, mapping otlpMetric) []*commonpb.KeyValue {
|
||||
attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel())+len(mapping.extra))
|
||||
for _, label := range series.GetLabel() {
|
||||
attrs = append(attrs, keyValue(cmp.Or(mapping.labels[label.GetName()], label.GetName()), cmp.Or(mapping.values[label.GetValue()], label.GetValue())))
|
||||
}
|
||||
return append(attrs, mapping.extra...)
|
||||
}
|
||||
|
||||
func keyValue(key, value string) *commonpb.KeyValue {
|
||||
return &commonpb.KeyValue{Key: key, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: value}}}
|
||||
}
|
||||
@@ -6,22 +6,29 @@ package telemetry
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/ver"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
"github.com/avast/retry-go/v5"
|
||||
"github.com/docker/go-connections/tlsconfig"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
||||
@@ -32,11 +39,14 @@ import (
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
tracenoop "go.opentelemetry.io/otel/trace/noop"
|
||||
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const scope = "gitea.com/gitea/runner"
|
||||
|
||||
var processCreation = time.Now() // package init, within milliseconds of the process start
|
||||
|
||||
type taskRunKey struct{}
|
||||
|
||||
type taskRun struct {
|
||||
@@ -51,48 +61,144 @@ func SetTracerProvider(provider trace.TracerProvider) {
|
||||
tracer = provider.Tracer(scope)
|
||||
}
|
||||
|
||||
func Setup(ctx context.Context, uuid, name string) (func(context.Context) error, error) {
|
||||
func Setup(ctx context.Context, uuid, name string, gatherer prometheus.Gatherer) (func(context.Context) error, error) {
|
||||
noop := func(context.Context) error { return nil }
|
||||
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
if base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint == "" && base != "" {
|
||||
endpoint = strings.TrimSuffix(base, "/") + "/v1/traces"
|
||||
}
|
||||
if endpoint == "" || strings.EqualFold(os.Getenv("OTEL_SDK_DISABLED"), "true") || strings.EqualFold(os.Getenv("OTEL_TRACES_EXPORTER"), "none") {
|
||||
if strings.EqualFold(os.Getenv("OTEL_SDK_DISABLED"), "true") {
|
||||
return noop, nil
|
||||
}
|
||||
if protocol := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"), os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL")); protocol != "" && !strings.EqualFold(protocol, "http/protobuf") {
|
||||
return noop, fmt.Errorf("OTLP protocol %q is not supported, only http/protobuf is", protocol)
|
||||
clients := map[string]*client{}
|
||||
for _, signal := range []string{"TRACES", "METRICS"} {
|
||||
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_" + signal + "_ENDPOINT")
|
||||
if base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint == "" && base != "" {
|
||||
endpoint = strings.TrimSuffix(base, "/") + "/v1/" + strings.ToLower(signal)
|
||||
}
|
||||
if endpoint == "" || !exportsOTLP(signal) {
|
||||
continue
|
||||
}
|
||||
if protocol := os.Getenv(otlpKey(signal, "PROTOCOL")); protocol != "" && !strings.EqualFold(protocol, "http/protobuf") {
|
||||
return noop, fmt.Errorf("OTLP protocol %q is not supported, only http/protobuf is", protocol)
|
||||
}
|
||||
headers := http.Header{}
|
||||
for _, key := range []string{"OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_" + signal + "_HEADERS"} {
|
||||
for pair := range strings.SplitSeq(os.Getenv(key), ",") {
|
||||
key, value, found := strings.Cut(pair, "=")
|
||||
if decoded, err := url.PathUnescape(strings.TrimSpace(value)); found && err == nil {
|
||||
headers.Set(strings.TrimSpace(key), decoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
compressionKey := otlpKey(signal, "COMPRESSION")
|
||||
compression := os.Getenv(compressionKey)
|
||||
if !slices.Contains([]string{"", "none", "gzip"}, strings.ToLower(compression)) {
|
||||
log.Warnf("ignoring unsupported %s=%q", compressionKey, compression)
|
||||
}
|
||||
httpClient, err := newHTTPClient(signal)
|
||||
if err != nil {
|
||||
return noop, err
|
||||
}
|
||||
clients[signal] = &client{
|
||||
endpoint: endpoint, headers: headers, timeout: millis(otlpKey(signal, "TIMEOUT"), 10*time.Second, noLimit),
|
||||
gzip: strings.EqualFold(compression, "gzip"), httpClient: httpClient,
|
||||
}
|
||||
}
|
||||
res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
|
||||
if len(clients) == 0 {
|
||||
return noop, nil
|
||||
}
|
||||
res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithProcessPID(), resource.WithProcessRuntimeName(), resource.WithProcessRuntimeVersion(), resource.WithAttributes(
|
||||
semconv.ServiceName("gitea-runner"),
|
||||
semconv.ServiceVersion(ver.Version()),
|
||||
semconv.ServiceInstanceID(uuid),
|
||||
semconv.CICDWorkerID(uuid),
|
||||
semconv.CICDWorkerName(name),
|
||||
semconv.ProcessCreationTime(processCreation.UTC().Format(time.RFC3339Nano)),
|
||||
), resource.WithFromEnv())
|
||||
if err != nil {
|
||||
return noop, err
|
||||
}
|
||||
headers := http.Header{}
|
||||
for _, key := range []string{"OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"} {
|
||||
for pair := range strings.SplitSeq(os.Getenv(key), ",") {
|
||||
key, value, found := strings.Cut(pair, "=")
|
||||
if decoded, err := url.PathUnescape(strings.TrimSpace(value)); found && err == nil {
|
||||
headers.Set(strings.TrimSpace(key), decoded)
|
||||
}
|
||||
}
|
||||
var shutdown []func(context.Context) error
|
||||
if c := clients["TRACES"]; c != nil {
|
||||
provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(otlptrace.NewUnstarted(c)), sdktrace.WithResource(res))
|
||||
SetTracerProvider(provider)
|
||||
shutdown = append(shutdown, provider.Shutdown)
|
||||
}
|
||||
provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(otlptrace.NewUnstarted(&client{endpoint, headers})), sdktrace.WithResource(res))
|
||||
SetTracerProvider(provider)
|
||||
return provider.Shutdown, nil
|
||||
if c := clients["METRICS"]; c != nil {
|
||||
shutdown = append(shutdown, pushMetrics(c, res, gatherer))
|
||||
}
|
||||
return func(ctx context.Context) error {
|
||||
var group errgroup.Group
|
||||
for _, fn := range shutdown {
|
||||
group.Go(func() error { return fn(ctx) })
|
||||
}
|
||||
return group.Wait()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// client replaces otlptracehttp, which links gRPC (+7 MB): https://github.com/open-telemetry/opentelemetry-go/issues/2579
|
||||
type client struct {
|
||||
endpoint string
|
||||
headers http.Header
|
||||
func exportsOTLP(signal string) bool {
|
||||
key := "OTEL_" + signal + "_EXPORTER"
|
||||
otlp, known := false, false
|
||||
for value := range strings.SplitSeq(os.Getenv(key), ",") {
|
||||
switch value = strings.TrimSpace(value); strings.ToLower(value) {
|
||||
case "":
|
||||
case "otlp":
|
||||
otlp, known = true, true
|
||||
case "none":
|
||||
known = true
|
||||
default:
|
||||
log.Warnf("ignoring unsupported %s value %q", key, value)
|
||||
}
|
||||
}
|
||||
return otlp || !known
|
||||
}
|
||||
|
||||
func otlpKey(signal, name string) string {
|
||||
if key := "OTEL_EXPORTER_OTLP_" + signal + "_" + name; os.Getenv(key) != "" {
|
||||
return key
|
||||
}
|
||||
return "OTEL_EXPORTER_OTLP_" + name
|
||||
}
|
||||
|
||||
const noLimit = math.MaxInt32 * time.Millisecond // the spec's stand-in for a 0 timeout
|
||||
|
||||
func millis(key string, fallback, zero time.Duration) time.Duration {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
ms, err := strconv.ParseInt(value, 10, 32)
|
||||
if err != nil || ms < 0 {
|
||||
log.Warnf("ignoring %s=%q, it must be a non-negative number of milliseconds", key, value)
|
||||
return fallback
|
||||
}
|
||||
return cmp.Or(time.Duration(ms)*time.Millisecond, zero)
|
||||
}
|
||||
|
||||
// newHTTPClient applies the OTLP certificate variables, a configured CA replaces the system roots like in the Go SDK.
|
||||
func newHTTPClient(signal string) (*http.Client, error) {
|
||||
caFile, certFile, keyFile := os.Getenv(otlpKey(signal, "CERTIFICATE")), os.Getenv(otlpKey(signal, "CLIENT_CERTIFICATE")), os.Getenv(otlpKey(signal, "CLIENT_KEY"))
|
||||
if caFile == "" && certFile == "" && keyFile == "" {
|
||||
return exportClient, nil
|
||||
}
|
||||
config, err := tlsconfig.Client(tlsconfig.Options{CAFile: caFile, CertFile: certFile, KeyFile: keyFile, ExclusiveRootPools: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||
transport = transport.Clone()
|
||||
transport.TLSClientConfig = config
|
||||
return &http.Client{Transport: transport, CheckRedirect: exportClient.CheckRedirect}, nil
|
||||
}
|
||||
|
||||
// client replaces the otlp*http exporters, which link gRPC (+7 MB): https://github.com/open-telemetry/opentelemetry-go/issues/2579
|
||||
type client struct {
|
||||
endpoint string
|
||||
headers http.Header
|
||||
timeout time.Duration
|
||||
gzip bool
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
var gzipWriters = sync.Pool{New: func() any { return gzip.NewWriter(io.Discard) }}
|
||||
|
||||
// exportClient refuses redirects, which would carry the collector's header credentials to another host.
|
||||
var exportClient = &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
|
||||
@@ -101,18 +207,36 @@ func (*client) Start(context.Context) error { return nil }
|
||||
func (*client) Stop(context.Context) error { return nil }
|
||||
|
||||
func (c *client) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
|
||||
body, err := proto.Marshal(&tracepb.TracesData{ResourceSpans: spans})
|
||||
return c.upload(ctx, &tracepb.TracesData{ResourceSpans: spans})
|
||||
}
|
||||
|
||||
func (c *client) upload(ctx context.Context, data proto.Message) error {
|
||||
body, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return retry.New(retry.Context(ctx), retry.Attempts(5), retry.LastErrorOnly(true)).Do(func() error {
|
||||
if c.gzip {
|
||||
var buffer bytes.Buffer
|
||||
writer, _ := gzipWriters.Get().(*gzip.Writer)
|
||||
writer.Reset(&buffer)
|
||||
_, _ = writer.Write(body)
|
||||
_ = writer.Close()
|
||||
gzipWriters.Put(writer)
|
||||
body = buffer.Bytes()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return retry.New(retry.Context(ctx), retry.Attempts(5), retry.LastErrorOnly(true), retry.DelayType(retryDelay)).Do(func() error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return retry.Unrecoverable(err)
|
||||
}
|
||||
req.Header = c.headers.Clone()
|
||||
req.Header.Set("Content-Type", "application/x-protobuf")
|
||||
resp, err := exportClient.Do(req)
|
||||
if c.gzip {
|
||||
req.Header.Set("Content-Encoding", "gzip")
|
||||
}
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -121,7 +245,7 @@ func (c *client) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpan
|
||||
if resp.StatusCode < http.StatusMultipleChoices {
|
||||
return nil
|
||||
}
|
||||
err = fmt.Errorf("OTLP export failed: %s", resp.Status)
|
||||
err = &statusError{resp.Status, resp.Header.Get("Retry-After")}
|
||||
if slices.Contains([]int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout}, resp.StatusCode) {
|
||||
return err
|
||||
}
|
||||
@@ -129,6 +253,19 @@ func (c *client) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpan
|
||||
})
|
||||
}
|
||||
|
||||
type statusError struct{ status, retryAfter string }
|
||||
|
||||
func (e *statusError) Error() string { return "OTLP export failed: " + e.status }
|
||||
|
||||
func retryDelay(attempt uint, err error, config retry.DelayContext) time.Duration {
|
||||
if status := (*statusError)(nil); errors.As(err, &status) {
|
||||
if seconds, err := strconv.ParseUint(status.retryAfter, 10, 31); err == nil {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
}
|
||||
return retry.CombineDelay(retry.BackOffDelay, retry.RandomDelay)(attempt, err, config)
|
||||
}
|
||||
|
||||
func StartJob(ctx context.Context, task *runnerv1.Task) (context.Context, func(runnerv1.Result)) {
|
||||
field := func(key string) string { return task.GetContext().GetFields()[key].GetStringValue() }
|
||||
pipeline, owner, repo := field("workflow"), field("repository_owner"), field("repository")
|
||||
|
||||
@@ -4,16 +4,29 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -23,6 +36,8 @@ import (
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
tracenoop "go.opentelemetry.io/otel/trace/noop"
|
||||
commonpb "go.opentelemetry.io/proto/otlp/common/v1"
|
||||
metricpb "go.opentelemetry.io/proto/otlp/metrics/v1"
|
||||
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
@@ -30,66 +45,199 @@ import (
|
||||
|
||||
func setEnv(t *testing.T, env map[string]string) {
|
||||
t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) })
|
||||
for _, key := range []string{
|
||||
"OTEL_SDK_DISABLED", "OTEL_TRACES_EXPORTER", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
|
||||
} {
|
||||
t.Setenv(key, env[key])
|
||||
for _, variable := range os.Environ() {
|
||||
if key, _, _ := strings.Cut(variable, "="); strings.HasPrefix(key, "OTEL_") {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
}
|
||||
for key, value := range env {
|
||||
t.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetup(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
||||
defer server.Close()
|
||||
for _, tc := range []struct {
|
||||
env map[string]string
|
||||
enabled bool
|
||||
err string
|
||||
env map[string]string
|
||||
traces, metrics bool
|
||||
err string
|
||||
}{
|
||||
{},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://collector:4318/v1/traces"}, enabled: true},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_TRACES_EXPORTER": "NONE"}},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://collector:4318/v1/traces"}, traces: true},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL, "OTEL_TRACES_EXPORTER": "NONE"}, metrics: true},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL, "OTEL_TRACES_EXPORTER": "zipkin, OTLP", "OTEL_METRICS_EXPORTER": "console"}, traces: true, metrics: true},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_TRACES_EXPORTER": "NONE", "OTEL_METRICS_EXPORTER": "none"}},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_SDK_DISABLED": "TRUE"}},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc"}, err: `OTLP protocol "grpc" is not supported, only http/protobuf is`},
|
||||
} {
|
||||
t.Run(fmt.Sprint(tc.env), func(t *testing.T) {
|
||||
setEnv(t, tc.env)
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner")
|
||||
gathered := false
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner", prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) {
|
||||
gathered = true
|
||||
return nil, nil
|
||||
}))
|
||||
if tc.err != "" {
|
||||
require.EqualError(t, err, tc.err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, disabled := tracer.(tracenoop.Tracer)
|
||||
assert.Equal(t, tc.enabled, !disabled)
|
||||
assert.Equal(t, tc.traces, !disabled)
|
||||
require.NoError(t, shutdown(t.Context()))
|
||||
assert.Equal(t, tc.metrics, gathered)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportRetriesAndMatchesOTLP(t *testing.T) {
|
||||
bodies := make(chan []byte, 2)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/otlp/v1/traces", r.URL.Path)
|
||||
traceBodies, metricBodies := make(chan []byte, 2), make(chan []byte, 1)
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Len(t, r.TLS.PeerCertificates, 1)
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
assert.Equal(t, "gzip", r.Header.Get("Content-Encoding"))
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
bodies <- body
|
||||
if len(bodies) == 1 {
|
||||
if r.URL.Path == "/otlp/v1/metrics" {
|
||||
select {
|
||||
case metricBodies <- body:
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
assert.Equal(t, "/otlp/v1/traces", r.URL.Path)
|
||||
traceBodies <- body
|
||||
if len(traceBodies) == 1 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
}))
|
||||
server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
setEnv(t, map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL + "/otlp/", "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20token"})
|
||||
certificate := server.TLS.Certificates[0]
|
||||
key, err := x509.MarshalPKCS8PrivateKey(certificate.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
certificatePath, keyPath := filepath.Join(t.TempDir(), "cert.pem"), filepath.Join(t.TempDir(), "key.pem")
|
||||
require.NoError(t, os.WriteFile(certificatePath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}), 0o600))
|
||||
setEnv(t, map[string]string{
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL + "/otlp/", "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20token",
|
||||
"OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", "OTEL_METRIC_EXPORT_INTERVAL": "1", "OTEL_EXPORTER_OTLP_CERTIFICATE": certificatePath,
|
||||
"OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE": certificatePath, "OTEL_EXPORTER_OTLP_CLIENT_KEY": keyPath,
|
||||
})
|
||||
registry := prometheus.NewRegistry()
|
||||
registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "running"}, func() float64 { return 2 }))
|
||||
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner")
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner", registry)
|
||||
require.NoError(t, err)
|
||||
_, span := tracer.Start(t.Context(), "op")
|
||||
span.End()
|
||||
gunzip := func(body []byte) []byte {
|
||||
reader, err := gzip.NewReader(bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
decoded, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
return decoded
|
||||
}
|
||||
var metricsData metricpb.MetricsData
|
||||
select {
|
||||
case body := <-metricBodies:
|
||||
require.NoError(t, proto.Unmarshal(gunzip(body), &metricsData))
|
||||
case <-time.After(30 * time.Second):
|
||||
require.FailNow(t, "no periodic metrics export")
|
||||
}
|
||||
require.NoError(t, shutdown(t.Context()))
|
||||
|
||||
require.Len(t, bodies, 2)
|
||||
<-bodies
|
||||
var decoded tracepb.TracesData
|
||||
require.NoError(t, proto.Unmarshal(<-bodies, &decoded))
|
||||
assert.Equal(t, "op", decoded.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
|
||||
require.Len(t, traceBodies, 2)
|
||||
<-traceBodies
|
||||
var tracesData tracepb.TracesData
|
||||
require.NoError(t, proto.Unmarshal(gunzip(<-traceBodies), &tracesData))
|
||||
assert.Equal(t, 2*time.Second, retryDelay(1, &statusError{"429 Too Many Requests", "2"}, nil))
|
||||
assert.Equal(t, "op", tracesData.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
|
||||
|
||||
resourceMetrics := metricsData.GetResourceMetrics()[0]
|
||||
pid := &commonpb.KeyValue{Key: "process.pid", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: int64(os.Getpid())}}}
|
||||
for _, want := range []*commonpb.KeyValue{keyValue("service.name", "gitea-runner"), keyValue("process.creation.time", processCreation.UTC().Format(time.RFC3339Nano)), pid} {
|
||||
assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool { return proto.Equal(kv, want) }), want.GetKey())
|
||||
}
|
||||
assert.InDelta(t, 2, resourceMetrics.GetScopeMetrics()[0].GetMetrics()[0].GetGauge().GetDataPoints()[0].GetAsDouble(), 0)
|
||||
}
|
||||
|
||||
func TestConvert(t *testing.T) {
|
||||
registry := prometheus.NewRegistry()
|
||||
counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "jobs_total", Help: "Jobs."}, []string{"status"})
|
||||
histogram := prometheus.NewHistogram(prometheus.HistogramOpts{Name: "duration_seconds", Unit: "seconds", Buckets: []float64{1, 2}})
|
||||
summary := prometheus.NewSummary(prometheus.SummaryOpts{Name: "gc_seconds", Objectives: map[float64]float64{0.5: 0.05}})
|
||||
jobs := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_job_total"}, []string{"status"})
|
||||
running := prometheus.NewGauge(prometheus.GaugeOpts{Name: "gitea_runner_job_running"})
|
||||
gauges := map[string]float64{
|
||||
"go_info": 1, "go_gc_gomemlimit_bytes": math.MaxInt64,
|
||||
"go_memstats_sys_bytes": 100, "go_memstats_heap_released_bytes": 30, "go_memstats_stack_inuse_bytes": 20,
|
||||
}
|
||||
for name, value := range gauges {
|
||||
registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: name}, func() float64 { return value }))
|
||||
}
|
||||
state := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "gitea_runner_state"}, []string{"state"})
|
||||
clientErrors := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_client_errors_total"}, []string{"method", "code"})
|
||||
registry.MustRegister(counter, histogram, summary, jobs, running, state, clientErrors)
|
||||
counter.WithLabelValues("success").Add(3)
|
||||
for _, value := range []float64{0.5, 1.5, 3} {
|
||||
histogram.Observe(value)
|
||||
}
|
||||
summary.Observe(4)
|
||||
jobs.WithLabelValues("cancelled").Inc()
|
||||
running.Set(2)
|
||||
state.WithLabelValues("idle").Set(1)
|
||||
state.WithLabelValues("unavailable").Set(0)
|
||||
clientErrors.WithLabelValues("FetchTask", "unavailable").Inc()
|
||||
families, err := registry.Gather()
|
||||
require.NoError(t, err)
|
||||
|
||||
exported := map[string]*metricpb.Metric{}
|
||||
for _, metric := range convert(families) {
|
||||
exported[metric.GetName()] = metric
|
||||
}
|
||||
assert.ElementsMatch(t, []string{
|
||||
"duration_seconds", "gc_seconds", "jobs_total", "gitea.runner.jobs", "gitea.runner.job.active", "cicd.system.errors",
|
||||
"go_memstats_sys_bytes", "go_memstats_heap_released_bytes", "go_memstats_stack_inuse_bytes", "go.memory.used", "cicd.worker.count",
|
||||
}, slices.Collect(maps.Keys(exported)))
|
||||
assert.Equal(t, "s", exported["duration_seconds"].GetUnit())
|
||||
buckets := exported["duration_seconds"].GetHistogram().GetDataPoints()[0]
|
||||
assert.Equal(t, []float64{1, 2}, buckets.GetExplicitBounds())
|
||||
assert.Equal(t, []uint64{1, 1, 1}, buckets.GetBucketCounts())
|
||||
assert.Equal(t, uint64(3), buckets.GetCount())
|
||||
assert.InDelta(t, 5, buckets.GetSum(), 0)
|
||||
assert.InDelta(t, 4, exported["gc_seconds"].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
|
||||
sum := exported["jobs_total"].GetSum()
|
||||
assert.Equal(t, "Jobs.", exported["jobs_total"].GetDescription())
|
||||
assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported["jobs_total"].GetMetadata()[0]))
|
||||
assert.NotZero(t, sum.GetDataPoints()[0].GetStartTimeUnixNano())
|
||||
assert.True(t, sum.GetIsMonotonic())
|
||||
assert.Equal(t, cumulative, sum.GetAggregationTemporality())
|
||||
assert.InDelta(t, 3, sum.GetDataPoints()[0].GetAsDouble(), 0)
|
||||
assert.True(t, proto.Equal(keyValue("status", "success"), sum.GetDataPoints()[0].GetAttributes()[0]))
|
||||
|
||||
assert.True(t, proto.Equal(keyValue("cicd.pipeline.result", "cancellation"), exported["gitea.runner.jobs"].GetSum().GetDataPoints()[0].GetAttributes()[0]))
|
||||
active := exported["gitea.runner.job.active"]
|
||||
assert.Equal(t, "{job}", active.GetUnit())
|
||||
assert.False(t, active.GetSum().GetIsMonotonic())
|
||||
assert.InDelta(t, 2, active.GetSum().GetDataPoints()[0].GetAsDouble(), 0)
|
||||
attributes := func(points []*metricpb.NumberDataPoint, key string) map[string]float64 {
|
||||
values := map[string]float64{}
|
||||
for _, point := range points {
|
||||
for _, kv := range point.GetAttributes() {
|
||||
if kv.GetKey() == key {
|
||||
values[kv.GetValue().GetStringValue()] = point.GetAsDouble()
|
||||
}
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
assert.Equal(t, map[string]float64{"stack": 20, "other": 50}, attributes(exported["go.memory.used"].GetSum().GetDataPoints(), "go.memory.type"))
|
||||
assert.Equal(t, map[string]float64{"available": 1, "offline": 0}, attributes(exported["cicd.worker.count"].GetSum().GetDataPoints(), "cicd.worker.state"))
|
||||
assert.True(t, proto.Equal(&commonpb.KeyValueList{Values: []*commonpb.KeyValue{
|
||||
keyValue("error.type", "unavailable"), keyValue("rpc.method", "runner.v1.RunnerService/FetchTask"), keyValue("cicd.system.component", "runner"),
|
||||
}}, &commonpb.KeyValueList{Values: exported["cicd.system.errors"].GetSum().GetDataPoints()[0].GetAttributes()}))
|
||||
}
|
||||
|
||||
func TestExportRefusesRedirects(t *testing.T) {
|
||||
@@ -97,7 +245,14 @@ func TestExportRefusesRedirects(t *testing.T) {
|
||||
defer target.Close()
|
||||
origin := httptest.NewServer(http.RedirectHandler(target.URL, http.StatusTemporaryRedirect))
|
||||
defer origin.Close()
|
||||
require.EqualError(t, (&client{origin.URL, http.Header{}}).UploadTraces(t.Context(), nil), "OTLP export failed: 307 Temporary Redirect")
|
||||
require.EqualError(t, (&client{endpoint: origin.URL, headers: http.Header{}, timeout: time.Second, httpClient: exportClient}).UploadTraces(t.Context(), nil), "OTLP export failed: 307 Temporary Redirect")
|
||||
}
|
||||
|
||||
func TestMillis(t *testing.T) {
|
||||
for value, want := range map[string]time.Duration{"": time.Minute, "1500": 1500 * time.Millisecond, "0": noLimit, "-1": time.Minute, "1s": time.Minute, "2147483648": time.Minute} {
|
||||
t.Setenv("OTEL_METRIC_EXPORT_TIMEOUT", value)
|
||||
assert.Equal(t, want, millis("OTEL_METRIC_EXPORT_TIMEOUT", time.Minute, noLimit), value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJob(t *testing.T) {
|
||||
|
||||
+9
-14
@@ -36,28 +36,23 @@ fi
|
||||
test -f "$RUNNER_STATE_FILE" || echo "$RUNNER_STATE_FILE is missing or not a regular file"
|
||||
|
||||
if [[ ! -s "$RUNNER_STATE_FILE" ]]; then
|
||||
try=$((try + 1))
|
||||
success=0
|
||||
|
||||
# The point of this loop is to make it simple, when running both runner and gitea in docker,
|
||||
# for the runner to wait a moment for gitea to become available before erroring out. Within
|
||||
# the context of a single docker-compose, something similar could be done via healthchecks, but
|
||||
# this is more flexible.
|
||||
while [[ $success -eq 0 ]] && [[ $try -lt ${GITEA_MAX_REG_ATTEMPTS:-10} ]]; do
|
||||
for ((try = 1; ; try++)); do # waits for Gitea like the daemon does, a container may have no restart policy
|
||||
gitea-runner register \
|
||||
--instance "${GITEA_INSTANCE_URL}" \
|
||||
--token "${GITEA_RUNNER_REGISTRATION_TOKEN}" \
|
||||
--name "${GITEA_RUNNER_NAME:-`hostname`}" \
|
||||
${CONFIG_ARG} ${EXTRA_ARGS} --no-interactive 2>&1 | tee /tmp/reg.log
|
||||
|
||||
cat /tmp/reg.log | grep 'Runner registered successfully' > /dev/null
|
||||
if [[ $? -eq 0 ]]; then
|
||||
if grep -q 'Runner registered successfully' /tmp/reg.log; then
|
||||
echo "SUCCESS"
|
||||
success=1
|
||||
else
|
||||
echo "Waiting to retry ..."
|
||||
sleep 5
|
||||
break
|
||||
fi
|
||||
if [[ ${GITEA_MAX_REG_ATTEMPTS:-0} -gt 0 ]] && [[ $try -ge $GITEA_MAX_REG_ATTEMPTS ]]; then
|
||||
echo "Registration failed after ${try} attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting to retry ..."
|
||||
sleep 5
|
||||
done
|
||||
fi
|
||||
# Prevent reading the token from the gitea-runner process
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exec /etc/s6/.s6-svscan/SIGTERM
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
(set -C; echo 0 > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
|
||||
s6-svc -wD -d /etc/s6/gitea-runner
|
||||
exec s6-svscanctl -t /etc/s6
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "/etc/s6/.s6-svscan/exitcode.$(cat /proc/sys/kernel/random/boot_id).$(cut -d ' ' -f 22 /proc/1/stat)" # unique per container start, so a killed run's record is ignored
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exitcode=$(cat "$(/etc/s6/.s6-svscan/exitcode-path)" 2> /dev/null)
|
||||
rm -f /etc/s6/.s6-svscan/exitcode.*
|
||||
exit "${exitcode:-0}"
|
||||
@@ -1,4 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exec s6-svscanctl -t /etc/s6
|
||||
if [ "$1" -eq 256 ]; then
|
||||
set -- "$((128 + $2))"
|
||||
fi
|
||||
(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
|
||||
|
||||
s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown
|
||||
@@ -1,3 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exec s6-notifyoncheck dockerd-entrypoint.sh
|
||||
exec s6-notifyoncheck -n 0 dockerd-entrypoint.sh
|
||||
@@ -1,3 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exec s6-svscanctl -t /etc/s6
|
||||
if [ "$1" -eq 256 ]; then
|
||||
set -- "$((128 + $2))"
|
||||
fi
|
||||
(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
|
||||
|
||||
s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown
|
||||
@@ -1,9 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
while [ ! -d /etc/s6/docker/supervise ]; do
|
||||
sleep 0.1
|
||||
until /etc/s6/docker/data/check &> /dev/null; do # s6-svwait can read missing or stale readiness status
|
||||
sleep 1
|
||||
done
|
||||
|
||||
s6-svwait -U /etc/s6/docker
|
||||
|
||||
exec run.sh
|
||||
+29
-3
@@ -10,8 +10,8 @@
|
||||
# Usage: scripts/test-dind.sh [target] [-- go-test-args...]
|
||||
# target: dind (default), dind-rootless, or podman to run PODMAN_TEST_IMAGE's API service instead
|
||||
# go-test-args: passed verbatim to `go test`. Defaults cover image env extraction,
|
||||
# symlink copying and a mounted Docker job using cached images, or the
|
||||
# Docker proxy probe for podman.
|
||||
# symlink copying, a mounted Docker job using cached images and the
|
||||
# image's s6 supervision, or the Docker proxy probe for podman.
|
||||
#
|
||||
# Env:
|
||||
# DIND_TEST_PORT host port for the daemon (default 32375)
|
||||
@@ -33,7 +33,7 @@ host_docker="${DOCKER_HOST:-$(docker context inspect --format '{{.Endpoints.dock
|
||||
test_dir=""
|
||||
|
||||
cleanup() {
|
||||
docker -H "$host_docker" rm -fv "$name" >/dev/null 2>&1 || true
|
||||
docker -H "$host_docker" rm -fv "$name" "$name-s6" >/dev/null 2>&1 || true
|
||||
if [ -n "$test_dir" ]; then
|
||||
rm -rf "$test_dir"
|
||||
fi
|
||||
@@ -147,4 +147,30 @@ if [ "$default_tests" = true ]; then
|
||||
echo "==> Running mounted Docker job in a container given the ${target} socket, expecting proxy access"
|
||||
docker -H "$host_docker" exec -e DOCKER_HOST="$socket" "$name" docker run --rm -v "${socket#unix://}:/var/run/docker.sock" -v /tmp/gitea-runner-proxy-test:/data -w /data \
|
||||
-e ACT_TEST_DOCKER_PROXY=proxy -e ACT_TEST_IMAGE="$job_image" "$job_image" ./runner.test -test.v -test.run '^TestDockerProxyMountedJob$' -test.timeout 3m
|
||||
|
||||
supervised="$name-s6"
|
||||
supervised_exit_is() {
|
||||
if [ "$(docker -H "$host_docker" inspect -f '{{.State.ExitCode}}' "$supervised")" != "$1" ] ||
|
||||
! docker -H "$host_docker" logs "$supervised" 2>&1 | grep -F "$2" >/dev/null; then
|
||||
docker -H "$host_docker" logs "$supervised" >&2
|
||||
echo "${target} supervision: expected exit $1 after \"$2\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
echo "==> Checking that a runner failure sets the ${target} container exit code"
|
||||
docker -H "$host_docker" run -d --privileged --name "$supervised" -e DOCKER_TLS_CERTDIR= -e RUNNER_STATE_FILE=/etc/hostname "$image" >/dev/null
|
||||
for _ in $(seq 1 120); do
|
||||
[ "$(docker -H "$host_docker" inspect -f '{{.State.Running}}' "$supervised")" = false ] && break
|
||||
sleep 1
|
||||
done
|
||||
supervised_exit_is 1 'registration file not found'
|
||||
docker -H "$host_docker" rm -fv "$supervised" >/dev/null
|
||||
echo "==> Checking that stopping a started ${target} runner exits 0"
|
||||
docker -H "$host_docker" run -d --privileged --name "$supervised" -e DOCKER_TLS_CERTDIR= "$image" >/dev/null
|
||||
for _ in $(seq 1 120); do
|
||||
docker -H "$host_docker" logs "$supervised" 2>&1 | grep -F 'is missing or not a regular file' >/dev/null && break
|
||||
sleep 1
|
||||
done
|
||||
docker -H "$host_docker" stop "$supervised" >/dev/null
|
||||
supervised_exit_is 0 'is missing or not a regular file'
|
||||
fi
|
||||
@@ -4,10 +4,12 @@ set -euo pipefail
|
||||
: "${E2E_CONCURRENCY:?E2E_CONCURRENCY is required}"
|
||||
: "${E2E_JOB_IMAGE:?E2E_JOB_IMAGE is required}"
|
||||
: "${E2E_GITEA_IMAGE:?E2E_GITEA_IMAGE is required}"
|
||||
: "${E2E_K3S_IMAGE:?E2E_K3S_IMAGE is required}"
|
||||
: "${SERVICE_IMAGE:?SERVICE_IMAGE is required}"
|
||||
|
||||
docker pull "$E2E_JOB_IMAGE"
|
||||
docker pull "$SERVICE_IMAGE"
|
||||
docker pull "$E2E_GITEA_IMAGE"
|
||||
docker pull "$E2E_K3S_IMAGE"
|
||||
|
||||
exec "${GO:-go}" test -tags e2e -count=1 -parallel "$E2E_CONCURRENCY" -timeout 20m -v ./e2e/...
|
||||
Reference in new issue
Block a user