Compare commits

..
Author SHA1 Message Date
Lunny Xiao de9db9c0fa fix: suppress tar leading slash warnings 2026-09-06 20:17:55 -07:00
Lunny Xiao d1da6ffa52 fix: create guest workspace in real macOS VM integration test 2026-09-06 18:37:15 -07:00
Lunny Xiao 880e0a148a fix: prepare macOS VM workspace and archive without AppleDouble 2026-09-05 21:56:15 -07:00
Lunny Xiao 2db24c7401 enhance: add macOS VM executor backend
Replace the Tart-based prototype with a runner-oriented macOS VM
executor backed by runner-executor-macos and Apple Virtualization.

Assisted-by: Codet:MODEL_VERSION
2026-09-05 13:20:03 -07:00
215 changed files with 5431 additions and 14841 deletions

No files matched your search

+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- run: make lint-pr-title
+3 -3
View File
@@ -74,10 +74,10 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
@@ -94,7 +94,7 @@ jobs:
echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile
+3 -3
View File
@@ -77,10 +77,10 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
@@ -103,7 +103,7 @@ jobs:
suffix=${{ matrix.variant.tag_suffix }},onlatest=true
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile
+5 -5
View File
@@ -1,7 +1,9 @@
name: checks
on:
push:
branches:
- main
pull_request:
workflow_dispatch:
env:
# The runner image ships a stale docker.io login; point docker at an empty config so
@@ -28,8 +30,8 @@ jobs:
# daemon retains them between runs, so this is usually a fast manifest re-check.
- name: pre-pull test images
env:
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:d6aa754f16b3197301076f047b5def2f02ea1dbbc2ca920407d46d7ec7f87b20 # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:ba849c60be29959425b8734d57b8b4b7d56f98edd9504c9af091d5281095a71e # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
run: |
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
for attempt in 1 2 3; do
@@ -51,8 +53,6 @@ jobs:
# after `make test` so the images it needs are already present on the host daemon.
- name: test against dind image
run: make test-dind
- name: test against podman
run: make test-dind TARGET=podman
- name: coverage report
run: |
make coverage-report
-9
View File
@@ -39,15 +39,6 @@ linters:
deny:
- pkg: golang.org/x/exp
desc: it's experimental and unreliable
logging:
files:
- $all
- "!**/act/common/logging/**"
deny:
- pkg: github.com/sirupsen/logrus
desc: log through gitea.com/gitea/runner/act/common/logging
- pkg: log/slog
desc: log through gitea.com/gitea/runner/act/common/logging
nolintlint:
allow-unused: false
require-explanation: true
+1
View File
@@ -0,0 +1 @@
@AGENTS.md
-22
View File
@@ -31,28 +31,6 @@ depending on the prefix:
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
decodes exactly those two when folding a location into an annotation.
## Logging
Log through `act/common/logging`; the linter rejects importing `log/slog` or logrus elsewhere.
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
fields.
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
Going around either leaks them.
- Libraries that log through logrus's or slog's global logger are forwarded to the process-wide
one, so they follow its level and masking too.
- That forwarding only works while logrus's global logger is left as `logging` configures it. A
dependency that calls `logrus.SetOutput` or `SetLevel` breaks it, so check
`logging.LogrusBridgeIntact()` when adding or upgrading one.
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
package functions.
## Built-in actions
Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions` in
`act/runner/step_action_builtin.go`.
## End-to-end compatibility tests
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
-3
View File
@@ -32,8 +32,6 @@ COPY scripts/s6 /etc/s6
VOLUME /data
ENV TINI_SUBREAPER=1
ENTRYPOINT ["s6-svscan","/etc/s6"]
### DIND-ROOTLESS VARIANT
@@ -58,7 +56,6 @@ VOLUME /data
RUN mkdir -p /data && chown -R rootless:rootless /etc/s6 /data
ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
ENV TINI_SUBREAPER=1
USER rootless
ENTRYPOINT ["s6-svscan","/etc/s6"]
+23 -16
View File
@@ -6,13 +6,7 @@ SHASUM ?= shasum -a 256
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
XGO_PACKAGE ?= src.techknowlogick.com/[email protected] # renovate: datasource=go
XGO_VERSION := go-1.27.x
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected]7 # renovate: datasource=go
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected]6 # renovate: datasource=go
LINUX_ARCHS ?= linux/amd64,linux/arm64
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
@@ -24,8 +18,8 @@ DOCKER_TAG ?= nightly
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]3.1 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.7.0 # renovate: datasource=go
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
@@ -102,8 +96,13 @@ go-check:
fi
.PHONY: fmt-check
fmt-check: ## check the Go code formatting
$(GO) run $(GOLANGCI_LINT_PACKAGE) fmt --diff-colored
fmt-check: fmt
@diff=$$(git diff --color=always -- '*.go'); \
if [ -n "$$diff" ]; then \
echo "Please run 'make fmt' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: deps-tools
deps-tools: ## install tool dependencies
@@ -150,8 +149,13 @@ tidy: ## run go mod tidy
fi
.PHONY: tidy-check
tidy-check: ## check that go.mod and go.sum are tidy
$(GO) mod tidy -diff
tidy-check: tidy
@diff=$$(git diff --color=always -- go.mod go.sum); \
if [ -n "$$diff" ]; then \
echo "Please run 'make tidy' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: test
test: ## test everything (integration tests self-skip without docker/network)
@@ -164,14 +168,17 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
@echo "Wrote .tmp/coverage.md"
.PHONY: test-dind
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless)
@./scripts/test-dind.sh $(TARGET)
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_CONCURRENCY ?= 8
.PHONY: test-e2e
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
.PHONY: install
install: $(GOFILES) ## install the runner binary via `go install`
+104 -108
View File
@@ -4,19 +4,7 @@
### Prerequisites
On the runner:
- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows.
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
Where steps run, in the job image or on the runner in host mode:
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
- `node` for JavaScript actions and `hashFiles()`.
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
### Download pre-built binary
@@ -91,24 +79,53 @@ If the registry succeed, it will run immediately. Next time, you could run the r
### Run with docker
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
```bash
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> -v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:nightly
```
The volumes keep the runner's registration and the job image cache across container recreation.
Mount a volume on `/data` if you want the registration file and optional config to survive container recreation (see [scripts/run.sh](scripts/run.sh)).
> **`/data` does not hold the image cache.** It is the runner's working directory and contains only the `.runner` registration file and, optionally, your config file. Images pulled for jobs live in the *Docker daemon's* data root, which for the `dind` flavours is inside the container (`/var/lib/docker`, or `/home/rootless/.local/share/docker` for `dind-rootless`). To keep the image cache across restarts, give that path its own volume as well — otherwise every new container re-pulls the job images. With the `basic` flavour the images live on whichever daemon you point the runner at, so there is nothing extra to persist.
### Image flavours
| Tag | Docker daemon for jobs |
| --- | --- |
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
The image is published in three flavours, all built from the single multi-stage [Dockerfile](Dockerfile) in this repository. They differ only in how a Docker daemon is made available to the jobs the runner executes; the `gitea-runner` binary inside them is identical.
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
| Tag | Build target | Base image | Docker daemon | Process supervisor | Runs as |
| --- | --- | --- | --- | --- | --- |
| `latest` (and `<version>`) | `basic` | `alpine` | none — uses an external daemon you provide | [`tini`](https://github.com/krallin/tini) | `root` |
| `latest-dind` | `dind` | `docker:dind` | bundled, started inside the container | [`s6`](https://skarnet.org/software/s6/) | `root` (privileged) |
| `latest-dind-rootless` | `dind-rootless` | `docker:dind-rootless` | bundled, started rootless inside the container | [`s6`](https://skarnet.org/software/s6/) | `rootless` (UID 1000) |
#### `latest` — basic
The default flavour ships only the runner on a minimal Alpine base. It contains **no Docker daemon of its own**: jobs that use `docker://` images need a daemon supplied from outside the container, typically by bind-mounting the host's socket:
```bash
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
-v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:latest
```
`tini` is the entrypoint (it reaps zombie processes), and it just runs [`scripts/run.sh`](scripts/run.sh), which registers the runner on first start and then execs `gitea-runner daemon`. This flavour does not need `--privileged`. The trade-off is that jobs share the host's daemon, so they can see other containers and images on that daemon.
#### `latest-dind` — Docker-in-Docker
This flavour is based on the official `docker:dind` image and bundles its own Docker daemon, so it needs no external socket — only the `--privileged` flag that Docker-in-Docker requires:
```bash
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
--name my_runner gitea/runner:latest-dind
```
Two processes have to run side by side here (the Docker daemon and the runner), so the entrypoint is the [`s6`](https://skarnet.org/software/s6/) supervision tree under [`scripts/s6`](scripts/s6) instead of `tini`. `s6` starts `dockerd`, and the runner service waits for the daemon to come up (`s6-svwait`) before launching [`run.sh`](scripts/run.sh). Each container has a private daemon isolated from the host's, at the cost of running privileged.
#### `latest-dind-rootless` — rootless Docker-in-Docker
Same idea as `dind`, but built on `docker:dind-rootless` so the bundled daemon and the runner run as an unprivileged user (`rootless`, UID 1000) rather than `root`. `DOCKER_HOST` is preset to `unix:///run/user/1000/docker.sock` so the runner talks to the rootless daemon. This reduces the blast radius compared to the privileged `dind` flavour, but rootless Docker carries the usual rootless limitations (networking, cgroups, storage drivers, and some operations that need additional host configuration such as `/etc/subuid` / `/etc/subgid` mappings and unprivileged user-namespace support).
> **The UID is fixed at 1000.** It comes from the `rootless` user baked into the upstream `docker:dind-rootless` base image, and the bundled daemon always listens on `/run/user/1000/docker.sock` inside the container, so running this flavour as a different user (`--user 1001`) does not work. If you need the runner to talk to a *host* rootless daemon that runs under some other UID, use the `basic` flavour instead and bind-mount that daemon's socket (see [examples/vm/rootless-docker.md](examples/vm/rootless-docker.md)); pointing `DOCKER_HOST` at a host socket from inside `dind-rootless` will not work. Changing the UID otherwise means rebuilding the image from a base with a different `rootless` user.
> **Note on Podman:** these images target the Docker daemon. The bundled `dind`/`dind-rootless` daemons are `dockerd`, not Podman, and the `basic` flavour expects a Docker-compatible socket. Running them under rootless Podman is not a supported configuration, though pointing the `basic` flavour at a Podman socket that emulates the Docker API may work for some workloads.
### Configuration
@@ -169,7 +186,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
#### Environment variables
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
### Labels
@@ -184,8 +201,8 @@ A label is written as:
| Part | Meaning |
| --- | --- |
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
| `schema` | Either `docker`, `host`, or `macos-vm`. Defaults to `host` when omitted. |
| `args` | Used by `docker` for the image, and by `macos-vm` for the source VM. |
Three schemas are supported:
@@ -195,29 +212,29 @@ Three schemas are supported:
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
```
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
```text
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
```
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
```text
macos:host
```
- **`macos-vm://<source-vm>`** — the job runs inside an ephemeral macOS VM created by `runner-executor-macos`. The VM image must include the runner guest agent. Service containers are not supported with this schema. See `docs/macos-vm-executor.md` for configuration, integration testing, and cleanup details:
```text
macos-latest:macos-vm://ghcr.io/cirruslabs/macos-sonoma-base:latest
```
So with the labels
```text
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host,macos-latest:macos-vm://ghcr.io/cirruslabs/macos-sonoma-base:latest
```
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, one with `runs-on: macos` is executed directly on the host, and one with `runs-on: macos-latest` is executed in the macOS VM.
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host`, `docker`, and `macos-vm` are treated as schemas.
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs: in `runner.default_image` where docker is available, on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
@@ -255,17 +272,6 @@ A job in a container reaches a service by its id on the job network, on the port
Unlike GitHub, a job whose steps run on the host (a `host` label without `container:`) starts no service containers, so `job.services` and `job.container` stay empty. Give such a job a `container:` when it needs services.
#### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
Containers a job starts through the Docker socket can bind-mount the workspace by the job's own path, as on a host, for example `docker run -v "$PWD:/src"`. Without the Docker proxy, for example with a remote `DOCKER_HOST`, use `GITEA_DOCKER_WORKSPACE`, the path the daemon sees, as the prefix of workspace binds, with `.` as the fallback for local use:
```yaml
volumes:
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
```
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
#### Proxy
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
@@ -280,14 +286,14 @@ The runner uses them for its own requests and gives them to every job, in lower
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
- the built-in cache server, whose address is assigned at startup
- the cache server
- `localhost`, `127.0.0.1` and `::1`
- the job's service containers
- the Docker daemon, when it is reached over `tcp://`
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
For one step, set it in the step's `env:`, for the rest of a job, write it to `$GITHUB_ENV`. Setting it at workflow or job level or in `container.env` has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
To change a value for one job, set it in a step's `env:` or in the job's `container.env`. Setting it at workflow or job level has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
@@ -295,68 +301,63 @@ Dockerfile actions are built with these variables as build arguments, so their `
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
#### Built-in actions (`uses: builtin:<name>`)
Use `uses: builtin:checkout` to check out a repository without downloading `actions/checkout` or needing Node in the job image, see **[docs/builtin-actions.md](docs/builtin-actions.md)**.
#### Caching (`actions/cache`)
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
**Sharing a cache between runners**
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
```yaml
# cache-server.yaml
cache:
dir: /data/actcache
port: 8088
external_secret: "<secret>" # or external_secret_file: /path/to/secret
```
```bash
gitea-runner -c cache-server.yaml cache-server
```
```yaml
# each runner's config
cache:
external_server: "http://<cache-server-host>:8088/"
external_secret: "<secret>"
```
`--dir`, `--host` and `--port` override the matching `cache` keys. Eviction settings take effect on the cache server, not on the runners.
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower cache requests.
**Storing the cache in an S3-compatible bucket**
The cache can also be stored in an S3-compatible bucket such as AWS S3, MinIO or RustFS, with no cache server. Runners with the same settings use the same cache:
```yaml
cache:
s3:
endpoint: "http://minio:9000"
region: "us-east-1"
bucket: "runner-cache"
prefix: "gitea-runner-cache"
path_style: true
access_key_file: "/run/secrets/cache-s3-access-key"
secret_key_file: "/run/secrets/cache-s3-secret-key"
```
A cache saved on one runner can be restored on the others within 30 seconds. The eviction settings below do not apply to the bucket, so add lifecycle rules to it that expire old caches and abort incomplete multipart uploads.
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
**Eviction**
Entries not read or written within `retention` (default `168h`) are removed. A repository over `repo_size_limit` (default `10GB`) loses its least recently used entries, and `size_limit` (off by default) caps the whole cache the same way. Entries in use are never removed. The cache also keeps 1024 MiB free on its volume, or `health_check.min_free_disk_space_mb` when health checks are enabled. See [config.example.yaml](internal/pkg/config/config.example.yaml) for all options.
An entry nothing has read or written for `retention` is removed, and a repository past `repo_size_limit` loses its least recently accessed entries until it fits; `size_limit` caps the whole cache the same way. Age alone never retires an entry still in use, and whatever these allow, the cache keeps free space above `health_check.min_free_disk_space_mb` when health checks are enabled.
These apply where the cache server runs, so on a shared server they belong in *its* config, not the runners'. See `retention`, `repo_size_limit`, `size_limit` and `sweep_interval` in [config.example.yaml](internal/pkg/config/config.example.yaml) for units and defaults.
**Cache service v2**
`actions/cache` v3.4.0, v4.2.0 and later use the cache service v2 API, which the runner serves by default. These actions fall back to v1 on hosts they do not recognize as GitHub, so the runner removes that check from them while a job runs. This also lets the stock `actions/upload-artifact` v4.4.0 and `actions/download-artifact` v4.1.8 and later work without the `gitea-upload-artifact` fork. With `runner.patch_actions: false`, the cache stays on v1 and the artifact actions fail.
`actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with:
With v2, artifact calls also go through the cache server, so jobs must be able to reach it. `cache.v2: false` sends them to Gitea directly, unless the Gitea URL has a path or `runner.insecure` is set with HTTPS.
```yaml
cache:
v2: false
```
Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle on its way into the job, undone whenever the action is downloaded again. A bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork, so it is made whatever `v2` says: that setting only governs the API the runner advertises. Set `runner.patch_actions: false` to leave every bundle exactly as shipped, an escape hatch for an action the edit breaks. The artifact actions then refuse again and the cache client keeps to v1.
**Shared cache across multiple runners**
Run one dedicated `gitea-runner cache-server` that all runners point at.
1. Create a config file for the cache server host:
```yaml
cache:
dir: /data/actcache
port: 8088
external_secret: "replace-with-a-strong-random-secret"
# external_secret_file: /path/to/secret # secret can also be passed via a file
```
2. Start the server:
```bash
gitea-runner -c cache-server-config.yaml cache-server
```
3. On every runner:
```yaml
cache:
external_server: "http://<cache-server-host>:8088/"
external_secret: "replace-with-a-strong-random-secret" # must match the server
# external_secret_file: /path/to/secret # secret can also be passed via a file
```
Jobs reach the cache server at `external_server`, so when a reverse proxy fronts the server, point `external_server` at the proxy. The cache server itself needs no extra configuration.
Alternatively, mount the same NFS/CIFS share on every runner and point `cache.dir` at it — simpler, but with weaker isolation between repositories.
**S3 / MinIO** — mount object storage as a FUSE filesystem (e.g. [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) or [goofys](https://github.com/kahing/goofys)) and set `cache.dir` to the mount point.
Flags `--dir`, `--host`, and `--port` on `cache-server` override the corresponding `cache.*` YAML keys; all other settings, including `external_secret`, require the config file.
#### Official Docker image
@@ -369,7 +370,6 @@ While the runner is idle it cleans up after earlier jobs:
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
#### Post-task script (`runner.post_task_script`)
@@ -392,10 +392,6 @@ Both hooks are synchronous and block the job while they run. Either one exiting
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
#### OpenTelemetry (`OTEL_*`)
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
#### Local job logs (`log.job.dir`)
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
+57 -234
View File
@@ -10,17 +10,17 @@ import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json/v2"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
@@ -29,10 +29,10 @@ import (
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/internal/pkg/disk"
"github.com/julienschmidt/httprouter"
"github.com/sirupsen/logrus"
"github.com/timshannon/bolthold"
"go.etcd.io/bbolt"
)
@@ -58,9 +58,10 @@ type JobCredential struct {
Repo string `json:"repo"`
// Results is the instance whose artifact service this server forwards for the job, and
// InsecureTLS how the runner reaches it; see results.go.
Results string `json:"-"`
InsecureTLS bool `json:"-"`
// InsecureTLS how the runner reaches it; see results.go. The tags are the wire format a
// remote runner registers with.
Results string `json:"results"`
InsecureTLS bool `json:"insecure_tls"`
// PublicURL is this server as a reverse proxy makes the job reach it, not the listen address.
PublicURL string `json:"public_url"`
@@ -78,20 +79,16 @@ type credEntry struct {
type Handler struct {
dir string
storage cacheStorage
s3 *s3Storage
storage *Storage
router *httprouter.Router
listener net.Listener
port int
server *http.Server
logger *logging.Logger
logger logrus.FieldLogger
gcing atomic.Bool
gcAt time.Time
stopSync context.CancelFunc
syncing sync.WaitGroup
outboundIP string
// internalSecret guards /_internal/{register,revoke}. When set, a remote
@@ -121,8 +118,6 @@ type Options struct {
Dir string
OutboundIP string
Port uint16
S3 *S3Options
Upstream string
// InternalSecret, when non-empty, enables a control-plane API at
// /_internal/{register,revoke} that lets a remote runner pre-register the
@@ -132,10 +127,10 @@ type Options struct {
InternalSecret string
Policy Policy
Logger *logging.Logger
Logger logrus.FieldLogger
}
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
// StartHandler opens the on-disk cache store and starts the HTTP server.
func StartHandler(opts Options) (*Handler, error) {
dir, logger := opts.Dir, opts.Logger
h := &Handler{
@@ -146,30 +141,13 @@ func StartHandler(opts Options) (*Handler, error) {
}
if logger == nil {
logger = logging.New(logging.DiscardHandler)
discard := logrus.New()
discard.Out = io.Discard
logger = discard
}
logger = logger.With("module", "artifactcache")
logger = logger.WithField("module", "artifactcache")
h.logger = logger
if opts.OutboundIP != "" {
h.outboundIP = opts.OutboundIP
} else if ip := common.GetOutboundIP(); ip == nil {
return nil, errors.New("unable to determine outbound IP address")
} else {
h.outboundIP = ip.String()
}
if opts.Upstream != "" {
handler, err := h.cacheForwarder(opts.Upstream)
if err != nil {
return nil, err
}
if err := h.serve(opts.Port, handler); err != nil {
return nil, err
}
return h, nil
}
if dir == "" {
home, err := os.UserHomeDir()
if err != nil {
@@ -188,11 +166,13 @@ func StartHandler(opts Options) (*Handler, error) {
return nil, err
}
h.storage = storage
if opts.S3 != nil {
if h.s3, err = newS3Storage(storage, *opts.S3); err != nil {
return nil, err
}
h.storage = h.s3
if opts.OutboundIP != "" {
h.outboundIP = opts.OutboundIP
} else if ip := common.GetOutboundIP(); ip == nil {
return nil, errors.New("unable to determine outbound IP address")
} else {
h.outboundIP = ip.String()
}
secret, err := loadOrCreateSecret(dir)
@@ -222,79 +202,39 @@ func StartHandler(opts Options) (*Handler, error) {
h.gcCache()
if err := h.serve(opts.Port, router); err != nil {
return nil, err
}
if h.s3 != nil {
ctx, cancel := context.WithCancel(context.Background())
h.stopSync = cancel
h.syncing.Go(func() { h.syncMetadata(ctx) })
}
return h, nil
}
func (h *Handler) serve(port uint16, handler http.Handler) error {
// Listen on all interfaces. Binding to outboundIP only would give no real
// security benefit (it is the LAN/internet-facing address either way) and
// can break Docker Desktop variants where the host's outbound IP is not
// routable from inside the container network. Authentication is enforced
// by the bearer middleware and per-repo scoping, not by reachability.
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", opts.Port))
if err != nil {
return err
return nil, err
}
addr, ok := listener.Addr().(*net.TCPAddr)
if !ok {
listener.Close()
return fmt.Errorf("cache server listens on %T, want a TCP address", listener.Addr())
return nil, fmt.Errorf("cache server listens on %T, want a TCP address", listener.Addr())
}
h.port = addr.Port
server := &http.Server{
ReadHeaderTimeout: 2 * time.Second,
Handler: handler,
Handler: router,
}
go func() {
if err := server.Serve(listener); err != nil && errors.Is(err, net.ErrClosed) {
h.logger.Errorf("http serve: %v", err)
logger.Errorf("http serve: %v", err)
}
}()
h.listener = listener
h.server = server
return nil
}
func (h *Handler) cacheForwarder(upstream string) (http.Handler, error) {
target, err := url.Parse(strings.TrimRight(upstream, "/"))
if err != nil {
return nil, fmt.Errorf("parse cache upstream: %w", err)
}
transport, _ := http.DefaultTransport.(*http.Transport)
transport = transport.Clone()
transport.MaxIdleConnsPerHost = 64 // parallel block uploads would otherwise redial past the default 2
proxy := &httputil.ReverseProxy{
Transport: transport,
Rewrite: func(r *httputil.ProxyRequest) {
r.SetURL(target)
r.Out.URL.RawQuery = r.In.URL.RawQuery // Rewrite drops unparsable query params, and signed URLs must pass verbatim.
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
h.logger.Warnf("cache forward %s %s: %v", r.Method, r.URL.Path, err)
h.twirpError(w, r, twirpUnavailable, errors.New("cache upstream is unavailable"))
},
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, apiPath+"/") || strings.HasPrefix(r.URL.Path, cacheServiceV2Path+"/") {
proxy.ServeHTTP(w, r)
} else {
h.forwardOrNotFound(w, r)
}
}), nil
return h, nil
}
func (h *Handler) ExternalURL() string {
// TODO: make the external url configurable if necessary
return "http://" + net.JoinHostPort(h.outboundIP, strconv.Itoa(h.port))
return fmt.Sprintf("http://%s:%d", h.outboundIP, h.port)
}
func (h *Handler) baseURL(cred JobCredential) string {
@@ -394,10 +334,6 @@ func (h *Handler) Close() error {
if h == nil {
return nil
}
if h.stopSync != nil {
h.stopSync()
h.syncing.Wait()
}
var retErr error
if h.server != nil {
err := h.server.Close()
@@ -431,72 +367,6 @@ func (h *Handler) openDB() (*bolthold.Store, error) {
})
}
// syncMetadata mirrors the bucket into the store until ctx ends, so no request waits on S3 for it.
func (h *Handler) syncMetadata(ctx context.Context) {
ticker := time.NewTicker(metadataSyncInterval)
defer ticker.Stop()
for {
if err := h.importMetadata(ctx); err != nil && ctx.Err() == nil {
h.logger.Warnf("sync S3 cache: %v", err)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// importMetadata lists after snapshotting the store, so an entry committed meanwhile, published before it is stored, stays.
func (h *Handler) importMetadata(ctx context.Context) error {
db, err := h.openDB()
if err != nil {
return err
}
local := h.caches(db, bolthold.Where("Complete").Eq(true).Index("Complete"))
db.Close()
published, err := h.s3.published(ctx)
if err != nil {
return err
}
var gone []*Cache
for _, cache := range local {
if !published[cache.ID] {
gone = append(gone, cache)
}
delete(published, cache.ID)
}
added := make([]*Cache, 0, len(published))
for id := range published {
cache, err := h.s3.fetch(ctx, id)
if err != nil {
return err
}
added = append(added, cache)
}
db, err = h.openDB()
if err != nil {
return err
}
defer db.Close()
return db.Bolt().Update(func(tx *bbolt.Tx) error {
for _, cache := range added {
if err := db.TxUpsert(tx, cache.ID, cache); err != nil {
return err
}
}
for _, cache := range gone {
_ = h.s3.Storage.Remove(cache.ID) // a blob kept on disk before the runner used S3
if err := db.TxDelete(tx, cache.ID, cache); err != nil && !errors.Is(err, bolthold.ErrNotFound) {
return err
}
}
return nil
})
}
// GET /_apis/artifactcache/cache
func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
@@ -539,7 +409,7 @@ func (h *Handler) lookupCache(db *bolthold.Store, repo string, keys []string, ve
return nil, err
}
if !ok {
h.deleteCache(db, cache)
_ = db.Delete(cache.ID, cache)
return nil, nil //nolint:nilnil // absence is not an error here
}
// Handing out a download URL counts as access, or eviction could drop the entry between
@@ -571,7 +441,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.P
now := time.Now().Unix()
cache.CreatedAt = now
cache.UsedAt = now
if err := h.insertCache(db, cache); err != nil {
if err := insertCache(db, cache); err != nil {
h.responseJSON(w, r, 500, err)
return
}
@@ -685,12 +555,6 @@ func (h *Handler) commitCache(cache *Cache) error {
cache.Size = written
cache.Complete = true
cache.UsedAt = time.Now().Unix() // a just-written entry counts as accessed, so it cannot be its own eviction victim
if h.s3 != nil {
if err := h.s3.publish(cache); err != nil {
_ = h.storage.Remove(cache.ID)
return err
}
}
db, err := h.openDB()
if err != nil {
@@ -700,9 +564,6 @@ func (h *Handler) commitCache(cache *Cache) error {
if err := db.Update(cache.ID, cache); err != nil {
return err
}
if h.s3 != nil {
return nil // lifecycle rules on the bucket bound its size
}
// A commit is the only thing that grows the store, so the only thing that can push the
// volume under the floor.
h.evictRepo(db, cache.Repo)
@@ -742,12 +603,12 @@ func (h *Handler) bearerAuth(handler httprouter.Handle) httprouter.Handle {
h.logger.Debugf("%s %s", r.Method, r.URL.Path)
token := bearerToken(r)
if token == "" {
h.unauthorized(w, r, errors.New("missing bearer token"))
h.responseJSON(w, r, http.StatusUnauthorized, errors.New("missing bearer token"))
return
}
cred, ok := h.lookupCredential(token)
if !ok {
h.unauthorized(w, r, errors.New("unknown bearer token"))
h.responseJSON(w, r, http.StatusUnauthorized, errors.New("unknown bearer token"))
return
}
ctx := context.WithValue(r.Context(), credKey{}, cred)
@@ -756,14 +617,6 @@ func (h *Handler) bearerAuth(handler httprouter.Handle) httprouter.Handle {
}
}
func (h *Handler) unauthorized(w http.ResponseWriter, r *http.Request, err error) {
if strings.HasPrefix(r.URL.Path, cacheServiceV2Path) {
h.twirpError(w, r, twirpUnauthenticated, err)
return
}
h.responseJSON(w, r, http.StatusUnauthorized, err)
}
// signedAuth authenticates a signed URL. purpose separates the flavours of URL the
// handler hands out, so one cannot be replayed as another; see computeSignature.
func (h *Handler) signedAuth(purpose string, handler httprouter.Handle) httprouter.Handle {
@@ -848,7 +701,8 @@ func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ htt
return
}
h.RegisterJob(body.Token, body.JobCredential)
h.responseJSON(w, r, http.StatusOK)
// A server too old to forward answers without this, which is how the caller knows.
h.responseJSON(w, r, http.StatusOK, map[string]any{"results_url": h.ResultsURL(body.JobCredential)})
}
// POST /_internal/revoke
@@ -866,12 +720,6 @@ func (h *Handler) internalRevoke(w http.ResponseWriter, r *http.Request, _ httpr
h.responseJSON(w, r, http.StatusOK)
}
// hashedToken fingerprints a job's bearer, so a reservation can tell its own retry from another job.
func hashedToken(token string) string {
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])
}
func bearerToken(r *http.Request) string {
auth := r.Header.Get("Authorization")
const prefix = "Bearer "
@@ -922,9 +770,14 @@ func findCache(db *bolthold.Store, repo string, keys []string, version string) (
if exact != nil {
return exact, nil
}
prefixPattern := "^" + regexp.QuoteMeta(prefix)
re, err := regexp.Compile(prefixPattern)
if err != nil {
continue
}
if err := db.FindOne(cache,
bolthold.Where("Repo").Eq(repo).Index("Repo").
And("Key").MatchFunc(func(key string) (bool, error) { return strings.HasPrefix(key, prefix), nil }).
bolthold.Where("Repo").Eq(repo).
And("Key").RegExp(re).
And("Version").Eq(version).
And("Complete").Eq(true).
SortBy("CreatedAt").Reverse()); err != nil {
@@ -952,8 +805,8 @@ func findExactCache(db *bolthold.Store, repo, key, version string, complete bool
}
cache := &Cache{}
err := db.FindOne(cache,
bolthold.Where("Key").Eq(key).Index("Key").
And("Repo").Eq(repo).
bolthold.Where("Repo").Eq(repo).
And("Key").Eq(key).
And("Version").Eq(version).
And("Complete").Eq(complete).
SortBy(sortBy).Reverse())
@@ -966,23 +819,15 @@ func findExactCache(db *bolthold.Store, repo, key, version string, complete bool
return cache, nil
}
func (h *Handler) insertCache(db *bolthold.Store, cache *Cache) error {
if h.s3 != nil {
var id [8]byte
_, _ = rand.Read(id[:])
cache.ID = binary.BigEndian.Uint64(id[:]) >> 11 // IDs must be unique across runners, and v1 clients parse an ID as a JS number
return db.Insert(cache.ID, cache)
func insertCache(db *bolthold.Store, cache *Cache) error {
if err := db.Insert(bolthold.NextSequence(), cache); err != nil {
return fmt.Errorf("insert cache: %w", err)
}
return db.Bolt().Update(func(tx *bbolt.Tx) error {
if err := db.TxInsert(tx, bolthold.NextSequence(), cache); err != nil {
return fmt.Errorf("insert cache: %w", err)
}
// write back id to db
if err := db.TxUpdate(tx, cache.ID, cache); err != nil {
return fmt.Errorf("write back id to db: %w", err)
}
return nil
})
// write back id to db
if err := db.Update(cache.ID, cache); err != nil {
return fmt.Errorf("write back id to db: %w", err)
}
return nil
}
// touchCache stamps UsedAt so gcCache does not reap an entry mid-upload. With requireIncomplete
@@ -1007,9 +852,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
if requireIncomplete && cache.Complete {
return fmt.Errorf("cache %d: already complete", id)
}
if !touchNeeded(cache) {
return nil
}
cache.UsedAt = time.Now().Unix()
return db.Update(cache.ID, cache)
}
@@ -1019,19 +861,15 @@ const (
defaultSweepInterval = time.Hour
touchStale = time.Minute // how stale a completed entry's UsedAt may get before an access rewrites it, sparing hits the fsync
// inUseGrace covers artifactURLTTL plus the touchStale lag so an entry outlives every signed URL
// still usable for it, and no sweep cuts off a download in progress.
inUseGrace = artifactURLTTL + touchStale
// inUseGrace matches artifactURLTTL so an entry outlives every signed URL still usable
// for it, and no sweep cuts off a download in progress.
inUseGrace = artifactURLTTL
// uploadStallTimeout is how long a reservation may sit without a chunk before it counts
// as abandoned. Widening it also widens the window for findExactCache to hand a finalize
// a stale reservation.
uploadStallTimeout = 5 * time.Minute
metadataSyncInterval = 30 * time.Second
defaultMinFreeDisk = 1024 * miB
)
@@ -1079,9 +917,6 @@ func (h *Handler) gcCache() {
defer db.Close()
h.evictIncomplete(db)
if h.s3 != nil {
return // lifecycle rules on the bucket expire its entries
}
h.evictExpired(db)
h.evictSuperseded(db)
h.evictOversized(db)
@@ -1122,7 +957,7 @@ func (h *Handler) evictExpired(db *bolthold.Store) {
return
}
// Never below inUseGrace, or a short retention would outrun a signed URL already issued.
window := max(h.policy.Retention+touchStale, inUseGrace)
window := max(h.policy.Retention, inUseGrace)
h.sweep(db, bolthold.Where("UsedAt").Lt(time.Now().Add(-window).Unix()).Index("UsedAt"))
}
@@ -1223,18 +1058,10 @@ func inUse(cache *Cache) bool {
return time.Since(time.Unix(cache.UsedAt, 0)) < inUseGrace
}
// touchNeeded skips a write that would not change the second, and completed entries fresher than touchStale.
func touchNeeded(cache *Cache) bool {
return cache.UsedAt < time.Now().Unix() && (!cache.Complete || time.Since(time.Unix(cache.UsedAt, 0)) >= touchStale)
}
// touch stamps UsedAt through the caller's store, a bolt write on the read path. It cannot
// go through touchCache, which opens its own store and would block on the exclusive lock
// for as long as the caller holds one.
func (h *Handler) touch(db *bolthold.Store, cache *Cache) {
if !touchNeeded(cache) {
return
}
cache.UsedAt = time.Now().Unix()
if err := db.Update(cache.ID, cache); err != nil {
h.logger.Warnf("touch cache: %v", err)
@@ -1279,11 +1106,7 @@ func totalSize(caches []*Cache) int64 {
// deleteCache drops an entry and its bytes, reporting whether it went fully. The blob goes
// first, so a failed unlink leaves the row for the next sweep instead of orphaning bytes.
func (h *Handler) deleteCache(db *bolthold.Store, cache *Cache) bool {
storage := h.storage
if h.s3 != nil && !cache.Complete {
storage = h.s3.Storage // an upload not yet committed has nothing in the bucket
}
if err := storage.Remove(cache.ID); err != nil {
if err := h.storage.Remove(cache.ID); err != nil {
h.logger.Warnf("remove cache blob: %v", err)
return false
}
+15 -96
View File
@@ -12,7 +12,6 @@ import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -53,59 +52,6 @@ func signArtifactURL(h *Handler, id int64) string {
return h.signedArtifactURL(JobCredential{}, uint64(id), time.Now().Add(artifactURLTTL))
}
func TestHandler_ExternalURL(t *testing.T) {
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
}
func TestHandler_Upstream(t *testing.T) {
var upstream *httptest.Server
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPut, r.Method)
assert.Equal(t, "/cache%20server"+blobPath+"/%34%32", r.URL.EscapedPath())
assert.Equal(t, "sig=opaque%2Fvalue&raw=a;b", r.URL.RawQuery)
assert.Equal(t, strings.TrimPrefix(upstream.URL, "http://"), r.Host)
assert.Equal(t, "Bearer unregistered", r.Header.Get("Authorization"))
_, err := io.Copy(w, r.Body)
assert.NoError(t, err)
}))
t.Cleanup(upstream.Close)
dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{
Dir: dir,
OutboundIP: "127.0.0.1",
Upstream: upstream.URL + "/cache%20server//",
})
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, handler.Close()) })
req, err := http.NewRequestWithContext(t.Context(), http.MethodPut,
handler.ExternalURL()+blobPath+"/%34%32?sig=opaque%2Fvalue&raw=a;b", strings.NewReader("cache content"))
require.NoError(t, err)
req.Header.Set("Authorization", "Bearer unregistered")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
forwarded, err := io.ReadAll(resp.Body)
resp.Body.Close()
require.NoError(t, err)
assert.Equal(t, "cache content", string(forwarded))
resp, err = http.Post(handler.ExternalURL()+internalPath+"/register", "application/json", nil)
require.NoError(t, err)
resp.Body.Close()
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
upstream.Close()
resp, err = http.Post(handler.ExternalURL()+cacheServiceV2Path+"/CreateCacheEntry", "application/json", strings.NewReader(`{}`))
require.NoError(t, err)
var failure map[string]string
err = json.UnmarshalRead(resp.Body, &failure)
resp.Body.Close()
require.NoError(t, err)
assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
assert.Equal(t, map[string]string{"code": "unavailable", "msg": "cache upstream is unavailable"}, failure)
assert.NoDirExists(t, dir)
}
func TestHandler(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(Options{Dir: dir})
@@ -821,7 +767,7 @@ func TestHandler_gcCache(t *testing.T) {
Key: "test_key_3",
Version: "test_version",
Complete: true,
UsedAt: now.Add(-(testRetention + touchStale + time.Second)).Unix(),
UsedAt: now.Add(-(testRetention + time.Second)).Unix(),
CreatedAt: now.Add(-(testRetention + time.Hour)).Unix(),
},
Kept: false,
@@ -864,7 +810,7 @@ func TestHandler_gcCache(t *testing.T) {
db, err := handler.openDB()
require.NoError(t, err)
for _, c := range cases {
require.NoError(t, handler.insertCache(db, c.Cache))
require.NoError(t, insertCache(db, c.Cache))
}
require.NoError(t, db.Close())
@@ -1036,7 +982,7 @@ func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
// A non-empty directory where the blob belongs makes os.Remove fail on every platform.
blob := handler.storage.(*Storage).filename(cache.ID)
blob := handler.storage.filename(cache.ID)
require.NoError(t, os.MkdirAll(blob, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(blob, "held"), []byte("x"), 0o600))
@@ -1096,51 +1042,27 @@ func TestHandler_gcCacheInterval(t *testing.T) {
assert.Empty(t, keptKeys(t, handler, []*Cache{cache}))
}
func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
fresh := &Cache{Repo: testRepo, Key: "fresh", Version: "v", Complete: true, UsedAt: time.Now().Unix()}
pending := &Cache{Repo: testRepo, Key: "pending", Version: "v", UsedAt: time.Now().Add(-time.Second).Unix()}
handler := newTestHandler(t, Policy{}, fresh, pending)
txID := func() int {
db, err := handler.openDB()
require.NoError(t, err)
defer db.Close()
var id int
require.NoError(t, db.Bolt().View(func(tx *bbolt.Tx) error { id = tx.ID(); return nil }))
return id
}
before := txID()
require.NoError(t, handler.touchCache(fresh.ID, false))
assert.Equal(t, before, txID())
require.NoError(t, handler.touchCache(pending.ID, false))
assert.Greater(t, txID(), before)
}
// newTestHandler starts a handler with testToken registered, seeded with entries.
func newTestHandler(t *testing.T, policy Policy, entries ...*Cache) *Handler {
t.Helper()
handler := startTestHandler(t, Options{Policy: policy})
handler, err := StartHandler(Options{
Dir: filepath.Join(t.TempDir(), "artifactcache"),
OutboundIP: "127.0.0.1",
Policy: policy,
})
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, handler.Close()) })
handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
db, err := handler.openDB()
require.NoError(t, err)
for _, e := range entries {
require.NoError(t, handler.insertCache(db, e))
require.NoError(t, insertCache(db, e))
}
require.NoError(t, db.Close())
return handler
}
func startTestHandler(t *testing.T, opts Options) *Handler {
t.Helper()
opts.Dir = filepath.Join(t.TempDir(), "artifactcache")
opts.OutboundIP = "127.0.0.1"
handler, err := StartHandler(opts)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, handler.Close()) })
handler.RegisterJob(testToken, JobCredential{Repo: testRepo})
return handler
}
// keptKeys reports which of entries are still in the store.
func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
t.Helper()
@@ -1481,8 +1403,8 @@ func TestHandler_GC_PerRepoDedup(t *testing.T) {
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
require.NoError(t, handler.insertCache(db, a))
require.NoError(t, handler.insertCache(db, b))
require.NoError(t, insertCache(db, a))
require.NoError(t, insertCache(db, b))
// Write the backing blobs so the dedup deletion has something to remove.
require.NoError(t, handler.storage.Write(a.ID, 0, strings.NewReader("a")))
_, err = handler.storage.Commit(a.ID, 1)
@@ -1576,10 +1498,7 @@ func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
}
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
cred, ok := handler.lookupCredential("via-internal-api")
require.True(t, ok)
assert.Equal(t, JobCredential{Repo: testRepo, PublicURL: "http://runner.example"}, cred)
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo"}`))
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
+13 -47
View File
@@ -39,9 +39,8 @@ const (
blobUploadURLTTL = time.Hour
twirpInternal = "internal"
twirpUnavailable = "unavailable"
twirpUnauthenticated = "unauthenticated"
// twirpInternal is the only error code that is not the client's fault.
twirpInternal = "internal"
)
func (h *Handler) registerV2Routes(router *httprouter.Router) {
@@ -57,7 +56,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2CreateRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
if req.Key == "" || req.Version == "" {
@@ -84,35 +83,16 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
return
}
// A second live reservation is finalized by whichever job calls last, against the other's upload.
owner := hashedToken(bearerToken(r))
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
if pending.Owner != owner {
h.twirpNotOK(w, r)
return
}
h.touch(db, pending) // still uploading, so it must not go stale under the sweep
h.responseJSON(w, r, http.StatusOK, map[string]any{ // this job retrying its own reservation
"ok": true,
"signed_upload_url": h.signedURL(cred, blobPath, blobUploadPurpose, pending.ID, time.Now().Add(blobUploadURLTTL)),
})
return
}
now := time.Now().Unix()
cache := &Cache{
Repo: cred.Repo,
Key: req.Key,
Version: req.Version,
Owner: owner,
Size: -1, // the size is only known at finalize time
CreatedAt: now,
UsedAt: now,
}
if err := h.insertCache(db, cache); err != nil {
if err := insertCache(db, cache); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
}
@@ -127,7 +107,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -139,13 +119,6 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
defer db.Close()
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
cache = nil // not the reservation this job made, so not this job's to commit
}
if err == nil && cache == nil {
// A retry whose first response was lost finds the entry already committed.
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
}
if err != nil {
h.twirpError(w, r, twirpInternal, err)
return
@@ -154,15 +127,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
h.twirpNotOK(w, r)
return
}
db.Close() // commitCache needs the store closed
if !cache.Complete {
db.Close() // commitCache needs the store closed
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r)
return
}
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r)
return
}
h.responseJSON(w, r, http.StatusOK, map[string]any{
@@ -176,7 +147,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2DownloadRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -259,13 +230,8 @@ func (h *Handler) twirpNotOK(w http.ResponseWriter, r *http.Request) {
func (h *Handler) twirpError(w http.ResponseWriter, r *http.Request, code string, err error) {
h.logger.Debugf("%s %s: %v", r.Method, r.URL.Path, err)
status := http.StatusBadRequest
switch code {
case twirpInternal:
if code == twirpInternal {
status = http.StatusInternalServerError
case twirpUnavailable:
status = http.StatusServiceUnavailable
case twirpUnauthenticated:
status = http.StatusUnauthorized
}
h.responseJSON(w, r, status, map[string]any{"code": code, "msg": err.Error()})
}
-110
View File
@@ -10,7 +10,6 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
@@ -142,17 +141,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
}
list.WriteString(`</BlockList>`)
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()))
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()),
"a client that lost the first answer retries the list it already sent")
var reordered bytes.Buffer
reordered.WriteString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
for _, blockID := range []string{order[1], order[0], order[2]} {
fmt.Fprintf(&reordered, "<Latest>%s</Latest>", blockID)
}
reordered.WriteString(`</BlockList>`)
assert.Equal(t, http.StatusInternalServerError, putBlob(t, uploadURL+"&comp=blocklist", reordered.Bytes()),
"the blocks are already assembled, so a different order would silently disagree with them")
finalized := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "blocks", "version": "v1", "size_bytes": len("hello world!"),
@@ -253,101 +241,3 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
}
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
handler := newTestHandler(t, Policy{})
tests := []struct {
name string
token string
body string
status int
code string
}{
{"no bearer at all", "", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a bearer nobody registered", "not-a-job", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a body that is not the request", testToken, `{`, http.StatusBadRequest, "malformed"},
{"a request missing its key", testToken, `{"version":"v"}`, http.StatusBadRequest, "invalid_argument"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
handler.ExternalURL()+cacheServiceV2Path+"/CreateCacheEntry", strings.NewReader(tt.body))
require.NoError(t, err)
if tt.token != "" {
req.Header.Set("Authorization", "Bearer "+tt.token)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.status, resp.StatusCode)
got := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got))
assert.Equal(t, tt.code, got["code"])
assert.NotEmpty(t, got["msg"])
})
}
}
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
handler := newTestHandler(t, Policy{})
content := []byte("the cached archive")
const otherJob = "another-jobs-token"
defer handler.RegisterJob(otherJob, JobCredential{Repo: testRepo})()
reserve := func(client *http.Client) map[string]any {
return v2Call(t, handler, client, "CreateCacheEntry", map[string]any{"key": "shared", "version": "v1"})
}
first := reserve(testClient)
require.Equal(t, true, first["ok"])
retry := reserve(testClient)
assert.Equal(t, true, retry["ok"], "a job retrying its own reservation is not a conflict")
entryOf := func(reserved map[string]any) string {
parsed, err := url.Parse(reserved["signed_upload_url"].(string))
require.NoError(t, err)
return parsed.Path
}
assert.Equal(t, entryOf(first), entryOf(retry), "the same reservation, however freshly its URL is signed")
assert.Equal(t, false, reserve(&http.Client{Transport: &bearerTransport{token: otherJob}})["ok"],
"another job's reservation would be finalized against the first upload")
otherClient := &http.Client{Transport: &bearerTransport{token: otherJob}}
reserved := v2Call(t, handler, otherClient, "CreateCacheEntry", map[string]any{"key": "owned", "version": "v1"})
require.Equal(t, true, reserved["ok"])
require.Equal(t, http.StatusCreated, putBlob(t, reserved["signed_upload_url"].(string), content))
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "owned", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"another job's upload is not this job's to finalize")
base := handler.ExternalURL() + apiPath
reserveV1, err := json.Marshal(&Request{Key: "v1-made", Version: "v1", Size: int64(len(content))})
require.NoError(t, err)
resp, err := testClient.Post(base+"/caches", "application/json", bytes.NewReader(reserveV1))
require.NoError(t, err)
var madeByV1 struct {
CacheID uint64 `json:"cacheId"`
}
require.NoError(t, json.UnmarshalRead(resp.Body, &madeByV1))
resp.Body.Close()
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
fmt.Sprintf("%s/caches/%d", base, madeByV1.CacheID), bytes.NewReader(content))
require.NoError(t, err)
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
resp, err = testClient.Do(upload)
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "v1-made", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"a reservation this job did not make through v2 is not its to commit")
finalized, _ := saveV2(t, handler, "deps", "v1", content)
require.Equal(t, true, finalized["ok"])
again := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "deps", "version": "v1", "size_bytes": strconv.Itoa(len(content)),
})
assert.Equal(t, true, again["ok"], "a retry whose first answer was lost must not report a failed save")
assert.Equal(t, finalized["entry_id"], again["entry_id"])
}
-1
View File
@@ -16,7 +16,6 @@ type Cache struct {
Key string `json:"key" boltholdIndex:"Key"`
Version string `json:"version" boltholdIndex:"Version"`
Size int64 `json:"cacheSize"`
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
Complete bool `json:"complete" boltholdIndex:"Complete"`
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
+9 -45
View File
@@ -5,8 +5,6 @@ package artifactcache
import (
"crypto/tls"
"errors"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
@@ -17,38 +15,23 @@ import (
// Gitea implements only the artifact half of it. Forwarding that half from here makes this origin
// the whole service, so ACTIONS_RESULTS_URL can point at it truthfully, which is what the clients
// this runner cannot patch need, docker buildx among them.
//
// The instance to forward to travels with the job registration rather than with configuration, so
// a cache server shared between runners serves each of their instances.
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService/"
// forwardOrNotFound is the router's fallback: the artifact service of the instance the job
// registered with, and the 404 the router would have written otherwise.
func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, artifactServicePath) {
cred, ok := h.lookupCredential(bearerToken(r))
if !ok || cred.Results == "" || !strings.HasPrefix(r.URL.Path, artifactServicePath) {
http.NotFound(w, r)
return
}
cred, ok := h.lookupCredential(bearerToken(r))
if !ok {
h.twirpError(w, r, twirpUnauthenticated, errors.New("unknown bearer token"))
return
}
if cred.Results == "" {
h.twirpError(w, r, twirpInternal, errors.New("no instance is registered for this job"))
return
}
target, err := url.Parse(strings.TrimSuffix(cred.Results, "/"))
if err == nil && (target.Hostname() == "" || (target.Scheme != "http" && target.Scheme != "https")) {
err = errors.New("not an absolute http or https URL the job could use")
}
if err != nil {
h.logger.Errorf("artifact service forward to %q: %v", cred.Results, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("artifact service address %q is unusable: %w", cred.Results, err))
return
}
if !h.mustProxy(r, cred, target) {
redirect := target.JoinPath(r.URL.EscapedPath())
redirect.RawQuery = mergeQuery(target.RawQuery, r.URL.RawQuery)
h.logger.Debugf("%s %s: redirecting to %s", r.Method, r.URL.Path, target)
http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect)
w.WriteHeader(http.StatusBadGateway)
return
}
h.logger.Debugf("%s %s: forwarding to %s", r.Method, r.URL.Path, target)
@@ -61,9 +44,9 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
// directly and never through here.
r.Out.Host = target.Host
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
h.logger.Warnf("artifact service forward to %s: %v", target, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
w.WriteHeader(http.StatusBadGateway)
},
}
if cred.InsecureTLS {
@@ -72,24 +55,5 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
proxy.ServeHTTP(w, r)
}
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
if cred.InsecureTLS && target.Scheme == "https" {
return true
}
scheme := r.Header.Get("X-Forwarded-Proto")
if scheme == "" {
scheme = "http"
}
return scheme != target.Scheme
}
func mergeQuery(target, request string) string {
if target == "" || request == "" {
return target + request
}
return target + "&" + request
}
// insecureTransport is shared, because a transport per request would pool no connections.
var insecureTransport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
var insecureTransport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
+16 -94
View File
@@ -4,8 +4,6 @@
package artifactcache
import (
"cmp"
"encoding/json/v2"
"io"
"net/http"
"net/http/httptest"
@@ -16,6 +14,8 @@ import (
"github.com/stretchr/testify/require"
)
// The artifact half is forwarded under the Host Gitea knows itself by, so the URLs it hands back
// still point at Gitea, and nothing else is proxied.
func TestFrontResultsService(t *testing.T) {
var gotHost, gotPath, gotProto string
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -23,112 +23,34 @@ func TestFrontResultsService(t *testing.T) {
_, _ = io.WriteString(w, `{"ok":true}`)
}))
defer gitea.Close()
selfSigned := httptest.NewTLSServer(gitea.Config.Handler)
defer selfSigned.Close()
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"})
require.NoError(t, err)
defer handler.Close()
const token, rpc = "forward-token", artifactServicePath + "CreateArtifact"
const token = "forward-token"
noRedirect := func(token string) *http.Client {
return &http.Client{
Transport: &bearerTransport{token: token},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
}
post := func(path string) (int, map[string]string) {
client := &http.Client{Transport: &bearerTransport{token: token}}
post := func(path string) int {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+path, nil)
require.NoError(t, err)
resp, err := noRedirect(token).Do(req)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := map[string]string{}
_ = json.UnmarshalRead(resp.Body, &body)
return resp.StatusCode, body
resp.Body.Close()
return resp.StatusCode
}
status, body := post(rpc)
assert.Equal(t, http.StatusUnauthorized, status, "an unregistered token is told so, not left to parse a 404 page")
assert.Equal(t, twirpUnauthenticated, body["code"])
assert.Contains(t, body["msg"], "bearer")
assert.NotEmpty(t, body["msg"])
assert.Equal(t, http.StatusNotFound, post(artifactServicePath+"CreateArtifact"),
"an unregistered token is forwarded nowhere")
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: selfSigned.URL, InsecureTLS: true})()
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: gitea.URL})()
status, _ = post(rpc)
require.Equal(t, http.StatusOK, status, "an http job reaching an https instance must proxy")
assert.Equal(t, strings.TrimPrefix(selfSigned.URL, "https://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Equal(t, http.StatusOK, post(artifactServicePath+"CreateArtifact"))
assert.Equal(t, strings.TrimPrefix(gitea.URL, "http://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Empty(t, gotProto, "a forwarded scheme would make an https Gitea mint http URLs")
assert.Equal(t, rpc, gotPath)
assert.Equal(t, artifactServicePath+"CreateArtifact", gotPath)
gotPath = ""
status, _ = post("/twirp/github.actions.results.api.v1.OtherService/Do")
assert.Equal(t, http.StatusNotFound, status)
status, _ = post("/api/v1/repos/owner/repo")
assert.Equal(t, http.StatusNotFound, status)
assert.Equal(t, http.StatusNotFound, post("/twirp/github.actions.results.api.v1.OtherService/Do"))
assert.Equal(t, http.StatusNotFound, post("/api/v1/repos/owner/repo"))
assert.Empty(t, gotPath, "only the artifact service is forwarded")
tests := []struct {
name string
cred JobCredential
forwardedProto string
reqQuery string
want int
location string
wantMsg string
}{
{name: "an instance the job reaches itself", cred: JobCredential{Results: gitea.URL}, want: http.StatusTemporaryRedirect},
{name: "a sub-path instance keeps its prefix", cred: JobCredential{Results: gitea.URL + "/sub"}, want: http.StatusTemporaryRedirect},
{name: "an instance with no certificate to distrust", cred: JobCredential{Results: gitea.URL, InsecureTLS: true}, want: http.StatusTemporaryRedirect},
{
name: "both queries survive", cred: JobCredential{Results: gitea.URL + "?tenant=one"}, reqQuery: "?run=7",
want: http.StatusTemporaryRedirect, location: gitea.URL + rpc + "?tenant=one&run=7",
},
{
name: "a trusted https instance is reached by the job itself",
cred: JobCredential{Results: selfSigned.URL}, forwardedProto: "https", want: http.StatusTemporaryRedirect,
},
{
name: "the job does not share this server's disregard for the certificate",
cred: JobCredential{Results: selfSigned.URL, InsecureTLS: true}, forwardedProto: "https", want: http.StatusOK,
},
{name: "an http job is not redirected to an https instance", cred: JobCredential{Results: selfSigned.URL}, want: http.StatusInternalServerError},
{
name: "a forwarded https scheme onto an http instance is proxied",
cred: JobCredential{Results: gitea.URL}, forwardedProto: "https", want: http.StatusOK,
},
{
name: "a scheme the job could not use is refused before it is sent one",
cred: JobCredential{Results: "ftp://gitea.example"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
{
name: "an address with no host is refused before it is sent one",
cred: JobCredential{Results: "http://:3000"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer handler.RegisterJob(tt.name, tt.cred)()
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+rpc+tt.reqQuery, nil)
require.NoError(t, err)
if tt.forwardedProto != "" {
req.Header.Set("X-Forwarded-Proto", tt.forwardedProto)
}
resp, err := noRedirect(tt.name).Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.want, resp.StatusCode)
switch tt.want {
case http.StatusTemporaryRedirect:
assert.Equal(t, cmp.Or(tt.location, tt.cred.Results+rpc), resp.Header.Get("Location"))
case http.StatusInternalServerError:
body := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &body))
assert.Equal(t, twirpInternal, body["code"])
assert.Contains(t, body["msg"], cmp.Or(tt.wantMsg, tt.cred.Results))
}
})
}
}
-326
View File
@@ -1,326 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"bytes"
"cmp"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/json/v2"
"encoding/xml"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net/http"
"net/url"
"os"
"path"
"slices"
"strconv"
"strings"
"time"
)
// S3Options configures the bucket that stores completed cache entries.
type S3Options struct {
Endpoint string
Region string
Bucket string
Prefix string
PathStyle *bool // nil uses virtual-hosted requests on AWS and path-style ones elsewhere
AccessKey string
SecretKey string
}
type cacheStorage interface {
Exist(uint64) (bool, error)
Write(uint64, int64, io.Reader) error
WriteBlock(uint64, string, io.Reader) error
OrderBlocks(uint64, []string) error
Commit(uint64, int64) (int64, error)
Serve(http.ResponseWriter, *http.Request, uint64)
Remove(uint64) error
}
const s3PartSize = 5 << 30 // the largest part S3 accepts
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
type s3Storage struct {
*Storage
client *http.Client
bucketURL *url.URL
opts S3Options
}
type s3Part struct {
PartNumber int
ETag string
}
func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
endpoint, err := url.Parse(opts.Endpoint)
if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") {
return nil, fmt.Errorf("invalid S3 endpoint %q", opts.Endpoint)
}
transport, _ := http.DefaultTransport.(*http.Transport)
transport = transport.Clone()
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
if opts.PathStyle != nil {
pathStyle = *opts.PathStyle
}
if pathStyle {
endpoint.Path = "/" + opts.Bucket
} else {
endpoint.Host = opts.Bucket + "." + endpoint.Host
endpoint.Path = "/"
}
opts.Region = cmp.Or(opts.Region, "us-east-1")
return &s3Storage{Storage: staging, client: &http.Client{Transport: transport}, bucketURL: endpoint, opts: opts}, nil
}
func (s *s3Storage) blobKey(id uint64) string {
return path.Join(s.opts.Prefix, "blobs", strconv.FormatUint(id, 10))
}
func (s *s3Storage) metadataDir() string {
return path.Join(s.opts.Prefix, "metadata") + "/"
}
func (s *s3Storage) metadataKey(id uint64) string {
return s.metadataDir() + strconv.FormatUint(id, 10) + ".json"
}
func (s *s3Storage) url(key string, query url.Values) *url.URL {
u := *s.bucketURL
u.Path = path.Join(u.Path, key)
u.RawPath = strings.NewReplacer("+", "%20", "%2F", "/").Replace(url.QueryEscape(u.Path)) // SigV4 encoding
u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20")
return &u
}
// sign authorizes req with AWS Signature Version 4, leaving the payload unsigned so bodies can stream.
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
func (s *s3Storage) sign(req *http.Request, now time.Time) {
date := now.UTC().Format("20060102T150405Z")
req.Header.Set("X-Amz-Date", date)
if req.Header.Get("X-Amz-Content-Sha256") == "" {
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
}
headers := map[string]string{"host": req.URL.Host}
for name, values := range req.Header {
if name = strings.ToLower(name); strings.HasPrefix(name, "x-amz-") {
headers[name] = strings.Join(values, ",")
}
}
names := slices.Sorted(maps.Keys(headers))
canonical := []string{req.Method, req.URL.EscapedPath(), req.URL.RawQuery}
for _, name := range names {
canonical = append(canonical, name+":"+headers[name])
}
canonical = append(canonical, "", strings.Join(names, ";"), req.Header.Get("X-Amz-Content-Sha256"))
scope := date[:8] + "/" + s.opts.Region + "/s3/aws4_request"
signature := []byte("AWS4" + s.opts.SecretKey)
for _, data := range []string{date[:8], s.opts.Region, "s3", "aws4_request", fmt.Sprintf("AWS4-HMAC-SHA256\n%s\n%s\n%x", date, scope, sha256.Sum256([]byte(strings.Join(canonical, "\n"))))} { // key derivation, the last round signs
mac := hmac.New(sha256.New, signature)
_, _ = mac.Write([]byte(data))
signature = mac.Sum(nil)
}
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%x", s.opts.AccessKey, scope, strings.Join(names, ";"), signature))
}
// do returns the response to a signed request when it succeeded, fs.ErrNotExist for a missing key.
func (s *s3Storage) do(ctx context.Context, method, key string, query url.Values, header http.Header, body io.Reader) (*http.Response, error) {
var size int64
if sized, ok := body.(interface{ Size() int64 }); ok {
size = sized.Size()
}
if size == 0 {
body = nil
}
req, err := http.NewRequestWithContext(ctx, method, s.url(key, query).String(), body)
if err != nil {
return nil, err
}
req.ContentLength = size
maps.Copy(req.Header, header)
s.sign(req, time.Now())
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < http.StatusMultipleChoices {
return resp, nil
}
defer drain(resp)
var failure struct{ Code, Message string }
_ = xml.NewDecoder(resp.Body).Decode(&failure)
if resp.StatusCode == http.StatusNotFound && (failure.Code == "NoSuchKey" || failure.Code == "" && method == http.MethodHead) {
return nil, fs.ErrNotExist
}
return nil, fmt.Errorf("S3 %s %s: %s", method, key, cmp.Or(strings.TrimSpace(failure.Code+" "+failure.Message), resp.Status))
}
func (s *s3Storage) call(ctx context.Context, method, key string, query url.Values, body io.Reader, result any) (http.Header, error) {
resp, err := s.do(ctx, method, key, query, nil, body)
if err != nil {
return nil, err
}
defer drain(resp)
if result == nil {
return resp.Header, nil
}
return resp.Header, xml.NewDecoder(resp.Body).Decode(result)
}
// drain reads a response to its end before closing it, so its connection gets reused.
func drain(resp *http.Response) {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
func (s *s3Storage) Exist(id uint64) (bool, error) {
_, err := s.call(context.Background(), http.MethodHead, s.blobKey(id), nil, nil, nil)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return err == nil, err
}
func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
written, err := s.Storage.Commit(id, size)
if err != nil {
return 0, err
}
defer func() { _ = s.Storage.Remove(id) }()
file, err := os.Open(s.filename(id))
if err != nil {
return 0, err
}
defer file.Close()
if err := s.upload(context.Background(), s.blobKey(id), file, written); err != nil {
return 0, fmt.Errorf("upload cache to S3: %w", err)
}
return written, nil
}
// upload always goes in parts, the only way S3 takes an object over 5 GiB.
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) error {
var created struct {
UploadID string `xml:"UploadId"`
}
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
return err
}
var completion struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []s3Part `xml:"Part"`
}
for offset := int64(0); offset == 0 || offset < size; offset += s3PartSize {
number, length := len(completion.Parts)+1, min(s3PartSize, size-offset)
query := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {created.UploadID}}
header, err := s.call(ctx, http.MethodPut, key, query, io.NewSectionReader(file, offset, length), nil)
if err != nil {
return err
}
completion.Parts = append(completion.Parts, s3Part{number, header.Get("ETag")})
}
body, err := xml.Marshal(completion)
if err != nil {
return err
}
var result struct{ Code, Message string } // a completion can fail after its 200 status
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploadId": {created.UploadID}}, bytes.NewReader(body), &result); err != nil {
return err
}
if result.Code != "" {
return fmt.Errorf("S3 complete upload %s: %s %s", key, result.Code, result.Message)
}
return nil
}
func (s *s3Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
header := http.Header{}
if byteRange := r.Header.Get("Range"); byteRange != "" {
header.Set("Range", byteRange)
}
resp, err := s.do(r.Context(), http.MethodGet, s.blobKey(id), nil, header, nil)
if errors.Is(err, fs.ErrNotExist) {
http.NotFound(w, r)
return
} else if err != nil {
http.Error(w, "read cache from S3", http.StatusBadGateway)
return
}
defer resp.Body.Close()
for _, name := range []string{"Content-Length", "Content-Range"} {
if value := resp.Header.Get(name); value != "" {
w.Header().Set(name, value)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
// Remove unpublishes an entry before deleting its blob, so no runner imports it without its bytes.
func (s *s3Storage) Remove(id uint64) error {
for _, key := range []string{s.metadataKey(id), s.blobKey(id)} {
if _, err := s.call(context.Background(), http.MethodDelete, key, nil, nil, nil); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove %s from S3: %w", key, err)
}
}
return s.Storage.Remove(id)
}
func (s *s3Storage) publish(cache *Cache) error {
data, err := json.Marshal(cache)
if err != nil {
return err
}
if _, err := s.call(context.Background(), http.MethodPut, s.metadataKey(cache.ID), nil, bytes.NewReader(data), nil); err != nil {
return fmt.Errorf("publish cache to S3: %w", err)
}
return nil
}
func (s *s3Storage) published(ctx context.Context) (map[uint64]bool, error) {
ids := make(map[uint64]bool)
query := url.Values{"list-type": {"2"}, "prefix": {s.metadataDir()}}
for {
var page struct {
Contents []struct{ Key string }
IsTruncated bool
NextContinuationToken string
}
if _, err := s.call(ctx, http.MethodGet, "", query, nil, &page); err != nil {
return nil, fmt.Errorf("list S3 cache metadata: %w", err)
}
for _, object := range page.Contents {
if id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(object.Key, s.metadataDir()), ".json"), 10, 64); err == nil {
ids[id] = true
}
}
if !page.IsTruncated {
return ids, nil
}
query.Set("continuation-token", page.NextContinuationToken)
}
}
func (s *s3Storage) fetch(ctx context.Context, id uint64) (*Cache, error) {
resp, err := s.do(ctx, http.MethodGet, s.metadataKey(id), nil, nil, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
cache := &Cache{}
if err := json.UnmarshalRead(resp.Body, cache); err != nil {
return nil, fmt.Errorf("read S3 cache metadata %d: %w", id, err)
}
return cache, nil
}
-46
View File
@@ -1,46 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
backend := s3mem.New()
require.NoError(t, backend.CreateBucket("cache"))
server := httptest.NewServer(gofakes3.New(backend).Server())
t.Cleanup(server.Close)
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
entry := map[string]any{"key": "deps", "version": "v1"}
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
require.Equal(t, true, finalized["ok"])
require.NoError(t, restorer.importMetadata(t.Context()))
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
require.Equal(t, true, found["ok"])
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
require.NoError(t, backend.ForceDeleteBucket("cache"))
require.NoError(t, backend.CreateBucket("cache"))
require.NoError(t, restorer.importMetadata(t.Context()))
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
}
func TestS3StorageSignsLikeAWS(t *testing.T) {
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
storage.sign(req, time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
assert.Equal(t, "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", req.Header.Get("Authorization"))
}
+1 -15
View File
@@ -5,10 +5,8 @@
package artifactcache
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json/v2"
"fmt"
"io"
"net/http"
@@ -68,18 +66,6 @@ func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error
// rename pass is safe because a staged name always carries blockFilePrefix and a target name
// never does, so no rename can collide with a block not yet moved.
func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
wanted, err := json.Marshal(blockIDs) // a block id is client-chosen, so it cannot be a separator
if err != nil {
return err
}
list := filepath.Join(s.tempDir(id), blockFilePrefix+"list")
if recorded, err := os.ReadFile(list); err == nil {
// A retry repeats the list it already sent; another one would reorder what is now staged.
if !bytes.Equal(recorded, wanted) {
return fmt.Errorf("cache %d was already assembled from a different block list", id)
}
return nil
}
for i, blockID := range blockIDs {
if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
if os.IsNotExist(err) {
@@ -88,7 +74,7 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
return err
}
}
return os.WriteFile(list, wanted, 0o600)
return nil
}
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
+3 -5
View File
@@ -11,7 +11,6 @@ import (
"fmt"
"io"
"io/fs"
"net"
"net/http"
"os"
"path/filepath"
@@ -221,17 +220,16 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
downloads(router, artifactPath)
server := &http.Server{
Addr: net.JoinHostPort(addr, port),
Addr: fmt.Sprintf("%s:%s", addr, port),
ReadHeaderTimeout: 2 * time.Second,
Handler: router,
}
// run server
go func() {
logger.Infof("Start server on http://%s", server.Addr)
logger.Infof("Start server on http://%s:%s", addr, port)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error(err)
os.Exit(1)
logger.Fatal(err)
}
}()
+25 -18
View File
@@ -4,15 +4,15 @@
package common
import (
"context"
"errors"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/sirupsen/logrus"
)
func TestDryrunContext(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
if Dryrun(ctx) {
t.Fatal("plain context should not be dryrun")
}
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
}
func TestJobErrorContainer(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
err := errors.New("job failed")
SetJobError(ctx, err)
@@ -40,27 +40,34 @@ func TestJobErrorContainer(t *testing.T) {
}
}
func TestLoggerAndHandlerContext(t *testing.T) {
ctx := t.Context()
if Logger(ctx) != logging.Default() {
t.Fatal("plain context should use the default logger")
func TestLoggerAndHookContext(t *testing.T) {
ctx := context.Background()
if Logger(ctx) != logrus.StandardLogger() {
t.Fatal("plain context should use standard logger")
}
if LoggerHandler(ctx) != nil {
t.Fatal("plain context should not have a logger handler")
}
if StepIDs(ctx) != nil {
t.Fatal("plain context should not have step IDs")
if LoggerHook(ctx) != nil {
t.Fatal("plain context should not have a logger hook")
}
logger := logging.New(logtest.New())
logger := logrus.New()
ctx = WithLogger(ctx, logger)
if Logger(ctx) != logger {
t.Fatal("WithLogger should set logger")
}
handler := logtest.New()
ctx = WithLoggerHandler(ctx, handler)
if LoggerHandler(ctx) != handler {
t.Fatal("WithLoggerHandler should set handler")
hook := testHook{}
ctx = WithLoggerHook(ctx, hook)
if LoggerHook(ctx) != hook {
t.Fatal("WithLoggerHook should set hook")
}
}
type testHook struct{}
func (testHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func (testHook) Fire(*logrus.Entry) error {
return nil
}
+7 -7
View File
@@ -9,7 +9,7 @@ import (
"fmt"
"runtime/debug"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
)
// Executor define contract for the steps of a workflow
@@ -74,33 +74,33 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
errs := make(chan error, len(executors))
if 1 > parallel {
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
parallel = 1
}
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
for i := 0; i < parallel; i++ {
go func(workerID int, work <-chan Executor, errs chan<- error) {
logging.Debugf("Worker %d started", workerID)
log.Debugf("Worker %d started", workerID)
taskCount := 0
for executor := range work {
taskCount++
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
log.Debugf("Worker %d executing task %d", workerID, taskCount)
// Recover from panics in executors to avoid crashing the worker
// goroutine which would leave the runner process hung.
// https://gitea.com/gitea/runner/issues/371
errs <- func() (err error) {
defer func() {
if r := recover(); r != nil {
logging.Errorf("panic in executor: %v\n%s", r, debug.Stack())
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
err = fmt.Errorf("panic: %v", r)
}
}()
return executor(ctx)
}()
}
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
}(i, work, errs)
}
+9 -9
View File
@@ -20,7 +20,7 @@ import (
func TestNewWorkflow(t *testing.T) {
assert := assert.New(t)
ctx := t.Context()
ctx := context.Background()
// empty
emptyWorkflow := NewPipelineExecutor()
@@ -72,7 +72,7 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
}
func TestNewParallelExecutor(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
exec, count, maxActive := concurrencyProbe(2)
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
@@ -89,10 +89,10 @@ func TestNewParallelExecutor(t *testing.T) {
func TestNewParallelExecutorEmpty(t *testing.T) {
assert := assert.New(t)
ctx := t.Context()
ctx := context.Background()
require.NoError(t, NewParallelExecutor(2)(ctx))
canceledCtx, cancel := context.WithCancel(t.Context())
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
err := NewParallelExecutor(2)(canceledCtx)
@@ -102,7 +102,7 @@ func TestNewParallelExecutorEmpty(t *testing.T) {
func TestNewParallelExecutorFailed(t *testing.T) {
assert := assert.New(t)
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
cancel()
count := 0
@@ -118,7 +118,7 @@ func TestNewParallelExecutorFailed(t *testing.T) {
func TestNewParallelExecutorCanceled(t *testing.T) {
assert := assert.New(t)
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
cancel()
errExpected := errors.New("fake error")
@@ -150,12 +150,12 @@ func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
}
}
require.Error(t, NewParallelExecutor(2, executors...)(t.Context()))
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
}
func TestExecutorConditionalsAndFinally(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
var calls []string
record := func(name string) Executor {
return func(ctx context.Context) error {
@@ -180,7 +180,7 @@ func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
mainErr := errors.New("main failed")
finalErr := errors.New("cleanup failed")
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
require.Error(t, err)
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
t.Fatalf("finally error = %q, want both cleanup and original error", err)
+410 -316
View File
@@ -6,66 +6,44 @@ package git
import (
"context"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"net/url"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"time"
"sync"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/internal/pkg/lock"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/mattn/go-isatty"
log "github.com/sirupsen/logrus"
)
var (
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
}
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
cloneLocks lock.Keyed[string] // key: clone target directory
ErrShortRef = errors.New("short SHA references are not supported")
)
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
dir = filepath.Clean(dir)
if runtime.GOOS == "plan9" {
return cloneLocks.Lock(dir), nil
}
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
return nil, err
}
for {
release, err := lock.TryLock(dir)
if err == nil {
return func() { _ = release() }, nil
}
if !errors.Is(err, lock.ErrLocked) {
return cloneLocks.Lock(dir), nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(50 * time.Millisecond):
}
}
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
func AcquireCloneLock(dir string) func() {
return cloneLocks.Lock(dir)
}
type Error struct {
@@ -73,97 +51,196 @@ type Error struct {
commit string
}
func (e *Error) Error() string { return e.err.Error() }
func (e *Error) Unwrap() error { return e.err }
func (e *Error) Commit() string { return e.commit }
func (e *Error) Error() string {
return e.err.Error()
}
func (e *Error) Unwrap() error {
return e.err
}
func (e *Error) Commit() string {
return e.commit
}
// goGitMu serializes go-git repository access across the process. go-git is not safe for
// concurrent use of the same repository (even read access decodes packfiles into shared
// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded
// operations are fast local reads; gitea runs one job per process, so the lock is effectively
// uncontended in production.
var goGitMu sync.Mutex
// FindGitRevision get the current git revision
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
goGitMu.Lock()
defer goGitMu.Unlock()
return findGitRevision(ctx, file)
}
func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
logger := common.Logger(ctx)
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
gitDir, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
logger.WithError(err).Error("path", file, "not located inside a git repository")
return "", "", err
}
if len(sha) < 7 {
return "", "", errors.New("HEAD could not be resolved")
head, err := gitDir.Reference(plumbing.HEAD, true)
if err != nil {
return "", "", err
}
logger.Debugf("Found revision: %s", sha)
return sha[:7], sha, nil
if head.Hash().IsZero() {
return "", "", errors.New("HEAD sha1 could not be resolved")
}
hash := head.Hash().String()
logger.Debugf("Found revision: %s", hash)
return hash[:7], strings.TrimSpace(hash), nil
}
// FindGitRef get the current git ref
func FindGitRef(ctx context.Context, file string) (string, error) {
logger := common.Logger(ctx)
logger.Debugf("Loading revision from git directory")
_, revision, err := FindGitRevision(ctx, file)
if err != nil {
return "", err
}
logger.Debugf("HEAD points to '%s'", revision)
goGitMu.Lock()
defer goGitMu.Unlock()
refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
logger := common.Logger(ctx)
logger.Debugf("Loading revision from git directory")
_, ref, err := findGitRevision(ctx, file)
if err != nil {
return "", err
}
var tag, branch string
for ref := range strings.FieldsSeq(refs) {
switch {
case strings.HasPrefix(ref, "refs/tags/"):
tag = ref
case strings.HasPrefix(ref, "refs/heads/"):
branch = ref
logger.Debugf("HEAD points to '%s'", ref)
// Prefer the git library to iterate over the references and find a matching tag or branch.
refTag := ""
refBranch := ""
repo, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
return "", err
}
iter, err := repo.References()
if err != nil {
return "", err
}
// find the reference that matches the revision's has
err = iter.ForEach(func(r *plumbing.Reference) error {
/* tags and branches will have the same hash
* when a user checks out a tag, it is not mentioned explicitly
* in the go-git package, we must identify the revision
* then check if any tag matches that revision,
* if so then we checked out a tag
* else we look for branches and if matches,
* it means we checked out a branch
*
* If a branches matches first we must continue and check all tags (all references)
* in case we match with a tag later in the interation
*/
if r.Hash().String() == ref {
if r.Name().IsTag() {
refTag = r.Name().String()
}
if r.Name().IsBranch() {
refBranch = r.Name().String()
}
}
// we found what we where looking for
if refTag != "" && refBranch != "" {
return storer.ErrStop
}
return nil
})
if err != nil {
return "", err
}
if tag != "" {
return tag, nil
// order matters here see above comment.
if refTag != "" {
return refTag, nil
}
if branch != "" {
return branch, nil
if refBranch != "" {
return refBranch, nil
}
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)
}
// FindGithubRepo get the repo
func FindGithubRepo(ctx context.Context, file string) (string, error) {
url, err := findGitRemoteURL(ctx, file)
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
goGitMu.Lock()
defer goGitMu.Unlock()
url, err := findGitRemoteURL(ctx, file, "origin")
if err != nil {
return "", err
}
return findGitSlug(url)
_, slug := findGitSlug(url, githubInstance)
return slug, nil
}
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
repo, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
return "", err
}
if urls == "" {
return "", errors.New("remote 'origin' exists but has no URL")
remote, err := repo.Remote(remoteName)
if err != nil {
return "", err
}
url, _, _ := strings.Cut(urls, "\n")
return url, nil
if len(remote.Config().URLs) < 1 {
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)
}
return remote.Config().URLs[0], nil
}
func findGitSlug(remoteURL string) (string, error) {
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
return matches[2], nil
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
return matches[2], nil
func findGitSlug(url, githubInstance string) (string, string) {
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2]
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2]
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if githubInstance != "github.com" {
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
}
}
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
}
parsed, err := url.Parse(remoteURL)
if err != nil {
return "", errors.New("cannot parse the origin remote URL")
}
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
owner = path.Base(owner)
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
return "", errors.New("cannot determine owner and repository from the origin remote URL")
}
return owner + "/" + repo, nil
return "", url
}
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
@@ -183,38 +260,96 @@ type NewGitCloneExecutorInput struct {
Quiet bool
// For Gitea
InsecureSkipTLS bool
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
InsecureSkipTLS bool
}
// CloneIfRequired reports whether an existing local clone was reused.
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
var origin string
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused.
func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) {
r, err := git.PlainOpen(input.Dir)
if err == nil {
origin, err = findGitRemoteURL(ctx, input.Dir)
// Verify the cached clone still points to the resolved URL before reusing it.
remote, err := r.Remote("origin")
if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL {
// Reuse existing clone
return r, true, nil
}
switch {
case err != nil:
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
case len(remote.Config().URLs) == 0:
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
default:
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
}
if err := os.RemoveAll(input.Dir); err != nil {
return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
}
}
if err == nil && origin == input.URL {
return true, nil
var progressWriter io.Writer
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
if entry, ok := logger.(*log.Entry); ok {
progressWriter = entry.WriterLevel(log.DebugLevel)
} else if lgr, ok := logger.(*log.Logger); ok {
progressWriter = lgr.WriterLevel(log.DebugLevel)
} else {
log.Errorf("Unable to get writer from logger (type=%T)", logger)
progressWriter = os.Stdout
}
}
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
return false, ctxErr
cloneOptions := git.CloneOptions{
URL: input.URL,
Progress: progressWriter,
InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea
}
if err == nil {
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
if input.Token != "" {
cloneOptions.Auth = &http.BasicAuth{
Username: "token",
Password: input.Token,
}
}
if err := os.RemoveAll(input.Dir); err != nil {
return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
r, err = cloneAtDepth(ctx, input, cloneOptions, logger)
if err != nil {
logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err)
return nil, false, err
}
if err := clone(ctx, input); err != nil {
return false, err
if err = os.Chmod(input.Dir, 0o755); err != nil {
return nil, false, err
}
if err := os.Chmod(input.Dir, 0o755); err != nil {
return false, err
return r, false, nil
}
func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.PullOptions) {
fetchOptions.RefSpecs = []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}
fetchOptions.Force = true
pullOptions.Force = true
if token != "" {
auth := &http.BasicAuth{
Username: "token",
Password: token,
}
fetchOptions.Auth = auth
pullOptions.Auth = auth
}
return false, nil
return fetchOptions, pullOptions
}
// staleRefreshErr reports why a failed refresh must abort: the resolve and
// checkout that follow are local and succeed on a cancelled context, which
// would hand back the cached revision as if it were fresh.
func staleRefreshErr(ctx context.Context, err error) error {
if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) {
return nil
}
return ctx.Err()
}
// NewGitCloneExecutor creates an executor to clone git repos
@@ -227,232 +362,191 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
}
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
unlock, err := AcquireCloneLock(ctx, input.Dir)
if err != nil {
return err
}
defer unlock()
reused, err := CloneIfRequired(ctx, input, logger)
defer AcquireCloneLock(input.Dir)()
refName := plumbing.ReferenceName("refs/heads/" + input.Ref)
r, reused, err := CloneIfRequired(ctx, refName, input, logger)
if err != nil {
return err
}
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
return err
isOfflineMode := input.OfflineMode
// fetch latest changes
fetchOptions, pullOptions := gitOptions(input.Token)
if input.InsecureSkipTLS { // For Gitea
fetchOptions.InsecureSkipTLS = true
pullOptions.InsecureSkipTLS = true
}
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
if err := fetch(ctx, input); err != nil {
// Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth
// Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone.
shallow := isShallow(r)
if shallow {
fetchOptions.Depth = 1
if spec, ok := shallowFetchRefSpec(r, input.Ref); ok {
fetchOptions.RefSpecs = []config.RefSpec{spec}
}
}
if !isOfflineMode {
err = r.FetchContext(ctx, &fetchOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
}
if err != nil {
var hash *plumbing.Hash
rev := plumbing.Revision(input.Ref)
if hash, err = r.ResolveRevision(rev); err != nil {
// ResolveRevision returns a nil hash on error, and a branch ref legitimately fails
// here (no local refs/heads/<ref>); the duck-typing below resolves it.
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
} else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) {
return &Error{
err: ErrShortRef,
commit: hash.String(),
}
}
// At this point we need to know if it's a tag or a branch
// And the easiest way to do it is duck typing
//
// If err is nil, it's a tag so let's proceed with that hash like we would if
// it was a sha
refType := "tag"
rev = plumbing.Revision(path.Join("refs", "tags", input.Ref))
if _, err := r.Tag(input.Ref); errors.Is(err, git.ErrTagNotFound) {
rName := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
if _, err := r.Reference(rName, false); errors.Is(err, plumbing.ErrReferenceNotFound) {
refType = "sha"
rev = plumbing.Revision(input.Ref)
} else {
refType = "branch"
rev = plumbing.Revision(rName)
}
}
if hash, err = r.ResolveRevision(rev); err != nil {
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
return err
}
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
return fmt.Errorf("checkout %s: %w", resolved, err)
var w *git.Worktree
if w, err = r.Worktree(); err != nil {
return err
}
// If the hash resolved doesn't match the ref provided in a workflow then we're
// using a branch or tag ref, not a sha
//
// Repos on disk point to commit hashes, and need to checkout input.Ref before
// we try and pull down any changes
if hash.String() != input.Ref && refType == "branch" {
logger.Debugf("Provided ref is not a sha. Checking out branch before pulling changes")
sourceRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
if err = w.Checkout(&git.CheckoutOptions{
Branch: sourceRef,
Force: true,
}); err != nil {
logger.Errorf("Unable to checkout %s: %v", sourceRef, err)
return err
}
}
reusedMsg := ""
if input.OfflineMode && reused {
switch {
case !isOfflineMode && !shallow:
// In shallow mode the depth-limited fetch above already advanced the ref.
if err = w.PullContext(ctx, &pullOptions); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
logger.Debugf("Unable to pull %s: %v", refName, err)
}
if err := staleRefreshErr(ctx, err); err != nil {
return err
}
case isOfflineMode && reused:
reusedMsg = " (reused in offline mode)"
}
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
if hash.String() != input.Ref && refType == "branch" {
logger.Debugf("Provided ref is not a sha. Updating branch ref after pull")
if hash, err = r.ResolveRevision(rev); err != nil {
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
return err
}
}
if err = w.Checkout(&git.CheckoutOptions{
Hash: *hash,
Force: true,
}); err != nil {
logger.Errorf("Unable to checkout %s: %v", *hash, err)
return err
}
if err = w.Reset(&git.ResetOptions{
Mode: git.HardReset,
Commit: *hash,
}); err != nil {
logger.Errorf("Unable to reset to %s: %v", hash.String(), err)
return err
}
logger.Debugf("Checked out %s", input.Ref)
return nil
}
}
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
if input.Depth > 0 && isFullObjectID(input.Ref) {
if err := shallowPinnedClone(ctx, input); err == nil {
return nil
} else if err := removePartialClone(input.Dir); err != nil {
return err
}
}
args := []string{"clone", "--no-checkout"}
// cloneAtDepth clones input.URL into input.Dir using opts.
// With input.Depth > 0 it first tries a shallow, single-branch clone of input.Ref, falling back when error.
func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
if input.Depth > 0 {
args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
args = append(args, "--branch", input.Ref)
}
}
args = append(args, input.URL, input.Dir)
if _, err := runGit(ctx, "", &input, args...); err == nil {
return nil
} else if input.Depth == 0 {
return err
}
if err := removePartialClone(input.Dir); err != nil {
return err
}
_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
return err
}
for _, refName := range []plumbing.ReferenceName{
plumbing.NewBranchReferenceName(input.Ref),
plumbing.NewTagReferenceName(input.Ref),
} {
shallowOpts := opts
shallowOpts.Depth = input.Depth
shallowOpts.SingleBranch = true
shallowOpts.ReferenceName = refName
shallowOpts.Tags = git.NoTags
func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
return err
}
if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
return err
}
_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
return err
}
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
args := []string{"fetch", "--force", "--prune"}
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
args = append(args, "--depth", "1", "--no-tags")
}
_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
return err
}
func cachedRef(ref string) string {
return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
}
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
if !isFullObjectID(ref) {
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
return sha, nil
r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
if err == nil {
return r, nil
}
logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
if rmErr := os.RemoveAll(input.Dir); rmErr != nil {
return nil, fmt.Errorf("remove partial clone %s: %w", input.Dir, rmErr)
}
}
logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
}
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
return "", &Error{err: ErrShortRef, commit: sha}
}
return sha, err
return git.PlainCloneContext(ctx, input.Dir, false, &opts)
}
func isFullObjectID(ref string) bool {
return ObjectFormat(ref) != ""
// isShallow reports whether the local repository was cloned with a limited depth.
func isShallow(r *git.Repository) bool {
shallows, err := r.Storer.Shallow()
return err == nil && len(shallows) > 0
}
func isFullHash(ref, resolved string) bool {
return isFullObjectID(ref) && len(ref) == len(resolved)
}
// ObjectFormat returns the object format of a full commit hash, empty for anything else.
func ObjectFormat(sha string) string {
switch {
case !hexRefRegex.MatchString(sha):
return ""
case len(sha) == 40:
return "sha1"
case len(sha) == 64:
return "sha256"
}
return ""
}
func removePartialClone(dir string) error {
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("remove partial clone %s: %w", dir, err)
}
return nil
}
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
output, err := runGit(ctx, dir, nil, args...)
return strings.TrimSpace(string(output)), err
}
func RemoteConfig(token string, insecureSkipTLS bool) []string {
var config []string
if insecureSkipTLS {
config = append(config, "http.sslVerify=false")
}
if token != "" {
config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
}
return config
}
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
func ConfigEnv(config ...string) map[string]string {
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
for i, entry := range config {
key, value, _ := strings.Cut(entry, "=")
env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
}
return env
}
func ListFiles(ctx context.Context, dir string) ([]string, error) {
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
if err != nil {
return nil, err
}
var files, repos []string
for entry := range strings.SplitSeq(string(listed), "\x00") {
tag, file, ok := strings.Cut(entry, " ")
if tag != "?" {
var meta string
meta, file, ok = strings.Cut(file, "\t")
if strings.HasPrefix(meta, "160000 ") {
file += "/"
}
}
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
repos = append(repos, repo)
} else if ok {
files = append(files, file)
}
}
slices.Sort(repos)
for _, repo := range slices.Compact(repos) {
files = append(files, repo)
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
continue // git lists an uninitialized submodule as "./" from inside it
}
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
if err != nil {
return nil, err
}
for _, file := range nested {
files = append(files, repo+"/"+file)
}
}
return files, nil
}
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
if input != nil {
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
if input.ClientCertFile != "" {
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
}
}
if dir != "" {
args = append([]string{"-C", dir}, args...)
}
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
name, _, _ := strings.Cut(entry, "=")
return slices.Contains(LocalEnvVars, name)
})
for key, value := range ConfigEnv(config...) {
cmd.Env = append(cmd.Env, key+"="+value)
}
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
if runtime.GOOS != "windows" {
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
}
cmd.WaitDelay = time.Second
var stderr strings.Builder
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
}
return output, nil
// shallowFetchRefSpec returns the single refspec that updates only input.Ref, keeping a shallow clone from re-downloading every branch's history.
// ok is false when the ref is not present locally as a tag or remote-tracking branch, in which case the broad default refspec is used.
func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
tagRef := plumbing.NewTagReferenceName(ref)
if _, err := r.Reference(tagRef, false); err == nil {
return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true
}
remoteRef := plumbing.NewRemoteReferenceName("origin", ref)
if _, err := r.Reference(remoteRef, false); err == nil {
branchRef := plumbing.NewBranchReferenceName(ref)
return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
}
return "", false
}
+123 -252
View File
@@ -6,7 +6,7 @@ package git
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/http/httptest"
@@ -18,46 +18,42 @@ import (
"sync"
"syscall"
"testing"
"testing/synctest"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.com/gitea/runner/internal/pkg/lock"
gogit "github.com/go-git/go-git/v5"
gogitconfig "github.com/go-git/go-git/v5/config"
log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFindGitSlug(t *testing.T) {
for _, testcase := range []struct {
remoteURL string
slug string
assert := assert.New(t)
slugTests := []struct {
url string // input
provider string // expected result
slug string // expected result
}{
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "my-repo-name"},
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "my-repo"},
{"[email protected]:nektos/act.git", "nektos/act"},
{"[email protected]:nektos/act", "nektos/act"},
{"https://github.com/nektos/act.git", "nektos/act"},
{"http://github.com/nektos/act.git", "nektos/act"},
{"https://github.com/nektos/act", "nektos/act"},
{"http://github.com/nektos/act", "nektos/act"},
{"git+ssh://[email protected]/owner/repo.git", "owner/repo"},
{"https://example.com:3000/gitea/owner/repo.git/", "owner/repo"},
{"ssh://[email protected]:2222/gitea/owner/repo.git", "owner/repo"},
{"example.com:owner/repo.git", "owner/repo"},
{"http://myotherrepo.com/act.git", ""},
{"https://user:[email protected]/act.git", ""},
{"/tmp/owner/repo.git", ""},
{"./owner/repo:branch", ""},
} {
t.Run(testcase.remoteURL, func(t *testing.T) {
slug, err := findGitSlug(testcase.remoteURL)
assert.Equal(t, testcase.slug == "", err != nil)
assert.NotContains(t, fmt.Sprint(err), "secret")
assert.Equal(t, testcase.slug, slug)
})
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "CodeCommit", "my-repo-name"},
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "CodeCommit", "my-repo"},
{"[email protected]:nektos/act.git", "GitHub", "nektos/act"},
{"[email protected]:nektos/act", "GitHub", "nektos/act"},
{"https://github.com/nektos/act.git", "GitHub", "nektos/act"},
{"http://github.com/nektos/act.git", "GitHub", "nektos/act"},
{"https://github.com/nektos/act", "GitHub", "nektos/act"},
{"http://github.com/nektos/act", "GitHub", "nektos/act"},
{"git+ssh://[email protected]/owner/repo.git", "GitHub", "owner/repo"},
{"http://myotherrepo.com/act.git", "", "http://myotherrepo.com/act.git"},
}
for _, tt := range slugTests {
provider, slug := findGitSlug(tt.url, "github.com")
assert.Equal(tt.provider, provider)
assert.Equal(tt.slug, slug)
}
}
@@ -89,37 +85,23 @@ func cleanGitHooks(dir string) error {
return nil
}
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
t.Parallel()
func TestFindGithubRepoUsesOrigin(t *testing.T) {
basedir := t.TempDir()
const remoteURL = "https://github.com/owner/repo.git"
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
require.NoError(t, gitCmd("init", basedir))
require.NoError(t, cleanGitHooks(basedir))
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
shortSHA, sha, err := FindGitRevision(t.Context(), basedir)
require.NoError(t, err)
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
require.Len(t, sha, 64)
require.Equal(t, sha[:7], shortSHA)
ref, err := FindGitRef(t.Context(), basedir)
require.NoError(t, err)
require.Equal(t, "refs/heads/main", ref)
url, err := findGitRemoteURL(t.Context(), basedir)
url, err := findGitRemoteURL(context.Background(), basedir, "origin")
require.NoError(t, err)
require.Equal(t, remoteURL, url)
slug, err := FindGithubRepo(t.Context(), basedir)
slug, err := FindGithubRepo(context.Background(), basedir, "github.com")
require.NoError(t, err)
require.Equal(t, "owner/repo", slug)
}
func TestGitFindRef(t *testing.T) {
t.Parallel()
basedir := t.TempDir()
for name, tt := range map[string]struct {
@@ -190,7 +172,7 @@ func TestGitFindRef(t *testing.T) {
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
require.NoError(t, cleanGitHooks(dir))
tt.Prepare(t, dir)
ref, err := FindGitRef(t.Context(), dir)
ref, err := FindGitRef(context.Background(), dir)
tt.Assert(t, ref, err)
})
}
@@ -206,9 +188,7 @@ func TestGitCloneExecutor(t *testing.T) {
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
@@ -222,13 +202,6 @@ func TestGitCloneExecutor(t *testing.T) {
require.NoError(t, err)
fullSha := strings.TrimSpace(string(out))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
t.Setenv("GIT_DIR", t.TempDir())
for name, tt := range map[string]struct {
Err error
Ref string
@@ -249,42 +222,26 @@ func TestGitCloneExecutor(t *testing.T) {
Err: &Error{ErrShortRef, fullSha},
Ref: fullSha[:7],
},
"HEAD": {
Ref: "HEAD",
},
"hex branch named like its own commit": {
Ref: fullSha[:4],
},
} {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: "https://example.invalid/action",
URL: remoteDir,
Ref: tt.Ref,
Dir: dir,
Dir: t.TempDir(),
})
err := clone(t.Context())
err := clone(context.Background())
if tt.Err != nil {
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.Err, err)
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
return
} else {
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
}
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
marker := filepath.Join(dir, "marker")
require.NoError(t, os.WriteFile(marker, nil, 0o644))
require.NoError(t, clone(t.Context()))
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
require.NoError(t, err)
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
})
}
}
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
t.Parallel()
createRemote := func(message string) string {
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
@@ -308,27 +265,16 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
Dir: cacheDir,
})(t.Context()))
nested := filepath.Join(cacheDir, "nested")
require.NoError(t, os.Mkdir(nested, 0o755))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context()))
assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository")
markerPath := filepath.Join(cacheDir, "stale-marker")
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
cancelled, cancel := context.WithCancel(t.Context())
cancel()
_, err := CloneIfRequired(cancelled, NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: cacheDir}, logging.New(logtest.New()))
require.ErrorIs(t, err, context.Canceled)
require.FileExists(t, markerPath)
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: newRemoteDir,
Ref: "main",
Dir: cacheDir,
})(t.Context()))
originURL, err := findGitRemoteURL(t.Context(), cacheDir)
originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin")
require.NoError(t, err)
assert.Equal(t, newRemoteDir, originURL)
@@ -340,36 +286,61 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning")
}
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
t.Parallel()
func TestGitCloneExecutorNonFastForwardRef(t *testing.T) {
// Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes
// non-fast-forward between two fetches. Before the fix, the fetch used Force=false,
// causing go-git to return ErrForceNeeded and short-circuit the checkout.
// Create a bare "remote" repo with an initial commit on main and a feature branch.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
// We need a working clone to push commits from.
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
require.NoError(t, clone(t.Context()))
// Create a feature branch (simulates refs/pull/N/head).
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature"))
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
// First clone via the executor — should succeed and cache the repo.
cloneDir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "main",
Dir: cloneDir,
})
require.NoError(t, clone(context.Background()))
// Now force-push the feature branch to a non-fast-forward commit (simulates
// a PR rebase). This makes refs/heads/feature non-fast-forward.
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature"))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
require.NoError(t, clone(t.Context()), "a non-fast-forward ref must update")
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature"))
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: t.TempDir()})
require.Error(t, tagClone(t.Context()))
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
require.NoError(t, tagClone(t.Context()), "a tag published after a failed lookup must be fetched")
// Also advance main so we can verify the clone picks up the new commit.
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
// Second clone to the same directory — before the fix this returned ErrForceNeeded
// and left the working tree at the old commit.
err := clone(context.Background())
require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true")
// Verify the working tree was actually updated to the latest main commit.
out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output()
require.NoError(t, err)
assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit")
}
func TestGitCloneExecutorOfflineMode(t *testing.T) {
t.Parallel()
// Build a local "remote" with a single commit on main.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
@@ -385,7 +356,7 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
URL: remoteDir,
Ref: "main",
Dir: cacheDir,
})(t.Context()))
})(context.Background()))
t.Run("cached branch resolves without fetching", func(t *testing.T) {
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
@@ -395,7 +366,7 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
Ref: "main",
Dir: cacheDir,
OfflineMode: true,
})(t.Context())
})(context.Background())
require.NoError(t, err)
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
@@ -410,13 +381,12 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
Ref: "never-fetched",
Dir: cacheDir,
OfflineMode: true,
})(t.Context())
})(context.Background())
require.Error(t, err)
})
}
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
t.Parallel()
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
@@ -428,9 +398,9 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
// Quiet callers report the download themselves, so the clone line must not reach the job log.
for name, quiet := range map[string]bool{"quiet": true, "not quiet": false} {
t.Run(name, func(t *testing.T) {
hook := logtest.New()
logger := logging.New(logging.NewLevelHandler(hook, logging.LevelInfo))
ctx := common.WithLogger(t.Context(), logger.With("job", "j1"))
logger, hook := logrustest.NewNullLogger()
logger.SetLevel(log.InfoLevel)
ctx := common.WithLogger(context.Background(), logger.WithField("job", "j1"))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
@@ -440,7 +410,7 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
})(ctx))
var cloneLines int
for _, entry := range hook.AllRecords() {
for _, entry := range hook.AllEntries() {
if strings.HasPrefix(entry.Message, "git clone ") {
cloneLines++
}
@@ -455,7 +425,6 @@ func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
}
func TestGitCloneExecutorShallow(t *testing.T) {
t.Parallel()
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
@@ -466,7 +435,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
}
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
sha := gitRevParse(t, workDir, "HEAD~1") // c2, a SHA that go-git cannot shallow-clone
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
@@ -492,105 +461,34 @@ func TestGitCloneExecutorShallow(t *testing.T) {
assert.Equal(t, "c3", gitHeadSubject(t, dir))
})
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
remote := filepath.Join(t.TempDir(), "remote.git")
require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
return remote
}
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
t.Run("SHA falls back to a full clone", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
URL: remoteDir, Ref: sha, Dir: dir, Depth: 1,
})(t.Context()))
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
// go-git cannot shallow-clone a raw SHA, so it falls back to a full clone; the absence of a shallow marker proves the fallback happened.
assert.NoFileExists(t, shallowMarker(dir), "a SHA ref must not produce a shallow clone")
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
})
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
remote := unadvertisedRemote(t, true)
t.Run("moving branch updates while staying shallow", func(t *testing.T) {
dir := t.TempDir()
cloneAt := func(ref string) error {
return NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remote, Ref: ref, Dir: dir, Depth: 1,
})(t.Context())
}
require.NoError(t, cloneAt(unadvertisedSHA))
assert.FileExists(t, shallowMarker(dir))
assert.Equal(t, 1, gitRevCount(t, dir))
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
olderSHA := gitRevParse(t, workDir, "HEAD~2")
require.NoError(t, cloneAt(olderSHA))
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
require.NoError(t, os.RemoveAll(remote))
require.NoError(t, cloneAt(olderSHA))
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
})
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1,
})
require.NoError(t, clone(t.Context()))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
})(t.Context()))
require.Equal(t, "c3", gitHeadSubject(t, dir))
// Advance main on the remote, then reuse the existing shallow clone.
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
require.NoError(t, clone(t.Context()))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
})(t.Context()))
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
assert.Equal(t, 1, gitRevCount(t, dir))
})
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
remote := filepath.Join(t.TempDir(), "remote.git")
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
work := t.TempDir()
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
sha := gitRevParse(t, work, "HEAD")
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
require.NoError(t, clone(t.Context()))
assert.FileExists(t, shallowMarker(dir))
require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef)
require.NoError(t, os.RemoveAll(remote))
require.NoError(t, clone(t.Context()))
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
})
}
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
t.Parallel()
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "pinned"))
pinned := gitRevParse(t, workDir, "HEAD")
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
})(t.Context()))
assert.Equal(t, pinned, gitRevParse(t, dir, "HEAD"))
})
}
}
func gitRevParse(t *testing.T, dir, rev string) string {
@@ -643,42 +541,14 @@ func gitCmd(args ...string) error {
}
func TestAcquireCloneLock(t *testing.T) {
t.Run("blocks on a lock held by another process despite a trailing separator", func(t *testing.T) {
dir := filepath.Join(t.TempDir(), "cache")
release, err := lock.TryLock(dir)
require.NoError(t, err)
defer func() { require.NoError(t, release()) }()
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
_, err := AcquireCloneLock(ctx, dir+string(filepath.Separator))
require.ErrorIs(t, err, context.DeadlineExceeded)
})
})
t.Run("read-only parent falls back to an in-process lock", func(t *testing.T) {
parent := t.TempDir()
dir := filepath.Join(parent, "cache")
require.NoError(t, os.Mkdir(dir, 0o755))
require.NoError(t, os.Chmod(parent, 0o555))
defer func() { require.NoError(t, os.Chmod(parent, 0o755)) }()
unlock, err := AcquireCloneLock(t.Context(), dir)
require.NoError(t, err)
unlock()
})
t.Run("same directory serializes", func(t *testing.T) {
dir := t.TempDir()
unlock1, err := AcquireCloneLock(t.Context(), dir)
require.NoError(t, err)
unlock1 := AcquireCloneLock(dir)
secondAcquired := make(chan struct{})
go func() {
unlock, err := AcquireCloneLock(t.Context(), dir)
if !assert.NoError(t, err) {
return
}
unlock := AcquireCloneLock(dir)
close(secondAcquired)
unlock()
}()
@@ -702,16 +572,12 @@ func TestAcquireCloneLock(t *testing.T) {
dirA := t.TempDir()
dirB := t.TempDir()
unlockA, err := AcquireCloneLock(t.Context(), dirA)
require.NoError(t, err)
unlockA := AcquireCloneLock(dirA)
defer unlockA()
done := make(chan struct{})
go func() {
unlock, err := AcquireCloneLock(t.Context(), dirB)
if !assert.NoError(t, err) {
return
}
unlock := AcquireCloneLock(dirB)
unlock()
close(done)
}()
@@ -725,17 +591,12 @@ func TestAcquireCloneLock(t *testing.T) {
}
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
t.Parallel()
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
block := make(chan struct{})
reached := make(chan struct{})
var once sync.Once
var authorization string
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
once.Do(func() {
authorization = request.Header.Get("Authorization")
close(reached)
})
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
once.Do(func() { close(reached) })
<-block
}))
t.Cleanup(func() {
@@ -744,15 +605,17 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
})
dir := filepath.Join(t.TempDir(), "cached-action")
require.NoError(t, gitCmd("init", dir))
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
repo, err := gogit.PlainInit(dir, false)
require.NoError(t, err)
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
require.NoError(t, err)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx)
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
}()
select {
@@ -760,7 +623,6 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
case <-time.After(10 * time.Second):
t.Fatal("the executor never reached the remote")
}
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
cancel()
select {
@@ -770,3 +632,12 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
t.Fatal("fetch ignored context cancellation")
}
}
func TestStaleRefreshErr(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
require.NoError(t, staleRefreshErr(ctx, errors.New("remote hung up")))
cancel()
require.ErrorIs(t, staleRefreshErr(ctx, errors.New("remote hung up")), context.Canceled)
require.NoError(t, staleRefreshErr(ctx, gogit.NoErrAlreadyUpToDate))
}
+23 -30
View File
@@ -7,53 +7,46 @@ package common
import (
"context"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
)
type loggerContextKey string
const loggerContextKeyVal = loggerContextKey("logging.Logger")
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
// Logger returns the appropriate logger for current context
func Logger(ctx context.Context) *logging.Logger {
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
return logger
func Logger(ctx context.Context) logrus.FieldLogger {
val := ctx.Value(loggerContextKeyVal)
if val != nil {
if logger, ok := val.(logrus.FieldLogger); ok {
return logger
}
}
return logging.Default()
return logrus.StandardLogger()
}
// WithLogger adds a value to the context for the logger
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
return context.WithValue(ctx, loggerContextKeyVal, logger)
}
type loggerHandlerKey string
type loggerHookKey string
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
// It affects the job logger, not the global logger.
func LoggerHandler(ctx context.Context) logging.Handler {
if h, ok := ctx.Value(loggerHandlerKeyVal).(logging.Handler); ok {
return h
// LoggerHook returns the appropriate logger hook for current context
// the hook affects job logger, not global logger
func LoggerHook(ctx context.Context) logrus.Hook {
val := ctx.Value(loggerHookKeyVal)
if val != nil {
if hook, ok := val.(logrus.Hook); ok {
return hook
}
}
return nil
}
// WithLoggerHandler adds the extra handler for the job logger to the context
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
return context.WithValue(ctx, loggerHandlerKeyVal, h)
}
type stepIDsContextKey struct{}
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
func StepIDs(ctx context.Context) []string {
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
return ids
}
// WithStepIDs replaces the step IDs in the context
func WithStepIDs(ctx context.Context, ids []string) context.Context {
return context.WithValue(ctx, stepIDsContextKey{}, ids)
// WithLoggerHook adds a value to the context for the logger hook
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
return context.WithValue(ctx, loggerHookKeyVal, hook)
}
-287
View File
@@ -1,287 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"context"
"fmt"
"io"
"log/slog"
"reflect"
"runtime"
"slices"
"strconv"
"strings"
"sync"
"time"
)
const (
ansiReset = "\x1b[0m"
ansiRed = "\x1b[31m"
ansiYellow = "\x1b[33m"
ansiCyan = "\x1b[36m"
ansiDimCyan = "\x1b[2;36m"
ansiDimWhite = "\x1b[2;37m"
)
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
var consoleStart = time.Now()
// console is the state shared by every consoleHandler derived from one another.
type console struct {
mu sync.Mutex
out io.Writer
terminal bool
opts TextOptions
reportCaller bool
level slog.LevelVar
}
func newConsole(out io.Writer) *console {
c := &console{}
c.setOutput(out)
return c
}
func (c *console) setOutput(w io.Writer) {
terminal := checkIfTerminal(w)
c.mu.Lock()
defer c.mu.Unlock()
c.out, c.terminal = w, terminal
}
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
// and a colored prefix with elapsed or full time on a terminal.
type consoleHandler struct {
c *console
attrs []slog.Attr
}
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
return level >= h.c.level.Level()
}
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
}
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
c := h.c
c.mu.Lock()
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
c.mu.Unlock()
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
for _, a := range h.attrs {
data[a.Key] = a.Value.Resolve().Any()
}
r.Attrs(func(a slog.Attr) bool {
data[a.Key] = a.Value.Resolve().Any()
return true
})
var caller *runtime.Frame
if reportCaller && r.PC != 0 {
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
caller = &frame
}
prefixFieldClashes(data, caller != nil)
keys := make([]string, 0, len(data))
for k := range data {
keys = append(keys, k)
}
slices.Sort(keys)
var b bytes.Buffer
if terminal && !opts.DisableColors {
printColored(&b, opts, r, caller, keys, data)
} else {
printPlain(&b, opts, r, caller, keys, data)
}
line := b.Bytes()
if opts.Filter != nil {
line = opts.Filter(line)
}
c.mu.Lock()
defer c.mu.Unlock()
_, err := c.out.Write(line)
return err
}
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
func prefixFieldClashes(data map[string]any, reportCaller bool) {
for _, key := range []string{"time", "level", "msg"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
delete(data, key)
}
}
if reportCaller {
for _, key := range []string{"func", "file"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
}
}
}
}
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
if !opts.DisableTimestamp {
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
}
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
if r.Message != "" {
appendKeyValue(b, "msg", r.Message)
}
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if funcVal != "" {
appendKeyValue(b, "func", funcVal)
}
if fileVal != "" {
appendKeyValue(b, "file", fileVal)
}
}
for _, k := range keys {
appendKeyValue(b, k, data[k])
}
b.WriteByte('\n')
}
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
if opts.CallerPrettyfier != nil {
return opts.CallerPrettyfier(caller)
}
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
}
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
msg := strings.TrimSuffix(r.Message, "\n")
var callerStr string
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if opts.CallerPrettyfier == nil && caller.Function != "" {
funcVal += "()"
}
switch {
case fileVal == "":
callerStr = funcVal
case funcVal == "":
callerStr = fileVal
default:
callerStr = fileVal + " " + funcVal
}
}
level := Level(r.Level)
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
switch {
case opts.DisableTimestamp:
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
case !opts.FullTimestamp:
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
default:
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
}
for _, k := range keys {
b.WriteByte(' ')
b.WriteString(colorize(level, k))
b.WriteByte('=')
appendValue(b, data[k])
}
b.WriteByte('\n')
}
// plainLevel spells Warn the way logrus did, as "warning".
func plainLevel(l Level) string {
if s := l.String(); s != "warn" {
return s
}
return "warning"
}
func colorize(l Level, s string) string {
var color string
switch {
case l <= LevelTrace:
color = ansiDimWhite
case l <= LevelDebug:
color = ansiDimCyan
case l <= LevelInfo:
color = ansiCyan
case l <= LevelWarn:
color = ansiYellow
default:
color = ansiRed
}
return color + s + ansiReset
}
func appendKeyValue(b *bytes.Buffer, key string, value any) {
if b.Len() > 0 {
b.WriteByte(' ')
}
b.WriteString(key)
b.WriteByte('=')
appendValue(b, value)
}
func appendValue(b *bytes.Buffer, value any) {
switch v := value.(type) {
case string:
appendString(b, v)
case []byte:
appendString(b, string(v))
case error:
appendFormatted(b, v, v.Error)
case fmt.Stringer:
appendFormatted(b, v, v.String)
default:
appendString(b, fmt.Sprint(value))
}
}
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
defer func() {
if r := recover(); r != nil {
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
appendString(b, "<nil>")
} else {
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
}
}
}()
appendString(b, get())
}
func appendString(b *bytes.Buffer, s string) {
if needsQuoting(s) {
b.WriteString(strconv.Quote(s))
return
}
b.WriteString(s)
}
func needsQuoting(s string) bool {
for i := range len(s) {
c := s[i]
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
default:
return true
}
}
return false
}
-78
View File
@@ -1,78 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
t.Helper()
var out bytes.Buffer
c := newConsole(&out)
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
c.level.Set(slog.LevelDebug)
build(New(NewSlogHandler(&consoleHandler{c: c})))
return out.String()
}
func TestConsolePlain(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
l.With("msg", "clash", "empty", "").Info("")
})
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
"level=info empty= fields.msg=clash\n", line)
}
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
opts := TextOptions{
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
},
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
}
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
}
func TestConsoleColored(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
l.With("job", "j").Warn("careful")
})
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
assert.Equal(t, "level=info msg=x\n", plain)
}
func TestConsoleLevelAndOutput(t *testing.T) {
var out bytes.Buffer
c := newConsole(&out)
h := &consoleHandler{c: c}
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
c.level.Set(slog.LevelError)
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
rec.AddAttrs(slog.String("c", "3"))
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
assert.Contains(t, out.String(), "a=1 c=3")
}
-91
View File
@@ -1,91 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"io"
"log/slog"
"os"
"runtime"
"sync/atomic"
"github.com/sirupsen/logrus"
)
// TextOptions describes how the process-wide Logger writes a record as text.
type TextOptions struct {
DisableColors bool
DisableTimestamp bool
FullTimestamp bool
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
CallerPrettyfier func(*runtime.Frame) (function, file string)
// Filter rewrites each formatted line, for example to mask secrets.
Filter func(line []byte) []byte
}
var (
std atomic.Pointer[Logger]
stdState = newConsole(os.Stderr)
)
func init() {
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
}
// Default returns the process-wide Logger.
func Default() *Logger { return std.Load() }
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
// initial console Logger.
func SetDefault(l *Logger) { std.Store(l) }
// SetLevel sets the minimum level of the initial process-wide Logger and of what libraries log through logrus.
func SetLevel(level Level) {
stdState.level.Set(slog.Level(level))
logrus.SetLevel(toLogrusLevel(level))
}
// GetLevel returns the minimum level of the initial process-wide Logger.
func GetLevel() Level { return Level(stdState.level.Level()) }
// SetOutput sets where the initial process-wide Logger writes.
func SetOutput(w io.Writer) { stdState.setOutput(w) }
// Output returns where the initial process-wide Logger writes.
func Output() io.Writer {
stdState.mu.Lock()
defer stdState.mu.Unlock()
return stdState.out
}
// SetTextFormat sets the text format of the initial process-wide Logger.
func SetTextFormat(o TextOptions) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.opts = o
}
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
func SetReportCaller(on bool) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.reportCaller = on
logrus.SetReportCaller(on)
}
func Enabled(level Level) bool { return Default().Enabled(level) }
func With(args ...any) *Logger { return Default().With(args...) }
func WithError(err error) *Logger { return Default().WithError(err) }
func Tracef(format string, args ...any) { Default().logf(LevelTrace, format, args) }
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
func Trace(args ...any) { Default().log(LevelTrace, args) }
func Debug(args ...any) { Default().log(LevelDebug, args) }
func Info(args ...any) { Default().log(LevelInfo, args) }
func Warn(args ...any) { Default().log(LevelWarn, args) }
func Error(args ...any) { Default().log(LevelError, args) }
-114
View File
@@ -1,114 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"errors"
"slices"
"time"
)
// Attr is a structured key/value pair attached to a log record.
type Attr struct {
Key string
Value any
}
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
// Logger.With are accumulated by the Handler.
type Record struct {
Time time.Time
Level Level
Message string
PC uintptr // program counter of the logging call, 0 if unknown
Attrs []Attr
}
// Attr returns the value of the last attribute named key.
func (r Record) Attr(key string) (any, bool) {
for _, a := range slices.Backward(r.Attrs) {
if a.Key == key {
return a.Value, true
}
}
return nil, false
}
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
type Handler interface {
Enabled(ctx context.Context, level Level) bool
Handle(ctx context.Context, r Record) error
// WithAttrs returns a Handler that adds attrs to every record it handles.
WithAttrs(attrs []Attr) Handler
}
type multiHandler []Handler
// NewMultiHandler fans every record out to all handlers that have its level enabled.
func NewMultiHandler(handlers ...Handler) Handler {
return multiHandler(handlers)
}
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
for _, h := range m {
if h.Enabled(ctx, level) {
return true
}
}
return false
}
func (m multiHandler) Handle(ctx context.Context, r Record) error {
var errs []error
for _, h := range m {
if h.Enabled(ctx, r.Level) {
errs = append(errs, h.Handle(ctx, r))
}
}
return errors.Join(errs...)
}
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
out := make(multiHandler, len(m))
for i, h := range m {
out[i] = h.WithAttrs(attrs)
}
return out
}
type discardHandler struct{}
// DiscardHandler drops every record.
var DiscardHandler Handler = discardHandler{}
func (discardHandler) Enabled(context.Context, Level) bool { return false }
func (discardHandler) Handle(context.Context, Record) error { return nil }
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
type levelHandler struct {
h Handler
minLevel Level
}
// NewLevelHandler passes only records at or above minLevel on to h.
func NewLevelHandler(h Handler, minLevel Level) Handler {
return levelHandler{h: h, minLevel: minLevel}
}
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
return level >= l.minLevel && l.h.Enabled(ctx, level)
}
func (l levelHandler) Handle(ctx context.Context, r Record) error {
if r.Level < l.minLevel {
return nil
}
return l.h.Handle(ctx, r)
}
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
}
-53
View File
@@ -1,53 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"fmt"
"strings"
)
// Level is a log severity. The values match log/slog, with Trace added below Debug.
type Level int
const (
LevelTrace Level = -8
LevelDebug Level = -4
LevelInfo Level = 0
LevelWarn Level = 4
LevelError Level = 8
)
func (l Level) String() string {
switch {
case l <= LevelTrace:
return "trace"
case l <= LevelDebug:
return "debug"
case l <= LevelInfo:
return "info"
case l <= LevelWarn:
return "warn"
default:
return "error"
}
}
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
// the Logger has no level above it.
func ParseLevel(s string) (Level, error) {
switch strings.ToLower(s) {
case "trace":
return LevelTrace, nil
case "debug":
return LevelDebug, nil
case "info":
return LevelInfo, nil
case "warn", "warning":
return LevelWarn, nil
case "error", "fatal", "panic":
return LevelError, nil
}
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
}
-106
View File
@@ -1,106 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"fmt"
"runtime"
"time"
)
const badKey = "!BADKEY"
// Logger writes printf-style messages with structured attributes to a Handler.
type Logger struct {
h Handler
ctx context.Context
}
func New(h Handler) *Logger {
return &Logger{h: h, ctx: context.Background()}
}
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
func (l *Logger) With(args ...any) *Logger {
if len(args) == 0 {
return l
}
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
}
func (l *Logger) WithError(err error) *Logger {
return l.With("error", err)
}
// WithContext returns a Logger that passes ctx to its Handler.
func (l *Logger) WithContext(ctx context.Context) *Logger {
return &Logger{h: l.h, ctx: ctx}
}
func (l *Logger) Enabled(level Level) bool {
return l.h.Enabled(l.ctx, level)
}
func (l *Logger) Tracef(format string, args ...any) { l.logf(LevelTrace, format, args) }
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
// Trace and the methods below format their arguments like fmt.Sprint.
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
func (l *Logger) logf(level Level, format string, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
}
func (l *Logger) log(level Level, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprint(args...))
}
// emit drops the Handler error, as a logger has nowhere to report it.
func (l *Logger) emit(pc uintptr, level Level, msg string) {
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
}
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
func callerPC(skip int) uintptr {
var pcs [1]uintptr
runtime.Callers(skip, pcs[:])
return pcs[0]
}
func argsToAttrs(args []any) []Attr {
attrs := make([]Attr, 0, len(args)/2+1)
for len(args) > 0 {
switch k := args[0].(type) {
case Attr:
attrs = append(attrs, k)
args = args[1:]
case string:
if len(args) == 1 {
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = nil
} else {
attrs = append(attrs, Attr{Key: k, Value: args[1]})
args = args[2:]
}
default:
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = args[1:]
}
}
return attrs
}
-203
View File
@@ -1,203 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging_test
import (
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseLevel(t *testing.T) {
for in, want := range map[string]logging.Level{
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
} {
got, err := logging.ParseLevel(in)
require.NoError(t, err, in)
assert.Equal(t, want, got, in)
}
_, err := logging.ParseLevel("loud")
require.Error(t, err)
assert.Equal(t, "warn", logging.LevelWarn.String())
}
type countingStringer struct{ n *int }
func (c countingStringer) String() string { *c.n++; return "x" }
func TestLogger(t *testing.T) {
h := logtest.NewWithLevel(logging.LevelTrace)
l := logging.New(h)
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
l.Info("a", 1, 2, "b")
l.With("k").With(7).Debug("x")
recs := h.AllRecords()
require.Len(t, recs, 3)
assert.Equal(t, "step 2", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
assert.False(t, recs[0].Time.IsZero())
job, _ := recs[0].Attr("job")
assert.Equal(t, "j1", job)
errv, _ := recs[0].Attr("error")
require.EqualError(t, errv.(error), "boom")
assert.Equal(t, "a1 2b", recs[1].Message) // fmt.Sprint spacing
bad, _ := recs[2].Attr("!BADKEY")
assert.Equal(t, 7, bad)
h.Reset()
assert.Nil(t, h.LastRecord())
}
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
n := 0
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
l.Debugf("%v", countingStringer{&n})
assert.Zero(t, n)
assert.False(t, l.Enabled(logging.LevelInfo))
}
func TestMultiHandler(t *testing.T) {
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
l.Info("i")
l.Error("e")
assert.Len(t, all.AllRecords(), 2)
require.Len(t, warnOnly.AllRecords(), 1)
v, _ := warnOnly.LastRecord().Attr("a")
assert.Equal(t, 1, v)
}
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
t.Helper()
require.NotZero(t, pc)
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
return f
}
func TestCallerPC(t *testing.T) {
l, h := logtest.NewLogger()
l.Infof("method")
l.Info("method")
prev := logging.Default()
t.Cleanup(func() { logging.SetDefault(prev) })
logging.SetDefault(l)
logging.Infof("global")
logging.Error("global")
require.Len(t, h.AllRecords(), 4)
for _, r := range h.AllRecords() {
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
}
}
func TestDefaultLevel(t *testing.T) {
prev := logging.GetLevel()
t.Cleanup(func() { logging.SetLevel(prev) })
logging.SetLevel(logging.LevelWarn)
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
assert.True(t, logging.Enabled(logging.LevelError))
assert.False(t, logging.Enabled(logging.LevelInfo))
}
func TestSlogHandler(t *testing.T) {
var buf strings.Builder
slogger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
l := logging.New(logging.NewSlogHandler(slogger.Handler())).With("job", "j1")
l.Tracef("dropped")
assert.False(t, l.Enabled(logging.LevelTrace))
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
out := buf.String()
assert.Contains(t, out, "level=ERROR")
assert.Contains(t, out, `msg="fail x"`)
assert.Contains(t, out, "job=j1")
assert.Contains(t, out, "error=boom")
}
func TestLogrusBridge(t *testing.T) {
prevLevel, prevDefault := logging.GetLevel(), logging.Default()
t.Cleanup(func() {
logging.SetLevel(prevLevel)
logging.SetReportCaller(false)
logging.SetDefault(prevDefault)
})
logging.SetLevel(logging.LevelInfo)
assert.False(t, logrus.IsLevelEnabled(logrus.DebugLevel))
logging.SetLevel(logging.LevelTrace)
logging.SetReportCaller(true)
h := logtest.NewWithLevel(logging.LevelTrace)
logging.SetDefault(logging.New(h))
logrus.WithField("lib", "docker").Warnf("from a library %d", 1)
_, _, line, _ := runtime.Caller(0)
logrus.Debug("below")
logrus.Trace("lowest")
recs := h.AllRecords()
require.Len(t, recs, 3)
assert.Equal(t, "from a library 1", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
v, _ := recs[0].Attr("lib")
assert.Equal(t, "docker", v)
assert.Equal(t, logging.LevelDebug, recs[1].Level)
f := frameOf(t, recs[1].PC)
assert.True(t, strings.HasSuffix(f.Function, "TestLogrusBridge"), f.Function)
assert.Equal(t, line+1, f.Line)
assert.Equal(t, logging.LevelTrace, recs[2].Level)
h = logtest.New() // logrus passes Debug, the Logger does not
logging.SetDefault(logging.New(h))
logrus.Debug("dropped")
assert.Empty(t, h.AllRecords())
}
func TestSlogBridge(t *testing.T) {
h := logtest.SetDefault(t)
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
_, _, line, _ := runtime.Caller(0)
slog.Debug("dropped")
recs := h.AllRecords()
require.Len(t, recs, 1)
assert.Equal(t, "from a library", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
lib, _ := recs[0].Attr("lib")
assert.Equal(t, "actionslib", lib)
n, _ := recs[0].Attr("n")
assert.EqualValues(t, 1, n)
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
}
func TestLogrusBridgeIntact(t *testing.T) {
assert.True(t, logging.LogrusBridgeIntact())
prev := logrus.StandardLogger().Out
t.Cleanup(func() { logrus.SetOutput(prev) })
logrus.SetOutput(&strings.Builder{}) // what a dependency doing this would do
assert.False(t, logging.LogrusBridgeIntact())
logrus.SetOutput(prev)
assert.True(t, logging.LogrusBridgeIntact())
prevLevel := logrus.GetLevel()
t.Cleanup(func() { logrus.SetLevel(prevLevel) })
logrus.SetLevel(logrus.PanicLevel)
assert.False(t, logging.LogrusBridgeIntact())
}
-102
View File
@@ -1,102 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"io"
"github.com/sirupsen/logrus"
)
// Libraries the runner depends on, such as docker/cli and moby/go-archive, log through
// logrus's global logger. Hand what they log to the process-wide Logger, so it follows the same
// level, format and secret masking as the rest of the output.
func init() {
logrus.SetOutput(io.Discard)
logrus.SetFormatter(discardFormatter{})
logrus.SetLevel(toLogrusLevel(GetLevel()))
logrus.AddHook(bridgeHook{})
}
type discardFormatter struct{}
func (discardFormatter) Format(*logrus.Entry) ([]byte, error) { return nil, nil }
type bridgeHook struct{}
func (bridgeHook) Levels() []logrus.Level { return logrus.AllLevels }
func (bridgeHook) Fire(e *logrus.Entry) error {
l := Default()
level := fromLogrusLevel(e.Level)
ctx := e.Context
if ctx == nil {
ctx = context.Background()
}
if !l.h.Enabled(ctx, level) {
return nil
}
attrs := make([]Attr, 0, len(e.Data))
for k, v := range e.Data {
attrs = append(attrs, Attr{Key: k, Value: v})
}
r := Record{Time: e.Time, Level: level, Message: e.Message, Attrs: attrs}
if e.Caller != nil {
r.PC = e.Caller.PC + 1 // Frame.PC is one below the return address Record.PC holds
}
return l.h.Handle(ctx, r)
}
func toLogrusLevel(l Level) logrus.Level {
switch {
case l <= LevelTrace:
return logrus.TraceLevel
case l <= LevelDebug:
return logrus.DebugLevel
case l <= LevelInfo:
return logrus.InfoLevel
case l <= LevelWarn:
return logrus.WarnLevel
default:
return logrus.ErrorLevel
}
}
func fromLogrusLevel(l logrus.Level) Level {
switch l {
case logrus.TraceLevel:
return LevelTrace
case logrus.DebugLevel:
return LevelDebug
case logrus.InfoLevel:
return LevelInfo
case logrus.WarnLevel:
return LevelWarn
default:
return LevelError
}
}
// LogrusBridgeIntact reports whether logrus's global logger is still set up as the bridge needs,
// discarding its own output and following the process-wide level. A dependency that calls
// logrus.SetOutput or SetLevel breaks that, and its lines then show up twice, or at the wrong level.
func LogrusBridgeIntact() bool {
std := logrus.StandardLogger()
if std.Out != io.Discard || std.GetLevel() != toLogrusLevel(GetLevel()) {
return false
}
for _, level := range logrus.AllLevels {
found := false
for _, h := range std.Hooks[level] {
if _, ok := h.(bridgeHook); ok {
found = true
}
}
if !found {
return false
}
}
return true
}
-89
View File
@@ -1,89 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Package logtest provides a Handler that keeps records in memory for assertions.
package logtest
import (
"context"
"slices"
"sync"
"testing"
"gitea.com/gitea/runner/act/common/logging"
)
type store struct {
mu sync.Mutex
records []logging.Record
}
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
type Handler struct {
store *store
attrs []logging.Attr
minLevel logging.Level
}
// New records from LevelInfo up, like the logger of logrus's hooks/test.
func New() *Handler {
return NewWithLevel(logging.LevelInfo)
}
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
func NewLogger() (*logging.Logger, *Handler) {
h := New()
return logging.New(h), h
}
func NewWithLevel(minLevel logging.Level) *Handler {
return &Handler{store: &store{}, minLevel: minLevel}
}
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
h.store.mu.Lock()
defer h.store.mu.Unlock()
h.store.records = append(h.store.records, r)
return nil
}
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
}
func (h *Handler) AllRecords() []logging.Record {
h.store.mu.Lock()
defer h.store.mu.Unlock()
return slices.Clone(h.store.records)
}
// LastRecord returns nil when nothing was logged.
func (h *Handler) LastRecord() *logging.Record {
h.store.mu.Lock()
defer h.store.mu.Unlock()
if len(h.store.records) == 0 {
return nil
}
r := h.store.records[len(h.store.records)-1]
return &r
}
func (h *Handler) Reset() {
h.store.mu.Lock()
defer h.store.mu.Unlock()
h.store.records = nil
}
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
func SetDefault(t testing.TB) *Handler {
t.Helper()
prev := logging.Default()
logger, h := NewLogger()
logging.SetDefault(logger)
t.Cleanup(func() { logging.SetDefault(prev) })
return h
}
-81
View File
@@ -1,81 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"log/slog"
"slices"
)
func init() {
slog.SetDefault(slog.New(slogBridge{}))
}
// slogBridge hands what libraries log through slog's global logger to the process-wide Logger.
type slogBridge struct {
attrs []Attr
}
func (b slogBridge) Enabled(ctx context.Context, level slog.Level) bool {
return Default().h.Enabled(ctx, Level(level))
}
func (b slogBridge) Handle(ctx context.Context, r slog.Record) error {
h := Default().h
if len(b.attrs) > 0 {
h = h.WithAttrs(b.attrs)
}
attrs := make([]Attr, 0, r.NumAttrs())
r.Attrs(func(a slog.Attr) bool {
attrs = append(attrs, fromSlogAttr(a))
return true
})
return h.Handle(ctx, Record{Time: r.Time, Level: Level(r.Level), Message: r.Message, PC: r.PC, Attrs: attrs})
}
func (b slogBridge) WithAttrs(attrs []slog.Attr) slog.Handler {
out := slices.Clone(b.attrs)
for _, a := range attrs {
out = append(out, fromSlogAttr(a))
}
return slogBridge{attrs: out}
}
func (b slogBridge) WithGroup(string) slog.Handler { return b }
func fromSlogAttr(a slog.Attr) Attr {
return Attr{Key: a.Key, Value: a.Value.Resolve().Any()}
}
type slogHandler struct {
h slog.Handler
}
// NewSlogHandler writes to h. Levels map one to one, as Level uses slog's values.
func NewSlogHandler(h slog.Handler) Handler {
return slogHandler{h: h}
}
func (s slogHandler) Enabled(ctx context.Context, level Level) bool {
return s.h.Enabled(ctx, slog.Level(level))
}
func (s slogHandler) Handle(ctx context.Context, r Record) error {
rec := slog.NewRecord(r.Time, slog.Level(r.Level), r.Message, r.PC)
rec.AddAttrs(toSlogAttrs(r.Attrs)...)
return s.h.Handle(ctx, rec)
}
func (s slogHandler) WithAttrs(attrs []Attr) Handler {
return slogHandler{h: s.h.WithAttrs(toSlogAttrs(attrs))}
}
func toSlogAttrs(attrs []Attr) []slog.Attr {
out := make([]slog.Attr, len(attrs))
for i, a := range attrs {
out[i] = slog.Any(a.Key, a.Value)
}
return out
}
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !windows
package logging
import (
"io"
"os"
"golang.org/x/term"
)
func checkIfTerminal(w io.Writer) bool {
if f, ok := w.(*os.File); ok {
return term.IsTerminal(int(f.Fd()))
}
return false
}
-27
View File
@@ -1,27 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build windows
package logging
import (
"io"
"os"
"golang.org/x/sys/windows"
)
// checkIfTerminal also turns on virtual terminal processing, which the console needs to show colors.
func checkIfTerminal(w io.Writer) bool {
f, ok := w.(*os.File)
if !ok {
return false
}
handle := windows.Handle(f.Fd())
var mode uint32
if err := windows.GetConsoleMode(handle, &mode); err != nil {
return false
}
return windows.SetConsoleMode(handle, mode|windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == nil
}
+34 -40
View File
@@ -10,11 +10,6 @@ import (
"strings"
)
type interfaceIP struct {
net.IP
net.Interface
}
// GetOutboundIP returns an outbound IP address of this machine.
// It tries to access the internet and returns the local IP address of the connection.
// If the machine cannot access the internet, it returns a preferred IP address from network interfaces.
@@ -31,7 +26,11 @@ func GetOutboundIP() net.IP {
// So the machine cannot access the internet. Pick an IP address from network interfaces.
if ifs, err := net.Interfaces(); err == nil {
var ips []interfaceIP
type IP struct {
net.IP
net.Interface
}
var ips []IP
for _, i := range ifs {
if addrs, err := i.Addrs(); err == nil {
for _, addr := range addrs {
@@ -43,45 +42,40 @@ func GetOutboundIP() net.IP {
ip = v.IP
}
if ip.IsGlobalUnicast() {
ips = append(ips, interfaceIP{ip, i})
ips = append(ips, IP{ip, i})
}
}
}
}
return preferredIP(ips)
if len(ips) > 1 {
sort.Slice(ips, func(i, j int) bool {
ifi := ips[i].Interface
ifj := ips[j].Interface
// ethernet is preferred
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
return vi
}
ipi := ips[i].IP
ipj := ips[j].IP
// IPv4 is preferred
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
return vi
}
// en0 is preferred to en1
if ifi.Name != ifj.Name {
return ifi.Name < ifj.Name
}
// fallback
return ipi.String() < ipj.String()
})
return ips[0].IP
}
}
return nil
}
func preferredIP(ips []interfaceIP) net.IP {
if len(ips) == 0 {
return nil
}
sort.Slice(ips, func(i, j int) bool {
ifi := ips[i].Interface
ifj := ips[j].Interface
// ethernet is preferred
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
return vi
}
ipi := ips[i].IP
ipj := ips[j].IP
// IPv4 is preferred
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
return vi
}
// en0 is preferred to en1
if ifi.Name != ifj.Name {
return ifi.Name < ifj.Name
}
// fallback
return ipi.String() < ipj.String()
})
return ips[0].IP
}
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPreferredIP(t *testing.T) {
assert.Nil(t, preferredIP(nil))
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{{IP: net.ParseIP("2001:db8::1")}}))
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{
{IP: net.ParseIP("192.0.2.1"), Name: "wlan0"},
{IP: net.ParseIP("2001:db8::1"), Name: "eth0"},
}))
}
+1 -37
View File
@@ -9,10 +9,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strings"
"sync"
"sync/atomic"
"gitea.com/gitea/runner/act/common"
@@ -75,31 +71,10 @@ const (
HealthUnhealthy = string(container.Unhealthy)
)
// marks what a runner creates for its jobs, telling its leftovers from another runner's
const runnerUUIDLabel = "com.gitea.runner.uuid"
// ErrContainerNotFound reports a container the daemon no longer knows. Its text is a
// fragment, missingContainerError composes it into the message every operation shares.
var ErrContainerNotFound = errors.New("does not exist")
// DockerProxy is a job's docker socket, fronting the daemon's for the job's lifetime.
type DockerProxy struct {
Socket string
close func(context.Context) error
closeOnce sync.Once
closeErr error
mounts atomic.Value
}
func (p *DockerProxy) SetMounts(mounts map[string]string) {
p.mounts.Store(mounts)
}
func (p *DockerProxy) Close(ctx context.Context) error {
p.closeOnce.Do(func() { p.closeErr = p.close(ctx) })
return p.closeErr
}
// Info is a snapshot of a container, as of one inspect.
type Info struct {
ID string
@@ -109,24 +84,13 @@ type Info struct {
// HealthOutput is the last healthcheck probe's output.
HealthOutput string
Ports map[string]string // container port ("5432") to the host port it is published on
Mounts map[string]string // container path to its source on the daemon
}
// DaemonPath maps target to its daemon path through the nearest mount with a source, empty without one.
func (info *Info) DaemonPath(target string) string {
for dir := target; dir != "/" && dir != "."; dir = path.Dir(dir) {
if source := info.Mounts[dir]; source != "" {
return path.Join(source, strings.TrimPrefix(target, dir))
}
}
return ""
}
// Container for managing docker run containers
type Container interface {
Create(capAdd, capDrop []string) common.Executor
Copy(destPath string, files ...*FileEntry) common.Executor
CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor
CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor
GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
Inspect(ctx context.Context) (*Info, error)
DumpLogs(ctx context.Context) error
+1 -7
View File
@@ -4,10 +4,9 @@
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts.go with:
// * appended with license information
// * regexp and loader.ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
// * jsontext in place of json.Compact for the seccomp profile
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
//
// docker/cli is licensed under the Apache License, Version 2.0.
@@ -153,7 +152,6 @@ type containerOptions struct {
runtime string
autoRemove bool
init bool
umask opts.UmaskOpt
annotations *opts.MapOpts
Image string
@@ -328,9 +326,6 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
flags.Var(&copts.shmSize, "shm-size", "Size of /dev/shm")
flags.StringVar(&copts.utsMode, "uts", "", "UTS namespace to use")
flags.StringVar(&copts.runtime, "runtime", "", "Runtime to use for this container")
flags.Var(&copts.umask, "umask", "Set umask for the container")
flags.SetAnnotation("umask", "version", []string{"1.56"})
flags.SetAnnotation("umask", "ostype", []string{"linux"})
flags.BoolVar(&copts.init, "init", false, "Run an init inside the container that forwards signals and reaps processes")
flags.SetAnnotation("init", "version", []string{"1.25"})
@@ -726,7 +721,6 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
MaskedPaths: maskedPaths,
ReadonlyPaths: readonlyPaths,
Annotations: copts.annotations.GetAll(),
Umask: copts.umask.Value(),
}
if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() {
+1 -14
View File
@@ -2,7 +2,7 @@
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts_test.go with:
// * appended with license information
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
//
@@ -167,19 +167,6 @@ func TestParseRunLinks(t *testing.T) {
}
}
func TestParseRunWithoutUmask(t *testing.T) {
_, hostConfig, _, err := parseRun([]string{"ubuntu", "bash"})
assert.NilError(t, err)
assert.Assert(t, hostConfig.Umask == nil)
}
func TestParseRunUmask(t *testing.T) {
_, hostConfig, _, err := parseRun([]string{"--umask", "0022", "ubuntu", "bash"})
assert.NilError(t, err)
assert.Assert(t, hostConfig.Umask != nil)
assert.Equal(t, uint32(0o22), *hostConfig.Umask)
}
func TestParseRunAttach(t *testing.T) {
tests := []struct {
input string
+6 -17
View File
@@ -8,10 +8,10 @@ package container
import (
"context"
"fmt"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/client"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// ImageExistsLocally returns a boolean indicating if an image with the
@@ -30,23 +30,11 @@ func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool,
return false, err
}
if platform == "" || platform == "any" {
if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform {
return true, nil
}
requested, err := parsePlatform(platform)
if err != nil {
return false, err
}
return platformMatches(requested, &specs.Platform{OS: inspectImage.Os, Architecture: inspectImage.Architecture, Variant: inspectImage.Variant}), nil
}
func platformMatches(requested, image *specs.Platform) bool {
variant := image.Variant
if variant == "" {
variant = map[string]string{"amd64": "v1", "arm": "v7", "arm64": "v8"}[image.Architecture]
}
return image.OS == requested.OS && image.Architecture == requested.Architecture && (requested.Variant == "" || requested.Variant == variant)
return false, nil
}
// RemoveImage removes image from local store, the function is used to run different
@@ -58,13 +46,14 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
}
defer cli.Close()
if _, err := cli.ImageInspect(ctx, imageName); cerrdefs.IsNotFound(err) {
inspectImage, err := cli.ImageInspect(ctx, imageName)
if cerrdefs.IsNotFound(err) {
return false, nil
} else if err != nil {
return false, err
}
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
if _, err = cli.ImageRemove(ctx, inspectImage.ID, client.ImageRemoveOptions{
Force: force,
PruneChildren: pruneChildren,
}); err != nil {
+4 -4
View File
@@ -5,20 +5,20 @@
package container
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func init() {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
}
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
@@ -41,7 +41,7 @@ func buildScratchImage(t *testing.T, platform string) string {
func TestImageExistsLocally(t *testing.T) {
requireDocker(t)
ctx := t.Context()
ctx := context.Background()
// a non-existent image is reported absent
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"errors"
"io"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
)
type dockerMessage struct {
@@ -26,7 +26,7 @@ type dockerMessage struct {
Progress string `json:"progress"`
}
func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isError bool) error {
func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error {
if dockerResponse == nil {
return nil
}
@@ -77,7 +77,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
return nil
}
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...any) {
if isError {
logger.Errorf(format, args...)
} else {
+43 -151
View File
@@ -10,20 +10,21 @@ import (
"context"
"errors"
"fmt"
"net/netip"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
)
const (
networkCreateAttempts = 3
networkCreateRetryDelay = time.Second
// marks the networks a runner creates for its jobs, so it can tell its own leftovers from
// those of another runner sharing the daemon
runnerUUIDLabel = "com.gitea.runner.uuid"
)
// RemoveOrphanNetworks removes the networks this runner created for jobs whose teardown did
@@ -31,17 +32,17 @@ const (
// had an endpoint on it at the time. Each one holds a subnet of the daemon's address pool
// until it is removed. Networks created after createdBefore are left alone, so a job starting
// while this runs cannot lose the network it has created but not yet attached a container to.
func RemoveOrphanNetworks(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
}
defer cli.Close()
return removeOrphanNetworks(ctx, cli, runnerUUID, detach, createdBefore)
return removeOrphanNetworks(ctx, cli, runnerUUID, createdBefore)
}
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID, detach string, createdBefore time.Time) error {
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID string, createdBefore time.Time) error {
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID),
})
@@ -50,7 +51,6 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID,
}
var errs []error
networks:
for _, n := range networks.Items {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if err != nil {
@@ -59,18 +59,9 @@ networks:
}
// the emptiness check, not the label, is what keeps a live job of another process
// sharing this registration safe
if len(result.Network.Containers) > 1 || result.Network.Created.After(createdBefore) {
if len(result.Network.Containers) != 0 || result.Network.Created.After(createdBefore) {
continue
}
for containerID := range result.Network.Containers {
if detach == "" || !strings.HasPrefix(containerID, detach) {
continue networks
}
if err := disconnectNetwork(ctx, cli, n.ID, containerID); err != nil {
errs = append(errs, fmt.Errorf("failed to disconnect container from network %s: %w", n.Name, err))
continue networks
}
}
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
continue
@@ -87,54 +78,45 @@ func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExec
return err
}
defer cli.Close()
return createDockerNetwork(ctx, cli, name, opts)
}
}
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
// Only create the network if it doesn't exist
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
var options map[string]string
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
common.Logger(ctx).Debugf("Network %v exists", name)
return nil
}
// the daemon's mtu only reaches its default bridge, Docker's default is left out to keep default-network-opts
if mtu := n.Options["com.docker.network.driver.mtu"]; n.Name == network.NetworkBridge && mtu != "" && mtu != "1500" {
options = map[string]string{"com.docker.network.driver.mtu": mtu}
}
}
for i := range networkCreateAttempts {
if i > 0 {
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
}
}
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Scope: "local",
EnableIPv4: opts.EnableIPv4,
EnableIPv6: opts.EnableIPv6,
Options: options,
Labels: runnerLabels(opts.RunnerUUID),
}); err == nil {
return nil
}
if !isAddressPoolExhausted(err) {
// Only create the network if it doesn't exist
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
common.Logger(ctx).Debugf("Network %v exists", name)
return nil
}
}
for i := range networkCreateAttempts {
if i > 0 {
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
}
}
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Scope: "local",
EnableIPv4: opts.EnableIPv4,
EnableIPv6: opts.EnableIPv6,
Labels: runnerLabels(opts.RunnerUUID),
}); err == nil {
return nil
}
if !isAddressPoolExhausted(err) {
return err
}
}
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
}
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
}
func runnerLabels(runnerUUID string) map[string]string {
@@ -152,50 +134,6 @@ func isAddressPoolExhausted(err error) bool {
strings.Contains(msg, "could not find an available, non-overlapping IPv4 address pool among the defaults") // docker 24 and older
}
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
if err := connectNetwork(ctx, cli, networkName, container); err != nil {
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
}
return nil
}
}
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
_, err := cli.NetworkConnect(ctx, networkName, client.NetworkConnectOptions{
Container: container,
EndpointConfig: &network.EndpointSettings{Aliases: []string{container}},
})
if err != nil && (strings.Contains(err.Error(), "already exists in network") || strings.Contains(err.Error(), "already connected")) {
return nil
}
return err
}
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return disconnectNetwork(ctx, cli, networkName, container)
}
}
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
_, err := cli.NetworkDisconnect(ctx, networkName, client.NetworkDisconnectOptions{Container: container, Force: true})
if cerrdefs.IsNotFound(err) || (err != nil && strings.Contains(err.Error(), "is not connected")) {
return nil
}
return err
}
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
@@ -234,49 +172,3 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return errors.Join(errs...)
}
}
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
func IsolatedCacheContainer(ctx context.Context, addr netip.Addr, jobNetwork string) (string, error) {
cli, err := GetDockerClient(ctx)
if err != nil {
return "", err
}
defer cli.Close()
return isolatedCacheContainer(ctx, cli, addr, jobNetwork)
}
func isolatedCacheContainer(ctx context.Context, cli client.APIClient, addr netip.Addr, jobNetwork string) (string, error) {
if jobNetwork == "host" || strings.HasPrefix(jobNetwork, "container:") {
return "", nil
}
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{})
if err != nil {
return "", err
}
var holderID, holderContainer string
for _, summary := range containers.Items {
if summary.NetworkSettings == nil {
continue
}
for _, endpoint := range summary.NetworkSettings.Networks {
if endpoint != nil && (endpoint.IPAddress == addr || endpoint.GlobalIPv6Address == addr) {
holderID = endpoint.NetworkID
holderContainer = summary.ID[:12]
}
}
}
if holderID == "" {
return "", nil
}
holder, err := cli.NetworkInspect(ctx, holderID, client.NetworkInspectOptions{})
if err != nil || holder.Network.Driver != "bridge" {
return "", err
}
if jobNetwork != "" {
job, err := cli.NetworkInspect(ctx, jobNetwork, client.NetworkInspectOptions{})
if err != nil || job.Network.ID == holderID {
return "", err
}
}
return holderContainer, nil
}
+6 -110
View File
@@ -6,118 +6,27 @@ package container
import (
"context"
"errors"
"net/netip"
"testing"
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIsolatedCacheContainer(t *testing.T) {
ctx := t.Context()
cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
client := &mockDockerClient{}
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
ID: "0123456789abcdef",
NetworkSettings: &container.NetworkSettingsSummary{Networks: map[string]*network.EndpointSettings{
"compose": {NetworkID: "c0ffee", IPAddress: cache},
"lan": {NetworkID: "beef", IPAddress: lan},
}},
}}}, nil)
for ref, inspected := range map[string]network.Network{
"c0ffee": {ID: "c0ffee", Name: "compose", Driver: "bridge"},
"beef": {ID: "beef", Name: "lan", Driver: "macvlan"},
"c0f": {ID: "c0ffee"},
"jobs": {ID: "f00d"},
} {
client.On("NetworkInspect", ctx, ref, mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: inspected}}, nil)
}
check := func(addr netip.Addr, jobNetwork, want string) {
got, err := isolatedCacheContainer(ctx, client, addr, jobNetwork)
require.NoError(t, err)
assert.Equal(t, want, got, jobNetwork)
}
check(cache, "", "0123456789ab")
check(cache, "jobs", "0123456789ab")
check(cache, "c0f", "")
check(cache, "host", "")
check(lan, "", "")
check(netip.MustParseAddr("10.0.0.5"), "", "")
}
func TestIsAddressPoolExhausted(t *testing.T) {
assert.True(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("Error response from daemon: all predefined address pools have been fully subnetted")))
assert.True(t, isAddressPoolExhausted(errors.New("could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network")))
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
}
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
ctx := t.Context()
for _, tc := range []struct {
bridgeOptions, want map[string]string
}{
{
map[string]string{"com.docker.network.bridge.host_binding_ipv4": "127.0.0.1", "com.docker.network.driver.mtu": "1450"},
map[string]string{"com.docker.network.driver.mtu": "1450"},
},
{map[string]string{"com.docker.network.driver.mtu": "1500"}, nil},
} {
client := &mockDockerClient{}
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).
Return(mobyclient.NetworkListResult{Items: []network.Summary{{Name: network.NetworkBridge, Options: tc.bridgeOptions}}}, nil)
client.On("NetworkCreate", ctx, "job-network", mobyclient.NetworkCreateOptions{Driver: "bridge", Scope: "local", Options: tc.want}).
Return(mobyclient.NetworkCreateResult{}, nil)
require.NoError(t, createDockerNetwork(ctx, client, "job-network", NewDockerNetworkCreateExecutorInput{}))
client.AssertExpectations(t)
}
}
func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
}
func (m *mockDockerClient) NetworkDisconnect(ctx context.Context, id string, opts mobyclient.NetworkDisconnectOptions) (mobyclient.NetworkDisconnectResult, error) {
return mobyclient.NetworkDisconnectResult{}, m.Called(ctx, id, opts).Error(0)
}
func TestConnectAndDisconnectNetwork(t *testing.T) {
ctx := t.Context()
connect := func(answer error) error {
client := &mockDockerClient{}
client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
Container: "0123456789ab",
EndpointConfig: &network.EndpointSettings{Aliases: []string{"0123456789ab"}},
}).Return(answer).Once()
return connectNetwork(ctx, client, "jobs", "0123456789ab")
}
require.NoError(t, connect(nil))
require.NoError(t, connect(errors.New("endpoint with name runner already exists in network jobs")))
require.NoError(t, connect(errors.New("container runner already connected to network jobs")))
require.ErrorIs(t, connect(context.Canceled), context.Canceled)
disconnect := func(answer error) error {
client := &mockDockerClient{}
client.On("NetworkDisconnect", ctx, "jobs", mobyclient.NetworkDisconnectOptions{
Container: "0123456789ab", Force: true,
}).Return(answer).Once()
return disconnectNetwork(ctx, client, "jobs", "0123456789ab")
}
require.NoError(t, disconnect(cerrdefs.ErrNotFound))
require.NoError(t, disconnect(errors.New("container runner is not connected to the network jobs")))
require.ErrorIs(t, disconnect(context.Canceled), context.Canceled)
}
// Of this runner's networks, only the ones nothing but its own cache container is attached to
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
// Of this runner's networks, only the ones nothing is attached to and old enough to predate
// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
// test on its own, since testify has no expectation to match it against.
func TestRemoveOrphanNetworks(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
client := &mockDockerClient{}
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
@@ -126,29 +35,16 @@ func TestRemoveOrphanNetworks(t *testing.T) {
{ID: "orphan"},
{ID: "busy"},
{ID: "starting"},
{ID: "cached"},
{ID: "shared"},
}}, nil)
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil)
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
Network: network.Network{Created: cutoff.Add(time.Second)},
Containers: map[string]network.EndpointResource{"0123456789abcdef": {}},
}}, nil)
client.On("NetworkInspect", ctx, "cached", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil)
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
Return(nil).Once()
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
client.AssertExpectations(t)
}
-23
View File
@@ -61,26 +61,3 @@ func TestParsePlatform(t *testing.T) {
})
}
}
func TestPlatformMatches(t *testing.T) {
for _, testcase := range []struct {
requested, image string
want bool
}{
{"linux/amd64", "linux/amd64", true},
{"linux/amd64/v1", "linux/amd64", true},
{"linux/arm64", "linux/amd64", false},
{"linux/arm64/v8", "linux/arm64", true},
{"linux/arm/v7", "linux/arm/v7", true},
{"linux/arm", "linux/arm/v7", true},
{"linux/arm/v6", "linux/arm/v7", false},
{"linux/arm", "linux/arm/v6", true},
{"windows/arm/v7", "linux/arm/v7", false},
} {
requested, err := parsePlatform(testcase.requested)
require.NoError(t, err)
image, err := parsePlatform(testcase.image)
require.NoError(t, err)
assert.Equal(t, testcase.want, platformMatches(requested, image), "%s on %s", testcase.requested, testcase.image)
}
}
-587
View File
@@ -1,587 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"mime"
"net"
"net/http"
"net/http/httputil"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"slices"
"strings"
"sync"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
)
const (
jobLabel = "com.gitea.runner.job."
maxCreateBody = 8 << 20
dockerProxyProbeTimeout = 5 * time.Second
)
var (
createPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers|networks|volumes)/create$`)
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
)
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
return nil
}
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
defer cancel()
cli, err := GetDockerClient(probeCtx)
if err != nil {
return nil
}
defer cli.Close()
daemonSocket, ok := strings.CutPrefix(cli.DaemonHost(), "unix://")
if !ok {
return nil
}
if info, err := os.Stat(daemonSocket); err != nil || info.Mode()&os.ModeSocket == 0 {
return nil
}
dir, daemonDir := runnerContainerWorkdir(probeCtx, cli)
seen := daemonDir != ""
if !seen {
if dir, err = filepath.Abs(os.TempDir()); err == nil {
daemonDir = dir
seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
}
}
if err != nil {
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
return nil
}
if !seen {
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
return nil
}
if ctx.Err() != nil {
return nil
}
proxy, err := StartDockerProxy(daemonSocket, dir, job)
if err != nil {
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
return nil
}
proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
return proxy
}
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
workdir, err := os.Getwd()
hostname, hostnameErr := os.Hostname()
if err != nil || hostnameErr != nil {
return "", ""
}
self, err := cli.ContainerInspect(ctx, hostname, client.ContainerInspectOptions{})
if err != nil {
return "", ""
}
if daemonDir = containerInfoFromInspect(self.Container).DaemonPath(workdir); daemonDir == "" {
return "", ""
}
if seen, _ := containerSeesMarker(ctx, cli, workdir, self.Container.ID, workdir); !seen { // the hostname may name another container
return "", ""
}
return workdir, daemonDir
}
func containerSeesMarker(ctx context.Context, cli client.APIClient, dir, id, containerDir string) (bool, error) {
marker, err := os.CreateTemp(dir, "gitea-runner-probe-")
if err != nil {
return false, err
}
defer func() {
if err := os.Remove(marker.Name()); err != nil {
common.Logger(ctx).Warnf("removing the docker proxy probe marker failed: %v", err)
}
}()
if err := marker.Close(); err != nil {
return false, err
}
_, err = cli.ContainerStatPath(ctx, id, client.ContainerStatPathOptions{Path: path.Join(containerDir, filepath.Base(marker.Name()))})
if cerrdefs.IsNotFound(err) {
return false, nil
}
return err == nil, err
}
// daemonSeesDir reports whether the daemon opens the files the runner writes in dir by their path in daemonDir,
// which is what a job's proxy socket mounted from there needs.
func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir string) (bool, error) {
images, err := cli.ImageList(ctx, client.ImageListOptions{})
if err != nil {
return false, err
}
if len(images.Items) == 0 {
return false, errors.New("no image available for the docker proxy probe")
}
created, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
}},
})
if cerrdefs.IsInvalidArgument(err) {
return false, nil
}
if err != nil {
return false, err
}
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
defer cancel()
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
return false, fmt.Errorf("removing the docker proxy probe container failed: %w", removeErr)
}
return seen, err
}
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
info, err := os.Stat(daemonSocket)
if err != nil {
return nil, err
}
if info.Mode()&os.ModeSocket == 0 {
return nil, errors.New("docker daemon path is not a Unix socket")
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
instance, err := os.MkdirTemp(dir, "p-")
if err != nil {
return nil, err
}
socket := filepath.Join(instance, "docker.sock")
listener, err := net.Listen("unix", socket)
if err != nil {
return nil, errors.Join(err, os.RemoveAll(instance))
}
if err := copyDockerSocketPermissions(daemonSocket, socket, info); err != nil {
return nil, errors.Join(err, listener.Close(), os.RemoveAll(instance))
}
dial := func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", daemonSocket)
}
transport := &http.Transport{DialContext: dial}
forward := &httputil.ReverseProxy{
Rewrite: func(r *httputil.ProxyRequest) {
r.Out.URL.Scheme = "http"
r.Out.URL.Host = "docker"
},
Transport: transport,
}
proxy := &DockerProxy{Socket: socket}
streams, cancelStreams := context.WithCancel(context.Background())
creates, cancelCreates := context.WithCancel(context.Background())
var admission sync.Mutex
var handlers sync.WaitGroup
server := &http.Server{ReadHeaderTimeout: 30 * time.Second, ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
return context.WithValue(ctx, dockerProxyConnKey{}, conn)
}, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
admission.Lock()
if streams.Err() != nil {
admission.Unlock()
http.Error(w, "docker proxy is closing", http.StatusServiceUnavailable)
return
}
handlers.Add(1)
admission.Unlock()
defer handlers.Done()
creating := r.Method == http.MethodPost && createPath.MatchString(r.URL.Path)
parent, lifetime := r.Context(), streams
if creating {
parent, lifetime = context.WithoutCancel(parent), creates
}
ctx, cancel := context.WithCancel(parent)
defer cancel()
stop := context.AfterFunc(lifetime, func() {
cancel()
if !creating {
if conn, ok := parent.Value(dockerProxyConnKey{}).(net.Conn); ok {
_ = conn.Close()
}
}
})
defer stop()
r = r.WithContext(ctx)
if creating {
r.Body = http.MaxBytesReader(w, r.Body, maxCreateBody)
mounts, _ := proxy.mounts.Load().(map[string]string)
if err := rewriteCreate(r, job, mounts); err != nil {
status := http.StatusBadRequest
if _, ok := errors.AsType[*http.MaxBytesError](err); ok {
status = http.StatusRequestEntityTooLarge
}
http.Error(w, err.Error(), status)
return
}
} else if r.Method == http.MethodPost && rawStreamPath.MatchString(r.URL.Path) {
tunnel(w, r, dial, forward)
return
}
forward.ServeHTTP(w, r)
})}
served := make(chan struct{})
go func() {
defer close(served)
_ = server.Serve(listener)
}()
proxy.close = func(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
admission.Lock()
listenerErr := listener.Close()
cancelStreams()
admission.Unlock()
<-served
shutdownErr := server.Shutdown(ctx)
cancelCreates()
serverErr := server.Close()
handlers.Wait()
transport.CloseIdleConnections()
return errors.Join(ctx.Err(), listenerErr, shutdownErr, serverErr, os.RemoveAll(instance))
}
return proxy, nil
}
func rewriteCreate(r *http.Request, job string, mounts map[string]string) error {
body, err := io.ReadAll(r.Body)
if err != nil {
return err
}
if len(bytes.TrimSpace(body)) == 0 {
body = []byte("{}")
}
var fields map[string]json.RawMessage
var config struct{ Labels map[string]string }
if err := json.Unmarshal(body, &fields); err != nil {
return fmt.Errorf("invalid create request: %w", err)
}
if err := json.Unmarshal(body, &config); err != nil {
return fmt.Errorf("invalid create labels: %w", err)
}
if fields == nil {
fields = make(map[string]json.RawMessage)
}
if len(mounts) > 0 && !hasAmbiguousFields(body) {
translateBinds(fields, createPath.FindStringSubmatch(r.URL.Path)[2], mounts)
}
maps.DeleteFunc(fields, func(name string, _ json.RawMessage) bool {
return strings.EqualFold(name, "Labels")
})
if config.Labels == nil {
config.Labels = make(map[string]string)
}
config.Labels[jobLabel+job] = "" // per-job key, so nested proxies keep each other's
if fields["Labels"], err = json.Marshal(config.Labels); err != nil {
return err
}
if body, err = json.Marshal(fields); err != nil {
return err
}
r.Body = io.NopCloser(bytes.NewReader(body))
r.ContentLength = int64(len(body))
r.TransferEncoding = nil
return nil
}
func translateBinds(fields map[string]json.RawMessage, kind string, mounts map[string]string) {
cleanedSource := func(source string) string {
cleaned := path.Clean(source)
if target := jobMount(cleaned, mounts); mounts[target] != "" {
return mounts[target] + cleaned[len(target):]
}
return source
}
spelledSource := func(source string) string { // dockerd checks these as spelled
target := jobMount(path.Clean(source), mounts)
if rest, spelled := strings.CutPrefix(source, target); mounts[target] != "" && spelled && (rest == "" || rest[0] == '/') && filepath.IsLocal("."+rest) {
return mounts[target] + rest
}
return cleanedSource(source)
}
switch kind {
case "volumes":
var driver string
var options map[string]any
decodeField(fields, "Driver", &driver)
if key := decodeField(fields, "DriverOpts", &options); options != nil {
translateDevice(driver, options, spelledSource)
if encoded, err := json.Marshal(options); err == nil {
fields[key] = encoded
}
}
case "containers":
var hostConfig map[string]any
key := decodeField(fields, "HostConfig", &hostConfig)
binds, _ := field(hostConfig, "Binds").([]any)
for i, bind := range binds {
bind, _ := bind.(string)
source, target, _ := strings.Cut(bind, ":")
if translated := cleanedSource(source); strings.HasPrefix(target, "/") && !strings.Contains(translated, ":") {
binds[i] = translated + ":" + target
}
}
specs, _ := field(hostConfig, "Mounts").([]any)
for _, spec := range specs {
spec, _ := spec.(map[string]any)
switch field(spec, "Type") {
case "bind":
if source, ok := field(spec, "Source").(string); ok {
spec["Source"] = spelledSource(source)
}
case "volume":
volumeOptions, _ := field(spec, "VolumeOptions").(map[string]any)
driverConfig, _ := field(volumeOptions, "DriverConfig").(map[string]any)
translateDevice(field(driverConfig, "Name"), field(driverConfig, "Options"), spelledSource)
}
}
if encoded, err := json.Marshal(hostConfig); err == nil && hostConfig != nil {
fields[key] = encoded
}
}
}
func decodeField(fields map[string]json.RawMessage, name string, value any) string {
for key, raw := range fields {
if strings.EqualFold(key, name) {
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
_ = decoder.Decode(value) // wrong types stay unset for dockerd to reject
return key
}
}
return ""
}
func translateDevice(driver, options any, translate func(string) string) {
optionMap, _ := options.(map[string]any)
device, ok := optionMap["device"].(string)
flags, _ := optionMap["o"].(string)
tokens := strings.Split(flags, ",")
local := driver == nil || driver == "" || driver == "local"
if ok && local && (slices.Contains(tokens, "bind") || slices.Contains(tokens, "rbind")) && !slices.Contains(tokens, "remount") {
optionMap["device"] = translate(device)
}
}
// field also renames the matched key to name.
func field(object map[string]any, name string) any {
for key, value := range object {
if strings.EqualFold(key, name) {
delete(object, key)
object[name] = value
return value
}
}
return nil
}
var (
requestFields = []string{"hostconfig", "driver", "driveropts"}
asciiFolds = strings.NewReplacer("ſ", "s", "K", "k") // the non-ASCII runes strings.EqualFold matches to ASCII letters
)
// dockerd settles repeated names by order, which re-encoding loses.
func hasAmbiguousFields(body []byte) bool {
type frame struct {
names map[string]bool
key string
expectKey bool
nested bool
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.UseNumber()
stack := []*frame{{}}
for {
token, err := decoder.Token()
if err != nil {
return !errors.Is(err, io.EOF)
}
top := stack[len(stack)-1]
if name, ok := token.(string); ok && top.expectKey {
top.key, top.expectKey = strings.ToLower(asciiFolds.Replace(name)), false
if top.names[top.key] && (top.nested || slices.Contains(requestFields, top.key)) {
return true
}
top.names[top.key] = true
continue
}
top.expectKey = top.names != nil
nested := top.nested || len(stack) == 2 && slices.Contains(requestFields, top.key)
switch token {
case json.Delim('{'):
stack = append(stack, &frame{names: map[string]bool{}, expectKey: true, nested: nested})
case json.Delim('['):
stack = append(stack, &frame{nested: nested})
case json.Delim('}'), json.Delim(']'):
stack = stack[:len(stack)-1]
}
}
}
// jobMount returns "" also for a path already naming a daemon source.
func jobMount(source string, mounts map[string]string) string {
target := ""
for destination, daemonSource := range mounts {
if daemonSource != "" && (source == daemonSource || strings.HasPrefix(source, daemonSource+"/")) {
return ""
}
if len(destination) > len(target) && (source == destination || strings.HasPrefix(source, destination+"/")) {
target = destination
}
}
return target
}
type dockerProxyConnKey struct{}
type dockerProxyResponse struct {
response *http.Response
}
func (r dockerProxyResponse) RoundTrip(_ *http.Request) (*http.Response, error) {
return r.response, nil
}
// tunnel splices attach and exec streams, which the daemon hijacks with or without an HTTP upgrade
func tunnel(w http.ResponseWriter, r *http.Request, dial func(context.Context, string, string) (net.Conn, error), forward *httputil.ReverseProxy) {
upstream, err := dial(r.Context(), "", "")
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
defer upstream.Close()
stop := context.AfterFunc(r.Context(), func() { _ = upstream.Close() })
defer stop()
if err := r.Write(upstream); err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
reader := bufio.NewReader(upstream)
var response *http.Response
for {
response, err = http.ReadResponse(reader, r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
if response.StatusCode >= 200 || response.StatusCode == http.StatusSwitchingProtocols {
break
}
maps.Copy(w.Header(), response.Header)
w.WriteHeader(response.StatusCode)
clear(w.Header())
_ = response.Body.Close()
}
defer func() {
_ = upstream.Close()
_ = response.Body.Close()
}()
mediaType, _, _ := mime.ParseMediaType(response.Header.Get("Content-Type"))
if response.StatusCode != http.StatusSwitchingProtocols && (response.StatusCode != http.StatusOK || mediaType != "application/vnd.docker.raw-stream") {
ordinary := *forward
ordinary.Transport = dockerProxyResponse{response: response}
ordinary.ServeHTTP(w, r)
return
}
downstream, buffered, err := http.NewResponseController(w).Hijack()
if err != nil {
return
}
defer downstream.Close()
if _, err := fmt.Fprintf(buffered, "%s %s\r\n", response.Proto, response.Status); err != nil {
return
}
if err := response.Header.Write(buffered); err != nil {
return
}
if _, err := buffered.WriteString("\r\n"); err != nil {
return
}
if err := buffered.Flush(); err != nil {
return
}
done := make(chan struct{})
go func() {
defer close(done)
if _, err := io.Copy(upstream, io.MultiReader(io.LimitReader(buffered, int64(buffered.Reader.Buffered())), downstream)); err != nil { // Bypass net/http after the prefix so stdin EOF preserves output.
_ = upstream.Close()
} else if writer, ok := upstream.(interface{ CloseWrite() error }); ok {
_ = writer.CloseWrite()
} else {
_ = upstream.Close()
}
}()
_, _ = io.Copy(downstream, reader)
_ = downstream.Close()
_ = upstream.Close()
<-done
}
func RemoveDockerJobResources(ctx context.Context, job string) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return removeLabelled(ctx, cli, job)
}
func removeLabelled(ctx context.Context, cli client.APIClient, job string) error {
logger := common.Logger(ctx)
filters := make(client.Filters).Add("label", jobLabel+job)
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
errs := []error{err}
for _, c := range containers.Items {
logger.Infof("removing container %s the job left behind", strings.TrimPrefix(strings.Join(c.Names, ","), "/"))
errs = append(errs, (&containerReference{cli: cli, id: c.ID}).remove()(ctx))
}
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{Filters: filters})
errs = append(errs, err)
for _, n := range networks.Items {
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); n.Scope == "swarm" && cerrdefs.IsInvalidArgument(err) { // swarm refuses while a service or its tasks use it
logger.Infof("keeping network %s, a swarm service still uses it", n.Name)
} else if err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
}
}
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{Filters: filters})
errs = append(errs, err)
for _, v := range volumes.Items {
if _, err := cli.VolumeRemove(ctx, v.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", v.Name, err))
}
}
return errors.Join(errs...)
}
-546
View File
@@ -1,546 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"archive/tar"
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"testing"
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/api/types/volume"
mobyclient "github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func (m *mockDockerClient) VolumeList(ctx context.Context, opts mobyclient.VolumeListOptions) (mobyclient.VolumeListResult, error) {
args := m.Called(ctx, opts)
return args.Get(0).(mobyclient.VolumeListResult), args.Error(1)
}
func (m *mockDockerClient) VolumeRemove(ctx context.Context, id string, opts mobyclient.VolumeRemoveOptions) (mobyclient.VolumeRemoveResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.VolumeRemoveResult), args.Error(1)
}
// unix socket paths are limited to about a hundred bytes, which the default macOS TMPDIR exceeds
func shortTempDir(t testing.TB) string {
t.Helper()
t.Setenv("TMPDIR", "/tmp")
return t.TempDir()
}
func daemonSocketPath(t testing.TB, cli mobyclient.APIClient) string {
t.Helper()
host := cli.DaemonHost()
if !strings.HasPrefix(host, "unix://") {
t.Skipf("skipping: daemon at %s is not a unix socket", host)
}
return strings.TrimPrefix(host, "unix://")
}
func TestDockerProxy(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Unix socket ownership is unavailable on Windows")
}
bodies := make(chan []byte, 8)
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
streamsDone := make(chan struct{}, 2)
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
listener, err := net.Listen("unix", daemonSocket)
require.NoError(t, err)
daemon := &http.Server{ReadHeaderTimeout: time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/create"):
body, _ := io.ReadAll(r.Body)
if r.URL.RawQuery == "wait" {
close(createStarted)
<-releaseCreate
}
bodies <- body
w.WriteHeader(http.StatusCreated)
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
if r.URL.Path == "/exec/error/start" {
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
w.WriteHeader(http.StatusBadRequest)
}
_, _ = w.Write([]byte("ordinary"))
case rawStreamPath.MatchString(r.URL.Path) || r.URL.Path == "/session":
_, _ = io.Copy(io.Discard, r.Body)
conn, buffered, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer conn.Close()
if r.Header.Get("Upgrade") != "" {
_, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\nready\n"))
} else {
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\nready\n"))
}
input, _ := io.ReadAll(buffered)
_, _ = conn.Write(append([]byte("final:"), input...))
streamsDone <- struct{}{}
default:
w.Header().Set("Api-Version", "1.47")
_, _ = w.Write([]byte("OK " + r.Method + " " + r.URL.Path))
}
})}
go func() { _ = daemon.Serve(listener) }()
t.Cleanup(func() { _ = daemon.Close() })
proxy, err := StartDockerProxy(daemonSocket, shortTempDir(t), "job-1")
require.NoError(t, err)
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
proxy.SetMounts(map[string]string{"/workspace/o/r": "/volumes/job/_data", "/workspace/o/r/tmp": "", "/var/run/docker.sock": "/tmp/p/docker.sock", "/volumes": "/daemon/volumes"})
client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", proxy.Socket)
}}}
t.Cleanup(client.CloseIdleConnections)
dialProxy := func(t *testing.T) *net.UnixConn {
t.Helper()
conn, err := net.DialUnix("unix", nil, &net.UnixAddr{Name: proxy.Socket, Net: "unix"})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.Close() })
require.NoError(t, conn.SetDeadline(time.Now().Add(5*time.Second)))
return conn
}
startStream := func(t *testing.T, path, upgrade string) (*net.UnixConn, *bufio.Reader) {
t.Helper()
conn := dialProxy(t)
_, err := fmt.Fprintf(conn, "POST %s HTTP/1.1\r\nHost: docker\r\n%sContent-Length: 2\r\n\r\n{}input", path, upgrade)
require.NoError(t, err)
reader := bufio.NewReader(conn)
resp, err := http.ReadResponse(reader, nil)
require.NoError(t, err)
t.Cleanup(func() {
_ = conn.Close()
_ = resp.Body.Close()
})
prefix, err := reader.ReadString('\n')
require.NoError(t, err)
require.Equal(t, "ready\n", prefix)
return conn, reader
}
t.Run("labels creates", func(t *testing.T) {
for _, testCase := range []struct{ path, body, want string }{
{
"/v1.47/containers/create", `{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":""}}`,
`{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":"","com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Binds":["/workspace/o/r:/src:ro","/workspace/o/r/../r/sub/:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2"}}`,
`{"HostConfig":{"Binds":["/volumes/job/_data:/src:ro","/volumes/job/_data/sub:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2","com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/var/run/docker.sock"},{"type":"bind","source":"/workspace/o/r/sub/","target":"/m"},{"Type":"bind","Source":"/workspace/o/r/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/workspace/o/r/../r","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]}}`,
`{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/tmp/p/docker.sock","Target":"/var/run/docker.sock"},{"Type":"bind","Source":"/volumes/job/_data/sub/","target":"/m"},{"Type":"bind","Source":"/volumes/job/_data/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/volumes/job/_data","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/workspace/o/r/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]}}`,
`{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/volumes/job/_data/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null}`,
`{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null,"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/networks/create", `{"Name":"n"}`,
`{"Name":"n","Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Name":"v","labels":null,"DriverOpts":{"type":"none","o":"rbind,ro","device":"/workspace/o/r/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
`{"Name":"v","Labels":{"com.gitea.runner.job.job-1":""},"DriverOpts":{"type":"none","o":"rbind,ro","device":"/volumes/job/_data/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
},
{
"/volumes/create", `{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"}}`,
`{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"}}`,
`{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", "",
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", "null",
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Labels":{"discard":"1"},"labels":null,"LABELS":{"own":"1"},"LABELS":{"extra":"2"}}`,
`{"Labels":{"own":"1","extra":"2","com.gitea.runner.job.job-1":""}}`,
},
} {
resp, err := client.Post("http://docker"+testCase.path, "application/json", strings.NewReader(testCase.body))
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, testCase.want, string(<-bodies), testCase.body)
}
})
t.Run("passes other requests through", func(t *testing.T) {
resp, err := client.Get("http://docker/v1.47/_ping")
require.NoError(t, err)
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
assert.Equal(t, "1.47", resp.Header.Get("Api-Version"))
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
})
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
for _, stream := range []struct{ path, upgrade string }{
{path: "/v1.47/exec/abc/start"},
{path: "/containers/abc/attach", upgrade: "Connection: Upgrade\r\nUpgrade: tcp\r\n"},
} {
conn, reader := startStream(t, stream.path, stream.upgrade)
require.NoError(t, conn.CloseWrite())
output, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "final:input", string(output))
<-streamsDone
}
})
t.Run("detached and error responses retain keepalive label injection", func(t *testing.T) {
conn := dialProxy(t)
reader := bufio.NewReader(conn)
for endpoint, status := range map[string]int{"detached": http.StatusOK, "error": http.StatusBadRequest} {
_, err := fmt.Fprintf(conn, "POST /exec/%s/start HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}", endpoint)
require.NoError(t, err)
resp, err := http.ReadResponse(reader, nil)
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, status, resp.StatusCode)
assert.Equal(t, "ordinary", string(body))
_, err = io.WriteString(conn, "POST /volumes/create HTTP/1.1\r\nHost: docker\r\nContent-Length: 0\r\n\r\n")
require.NoError(t, err)
resp, err = http.ReadResponse(reader, nil)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
}
})
t.Run("close joins streams and admitted create", func(t *testing.T) {
t.Cleanup(func() { close(releaseCreate) })
_, raw := startStream(t, "/exec/live/start", "")
_, upgraded := startStream(t, "/session", "Connection: Upgrade\r\nUpgrade: tcp\r\n")
creator := dialProxy(t)
_, err := io.WriteString(creator, "POST /volumes/create?wait HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}")
require.NoError(t, err)
<-createStarted
require.NoError(t, creator.CloseWrite())
closed := make(chan error, 1)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
go func() { closed <- proxy.Close(ctx) }()
for _, reader := range []*bufio.Reader{raw, upgraded} {
_, err := reader.ReadByte()
require.ErrorIs(t, err, io.EOF)
<-streamsDone
}
select {
case err := <-closed:
t.Fatalf("Close returned before admitted create settled: %v", err)
default:
}
releaseCreate <- struct{}{}
resp, err := http.ReadResponse(bufio.NewReader(creator), nil)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
require.NoError(t, <-closed)
})
}
func TestRemoveLabelledRemovesContainersNetworksAndVolumes(t *testing.T) {
containerFailure := errors.New("container removal failed")
listFailure := errors.New("listing failed")
volumeFailure := errors.New("volume removal failed")
ctx := t.Context()
filters := make(mobyclient.Filters).Add("label", jobLabel+"job-1")
cli := &mockDockerClient{}
cli.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true, Filters: filters}).
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil).Once()
cli.On("ContainerKill", ctx, "c1", mock.Anything).Return(mobyclient.ContainerKillResult{}, nil).Once()
cli.On("ContainerRemove", ctx, "c1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
Return(mobyclient.ContainerRemoveResult{}, containerFailure).Once()
cli.On("NetworkList", ctx, mobyclient.NetworkListOptions{Filters: filters}).
Return(mobyclient.NetworkListResult{Items: []network.Summary{{ID: "n1", Name: "stack_default", Scope: "swarm"}}}, listFailure).Once()
cli.On("NetworkRemove", ctx, "n1", mobyclient.NetworkRemoveOptions{}).Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrInvalidArgument).Once()
cli.On("VolumeList", ctx, mobyclient.VolumeListOptions{Filters: filters}).
Return(mobyclient.VolumeListResult{Items: []volume.Volume{{Name: "app_data"}}}, nil).Once()
cli.On("VolumeRemove", ctx, "app_data", mobyclient.VolumeRemoveOptions{}).Return(mobyclient.VolumeRemoveResult{}, volumeFailure).Once()
err := removeLabelled(ctx, cli, "job-1")
require.ErrorIs(t, err, containerFailure)
require.ErrorIs(t, err, listFailure)
require.ErrorIs(t, err, volumeFailure)
require.NotErrorIs(t, err, cerrdefs.ErrInvalidArgument)
require.ErrorContains(t, err, "failed to remove container c1")
require.ErrorContains(t, err, "failed to remove volume app_data")
cli.AssertExpectations(t)
}
func TestDockerProxyWithDaemon(t *testing.T) {
requireDocker(t)
ctx := t.Context()
cleanupCtx := context.WithoutCancel(ctx)
require.NoError(t, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
direct, err := GetDockerClient(ctx)
require.NoError(t, err)
defer direct.Close()
dir := shortTempDir(t)
seen, err := daemonSeesDir(ctx, direct, dir, dir)
require.NoError(t, err)
t.Logf("daemon sees the runner's filesystem: %v", seen)
seen, err = daemonSeesDir(ctx, direct, dir, shortTempDir(t))
require.NoError(t, err)
assert.False(t, seen)
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
require.NoError(t, err)
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
require.NoError(t, err)
defer viaProxy.Close()
created, err := viaProxy.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{
Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "300"}, Labels: map[string]string{"own": "1"}},
})
require.NoError(t, err)
t.Cleanup(func() {
_, _ = direct.ContainerRemove(cleanupCtx, created.ID, mobyclient.ContainerRemoveOptions{Force: true})
})
_, err = viaProxy.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
require.NoError(t, err)
net, err := viaProxy.NetworkCreate(ctx, job, mobyclient.NetworkCreateOptions{})
require.NoError(t, err)
t.Cleanup(func() { _, _ = direct.NetworkRemove(cleanupCtx, net.ID, mobyclient.NetworkRemoveOptions{}) })
_, err = viaProxy.VolumeCreate(ctx, mobyclient.VolumeCreateOptions{Name: job})
require.NoError(t, err)
t.Cleanup(func() { _, _ = direct.VolumeRemove(cleanupCtx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
inspected, err := direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
require.NoError(t, err)
assert.Subset(t, inspected.Container.Config.Labels, map[string]string{"own": "1", jobLabel + job: ""})
vol, err := direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
require.NoError(t, err)
assert.Contains(t, vol.Volume.Labels, jobLabel+job)
exec, err := viaProxy.ExecCreate(ctx, created.ID, mobyclient.ExecCreateOptions{Cmd: []string{"cat"}, AttachStdin: true, AttachStdout: true, TTY: true})
require.NoError(t, err)
attached, err := viaProxy.ExecAttach(ctx, exec.ID, mobyclient.ExecAttachOptions{TTY: true})
require.NoError(t, err)
_, err = attached.Conn.Write([]byte("hello\n"))
require.NoError(t, err)
echoed, err := attached.Reader.ReadString('\n')
require.NoError(t, err)
assert.Equal(t, "hello\r\n", echoed)
attached.Close()
require.NoError(t, proxy.Close(ctx))
require.NoError(t, RemoveDockerJobResources(ctx, job))
_, err = direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
_, err = direct.NetworkInspect(ctx, net.ID, mobyclient.NetworkInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
_, err = direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
}
func BenchmarkDockerProxy(b *testing.B) {
ctx := context.Background()
direct, err := GetDockerClient(ctx)
require.NoError(b, err)
defer direct.Close()
if _, err := direct.Ping(ctx, mobyclient.PingOptions{}); err != nil {
b.Skipf("docker daemon unreachable: %v", err)
}
proxy, err := StartDockerProxy(daemonSocketPath(b, direct), shortTempDir(b), "bench")
require.NoError(b, err)
defer func() { _ = proxy.Close(ctx) }()
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
require.NoError(b, err)
defer viaProxy.Close()
require.NoError(b, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
created, err := direct.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "600"}}})
require.NoError(b, err)
defer func() { _, _ = direct.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true}) }()
_, err = direct.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
require.NoError(b, err)
var archive bytes.Buffer
writer := tar.NewWriter(&archive)
payload := make([]byte, 64<<20)
require.NoError(b, writer.WriteHeader(&tar.Header{Name: "blob", Mode: 0o600, Size: int64(len(payload))}))
_, _ = writer.Write(payload)
require.NoError(b, writer.Close())
for name, cli := range map[string]mobyclient.APIClient{"direct": direct, "proxy": viaProxy} {
b.Run("ping/"+name, func(b *testing.B) {
for b.Loop() {
if _, err := cli.Ping(ctx, mobyclient.PingOptions{}); err != nil {
b.Fatal(err)
}
}
})
b.Run("copy64MiB/"+name, func(b *testing.B) {
b.SetBytes(int64(archive.Len()))
for b.Loop() {
_, err := cli.CopyToContainer(ctx, created.ID, mobyclient.CopyToContainerOptions{DestinationPath: "/tmp", Content: bytes.NewReader(archive.Bytes())})
if err != nil {
b.Fatal(err)
}
}
})
}
}
type probeClient struct {
mobyclient.APIClient
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
stat func(string) error
mounts []container.MountPoint
}
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
return mobyclient.ContainerInspectResult{Container: container.InspectResponse{Mounts: c.mounts}}, nil
}
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "probe-image"}}}, nil
}
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
return c.create(opts)
}
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
return c.remove(ctx, id, opts)
}
func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobyclient.ContainerStatPathOptions) (mobyclient.ContainerStatPathResult, error) {
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
}
func TestDaemonSeesDir(t *testing.T) {
dir := t.TempDir()
markers := make(map[string]bool)
for _, testCase := range []struct {
name string
daemonDir string
removeErr error
seen bool
removed bool
}{
{name: "cleanup after cancellation", daemonDir: dir, seen: true, removed: true},
{name: "daemon without the directory", daemonDir: filepath.Join(dir, "missing")},
{name: "daemon with a different directory at that path", daemonDir: t.TempDir(), removed: true},
{name: "cleanup error after cancellation", daemonDir: dir, removeErr: errors.New("cleanup failed"), removed: true},
} {
t.Run(testCase.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
removed := false
cli := &probeClient{
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
cancel()
if _, err := os.Stat(testCase.daemonDir); err != nil {
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
}
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
},
stat: func(path string) error {
marker := strings.TrimPrefix(path, "/gitea-runner-probe/")
assert.False(t, markers[marker])
markers[marker] = true
if _, err := os.Stat(filepath.Join(testCase.daemonDir, marker)); err != nil {
return cerrdefs.ErrNotFound
}
return nil
},
remove: func(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
removed = true
assert.Equal(t, "probe", id)
assert.Equal(t, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true}, opts)
require.NoError(t, ctx.Err())
_, bounded := ctx.Deadline()
assert.True(t, bounded)
return mobyclient.ContainerRemoveResult{}, testCase.removeErr
},
}
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
require.ErrorIs(t, err, testCase.removeErr)
assert.Equal(t, testCase.seen, seen)
assert.Equal(t, testCase.removed, removed)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
assert.Empty(t, entries)
})
}
}
func TestRunnerContainerWorkdir(t *testing.T) {
workdir := t.TempDir()
t.Chdir(workdir)
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
for _, testCase := range []struct {
name string
mount container.MountPoint
foreign bool
daemonDir string
}{
{name: "volume above the working directory", mount: volume, daemonDir: filepath.Join(volume.Source, filepath.Base(workdir))},
{name: "tmpfs", mount: container.MountPoint{Type: mount.TypeTmpfs, Destination: volume.Destination}},
{name: "hostname of another container", mount: volume, foreign: true},
} {
cli := &probeClient{
mounts: []container.MountPoint{testCase.mount},
stat: func(path string) error {
if _, err := os.Stat(path); err != nil || testCase.foreign {
return cerrdefs.ErrNotFound
}
return nil
},
}
dir, daemonDir := runnerContainerWorkdir(t.Context(), cli)
assert.Equal(t, testCase.daemonDir, daemonDir, testCase.name)
assert.Equal(t, testCase.daemonDir != "", dir == workdir, testCase.name)
}
entries, err := os.ReadDir(workdir)
require.NoError(t, err)
assert.Empty(t, entries)
}
-48
View File
@@ -1,48 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
package container
import (
"encoding/binary"
"errors"
"io/fs"
"math"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
)
func copyDockerSocketPermissions(daemonSocket, socket string, info os.FileInfo) error {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return errors.New("docker socket ownership is unavailable")
}
groupErr := os.Chown(socket, int(stat.Uid), int(stat.Gid))
if groupErr != nil && (!errors.Is(groupErr, fs.ErrPermission) || int(stat.Uid) != os.Geteuid()) {
return groupErr
}
if err := os.Chown(filepath.Dir(socket), int(stat.Uid), -1); err != nil {
return err
}
mode := uint16(info.Mode().Perm())
if err := os.Chmod(socket, fs.FileMode(mode)); err != nil || groupErr == nil {
return err
}
if size, err := unix.Getxattr(daemonSocket, "system.posix_acl_access", nil); err == nil && size > 0 {
return groupErr
}
owner, group, other := mode>>6, mode>>3&7, mode&7
acl := binary.LittleEndian.AppendUint32(nil, 2)
for _, entry := range []struct {
tag, perm uint16
id uint32
}{{1, owner, math.MaxUint32}, {4, other, math.MaxUint32}, {8, group, stat.Gid}, {16, group | other, math.MaxUint32}, {32, other, math.MaxUint32}} {
acl = binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint16(binary.LittleEndian.AppendUint16(acl, entry.tag), entry.perm), entry.id)
}
return unix.Setxattr(socket, "system.posix_acl_access", acl, 0) // names the group a rootless runner cannot chown to, its own group keeps what others had
}
-15
View File
@@ -1,15 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER
package container
import (
"errors"
"os"
)
func copyDockerSocketPermissions(_, _ string, _ os.FileInfo) error {
return errors.New("docker socket ownership cannot be preserved on Windows")
}
+7 -7
View File
@@ -5,20 +5,20 @@
package container
import (
"context"
"io"
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/docker/cli/cli/config"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
assert "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func init() {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
}
func TestCleanImage(t *testing.T) {
@@ -36,13 +36,13 @@ func TestCleanImage(t *testing.T) {
}
for _, table := range tables {
imageOut := cleanImage(t.Context(), table.imageIn)
imageOut := cleanImage(context.Background(), table.imageIn)
assert.Equal(t, table.imageOut, imageOut)
}
}
func TestGetImagePullOptions(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
orig := config.Dir()
t.Cleanup(func() { config.SetDir(orig) })
@@ -79,7 +79,7 @@ func TestIsPinnedImage(t *testing.T) {
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
// whichever of the two shapes the daemon used.
func TestLogDockerResponseError(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
streamErr := func(line string) error {
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
}
+223 -164
View File
@@ -7,14 +7,13 @@
package container
import (
"archive/tar"
"bytes"
"cmp"
"context"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"reflect"
"regexp"
@@ -25,14 +24,17 @@ import (
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/filecollector"
"dario.cat/mergo"
"github.com/bmatcuk/doublestar/v4"
cerrdefs "github.com/containerd/errdefs"
"github.com/docker/cli/cli/compose/loader"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/cli/opts"
"github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/gobwas/glob"
"github.com/joho/godotenv"
"github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
@@ -41,6 +43,7 @@ import (
"github.com/moby/moby/client"
"github.com/moby/moby/client/pkg/versions"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
// drainGracePeriod bounds how long we wait for an output-copy goroutine to
@@ -95,8 +98,8 @@ func (cr *containerReference) Start(attach bool) common.Executor {
cr.attach().IfBool(attach),
cr.start(),
cr.wait().IfBool(attach),
cr.tryReadID("-u", func(id int) { cr.UID = id }),
cr.tryReadID("-g", func(id int) { cr.GID = id }),
cr.tryReadUID(),
cr.tryReadGID(),
func(ctx context.Context) error {
// If this fails, then folders have wrong permissions on non root container
if cr.UID != 0 || cr.GID != 0 {
@@ -135,12 +138,12 @@ func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common.
).IfNot(common.Dryrun)
}
func (cr *containerReference) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
func (cr *containerReference) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return common.NewPipelineExecutor(
common.NewInfoExecutor("docker cp src=%s dst=%s", srcPath, destPath),
cr.connect(),
cr.find(),
cr.copyDir(destPath, srcPath, useGitIgnore, skipGitDir),
cr.copyDir(destPath, srcPath, useGitIgnore),
func(ctx context.Context) error {
// If this fails, then folders have wrong permissions on non root container
if cr.UID != 0 || cr.GID != 0 {
@@ -249,31 +252,6 @@ func containerInfoFromInspect(inspect container.InspectResponse) *Info {
ID: inspect.ID,
Health: HealthNone,
Ports: map[string]string{}, // an empty map, never null, in the expression context
Mounts: make(map[string]string, len(inspect.Mounts)),
}
for _, mountPoint := range inspect.Mounts {
info.Mounts[mountPoint.Destination] = mountPoint.Source
}
if hostConfig := inspect.HostConfig; hostConfig != nil { // Mounts omits --tmpfs targets and subpaths
for target := range hostConfig.Tmpfs {
info.Mounts[path.Clean(target)] = ""
}
for _, spec := range hostConfig.Mounts {
subpath := ""
if spec.VolumeOptions != nil {
subpath = spec.VolumeOptions.Subpath
} else if spec.ImageOptions != nil {
subpath = spec.ImageOptions.Subpath
}
if target := path.Clean(spec.Target); subpath != "" && info.Mounts[target] != "" {
info.Mounts[target] = path.Join(info.Mounts[target], subpath)
}
}
}
for _, target := range []string{"/etc/hosts", "/etc/hostname", "/etc/resolv.conf"} { // specific to the job's network namespace
if _, mounted := info.Mounts[target]; !mounted {
info.Mounts[target] = ""
}
}
if state := inspect.State; state != nil {
@@ -305,25 +283,7 @@ func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]stri
}
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
envMap := *env
return common.Executor(func(ctx context.Context) error {
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
if err != nil {
common.Logger(ctx).Error(err)
return fmt.Errorf("inspect container: %w", err)
}
if inspect.Container.Config == nil {
return nil
}
for key, value := range opts.ConvertKVStringsToMap(inspect.Container.Config.Env) {
if envMap[key] == "" {
envMap[key] = value
} else if key == "PATH" {
envMap[key] += ":" + value
}
}
return nil
}).IfNot(common.Dryrun)
return cr.extractFromImageEnv(env).IfNot(common.Dryrun)
}
func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor {
@@ -524,11 +484,14 @@ func (cr *containerReference) remove() common.Executor {
RemoveVolumes: true,
Force: true,
})
if cerrdefs.IsConflict(err) {
err = cr.waitForRemoval(ctx, idOrName)
}
if err != nil && !cerrdefs.IsNotFound(err) {
return fmt.Errorf("failed to remove container %s: %w", idOrName, err)
switch {
case cerrdefs.IsConflict(err):
// the daemon's own AutoRemove teardown is running, and it releases the volume
// references and the network endpoint only once it finishes
cr.waitForRemoval(ctx, idOrName)
case err != nil && !cerrdefs.IsNotFound(err):
logger.Error(fmt.Errorf("failed to remove container %s: %w", idOrName, err))
return nil // keep the id, the container is still there for a later Remove()
}
logger.Debugf("Removed container: %v", idOrName)
@@ -537,7 +500,7 @@ func (cr *containerReference) remove() common.Executor {
}
}
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) error {
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) {
// per container, against the one minute the post-job executor allows for the whole
// cleanup, so a job with several services can spend most of that budget here
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
@@ -547,13 +510,18 @@ func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName strin
Condition: container.WaitConditionRemoved,
})
select {
case result := <-waitResult.Result:
if result.Error != nil {
return errors.New(result.Error.Message)
}
return nil
case err := <-waitResult.Error:
return err
case <-waitResult.Result:
case <-waitResult.Error:
case <-ctx.Done():
// the client delivers the result over an unbuffered channel, so leave a receiver
// behind or its goroutine parks on the send for the lifetime of the process
go func() {
select {
case <-waitResult.Result:
case <-waitResult.Error:
}
}()
common.Logger(ctx).Warnf("Timed out waiting for the daemon to remove container %s, its volumes and network may be left behind", idOrName)
}
}
@@ -585,6 +553,20 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
}
// FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment.
// In the old fork version, the code is
// if len(copts.netMode.Value()) == 0 {
// if err = copts.netMode.Set("host"); err != nil {
// return nil, nil, fmt.Errorf("cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err)
// }
// }
// And it has been commented with:
// If a service container's network is set to `host`, the container will not be able to
// connect to the specified network created for the job container and the service containers.
// So comment out the following code.
// Not the if it's necessary to comment it in the new version,
// since it's cr.input.NetworkMode now.
if len(copts.netMode.Value()) == 0 {
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
return nil, nil, fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
@@ -593,8 +575,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
// If the `privileged` config has been disabled, `copts.privileged` need to be forced to false,
// even if the user specifies `--privileged` in the options string.
if !hostConfig.Privileged && copts.privileged {
logger.Warnf("container option %q is not allowed when privileged mode is disabled and will be ignored", "--privileged")
if !hostConfig.Privileged {
copts.privileged = false
}
@@ -630,7 +611,6 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
cr.sanitizeConfig(ctx, nil, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
overlayVolumes(hostConfig, containerConfig.HostConfig)
binds := hostConfig.Binds
mounts := hostConfig.Mounts
@@ -648,9 +628,6 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
if flags.Changed("network") || flags.Changed("net") {
logger.Warn("--network and --net in the options will be ignored.")
}
if len(cr.input.Entrypoint) != 0 && flags.Changed("entrypoint") {
logger.Warn("--entrypoint in the options will be ignored.")
}
hostConfig.NetworkMode = networkMode
logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
@@ -680,11 +657,17 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
ExposedPorts: exposedPorts,
Tty: input.AllocatePTY,
}
// For Gitea, reduce log noise
// logger.Debugf("Common container.Config ==> %+v", config)
if len(input.Cmd) != 0 {
config.Cmd = input.Cmd
}
if len(input.Entrypoint) != 0 {
config.Entrypoint = input.Entrypoint
}
mounts := make([]mount.Mount, 0)
for mountSource, mountTarget := range input.Mounts {
mounts = append(mounts, mount.Mount{
@@ -719,20 +702,20 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
PortBindings: portBindings,
AutoRemove: input.AutoRemove,
}
// For Gitea, reduce log noise
// logger.Debugf("Common container.HostConfig ==> %+v", hostConfig)
config, hostConfig, err = cr.mergeContainerConfigs(ctx, config, hostConfig)
if err != nil {
return err
}
if len(input.Entrypoint) != 0 {
config.Entrypoint = input.Entrypoint
}
// For Gitea
config, hostConfig = cr.sanitizeConfig(ctx, config, hostConfig)
var networkingConfig *network.NetworkingConfig
// For Gitea, reduce log noise
// logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases)
n := hostConfig.NetworkMode
// IsUserDefined and IsHost are broken on windows
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
@@ -765,6 +748,44 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
}
}
func (cr *containerReference) extractFromImageEnv(env *map[string]string) common.Executor {
envMap := *env
return func(ctx context.Context) error {
logger := common.Logger(ctx)
inspect, err := cr.cli.ImageInspect(ctx, cr.input.Image)
if err != nil {
logger.Error(err)
return fmt.Errorf("inspect image: %w", err)
}
if inspect.Config == nil {
return nil
}
imageEnv, err := godotenv.Unmarshal(strings.Join(inspect.Config.Env, "\n"))
if err != nil {
logger.Error(err)
return fmt.Errorf("unmarshal image env: %w", err)
}
for k, v := range imageEnv {
if k == "PATH" {
if envMap[k] == "" {
envMap[k] = v
} else {
envMap[k] += `:` + v
}
} else if envMap[k] == "" {
envMap[k] = v
}
}
env = &envMap
return nil
}
}
func (cr *containerReference) exec(cmd []string, env map[string]string, user, workdir string) common.Executor {
return func(ctx context.Context) error {
if cr.id == "" {
@@ -782,6 +803,11 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
logger.Debugf("Exec command '%s'", cmd)
isTerminal := cr.input.AllocatePTY
envList := make([]string, 0)
for k, v := range env {
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
}
var wd string
if workdir != "" {
if strings.HasPrefix(workdir, "/") {
@@ -798,7 +824,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
User: user,
Cmd: cmd,
WorkingDir: wd,
Env: getEnvListFromMap(env),
Env: envList,
TTY: isTerminal,
AttachStderr: true,
AttachStdout: true,
@@ -815,7 +841,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
}
defer resp.Close()
err = cr.waitForCommand(ctx, resp.HijackedResponse)
err = cr.waitForCommand(ctx, resp.HijackedResponse, idResp, user, workdir)
if err != nil {
return err
}
@@ -865,7 +891,15 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe
}
}
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse) error {
func (cr *containerReference) tryReadUID() common.Executor {
return cr.tryReadID("-u", func(id int) { cr.UID = id })
}
func (cr *containerReference) tryReadGID() common.Executor {
return cr.tryReadID("-g", func(id int) { cr.GID = id })
}
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse, _ client.ExecCreateResult, _, _ string) error {
logger := common.Logger(ctx)
// Buffered so the copy goroutine never blocks on send if the grace-period
@@ -906,8 +940,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
}
}
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
dstPath = path.Clean(dstPath)
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error {
if cr.id == "" {
return cr.missingContainerError("copy directory to %s", dstPath)
@@ -929,11 +962,41 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
logger.Error(err)
}
}(tarFile)
extractionRoot := "/"
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
tw := tar.NewWriter(tarFile)
srcPrefix := filepath.Dir(srcPath)
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
srcPrefix += string(filepath.Separator)
}
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
var ignorer gitignore.Matcher
if useGitIgnore {
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
if err != nil {
logger.Debugf("Error loading .gitignore: %v", err)
}
ignorer = gitignore.NewMatcher(ps)
}
fc := &filecollector.FileCollector{
Ignorer: ignorer,
SrcPath: srcPath,
SrcPrefix: srcPrefix,
Handler: &filecollector.TarCollector{
TarWriter: tw,
UID: cr.UID,
GID: cr.GID,
DstDir: dstPath[1:],
},
}
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
if err != nil {
return err
}
if err := tw.Close(); err != nil {
return err
}
@@ -942,7 +1005,14 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
if err != nil {
return fmt.Errorf("failed to seek tar archive: %w", err)
}
return cr.upload(ctx, extractionRoot, tarFile)
_, err = cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
DestinationPath: "/",
Content: tarFile,
})
if err != nil {
return fmt.Errorf("failed to copy content to container: %w", err)
}
return nil
}
}
@@ -951,23 +1021,41 @@ func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) c
if cr.id == "" {
return cr.missingContainerError("copy to %s", dstPath)
}
logger := common.Logger(ctx)
var buf bytes.Buffer
if err := writeFilesTar(ctx, &buf, cr.UID, cr.GID, files...); err != nil {
tw := tar.NewWriter(&buf)
for _, file := range files {
logger.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
hdr := &tar.Header{
Name: file.Name,
Mode: file.Mode,
Size: int64(len(file.Body)),
Uid: cr.UID,
Gid: cr.GID,
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if _, err := tw.Write([]byte(file.Body)); err != nil {
return err
}
}
if err := tw.Close(); err != nil {
return err
}
common.Logger(ctx).Debugf("Extracting content to '%s'", dstPath)
return cr.upload(ctx, dstPath, &buf)
logger.Debugf("Extracting content to '%s'", dstPath)
_, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
DestinationPath: dstPath,
Content: &buf,
})
if err != nil {
return fmt.Errorf("failed to copy content to container: %w", err)
}
return nil
}
}
func (cr *containerReference) upload(ctx context.Context, dstPath string, content io.Reader) error {
if _, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{DestinationPath: dstPath, Content: content}); err != nil {
return fmt.Errorf("failed to copy content to container: %w", err)
}
return nil
}
func (cr *containerReference) attach() common.Executor {
return func(ctx context.Context) error {
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
@@ -1046,7 +1134,7 @@ func (cr *containerReference) wait() common.Executor {
// sanitizeOptionsHostConfig takes back everything a workflow could escape the container with,
// setting each field to trusted, which is what the runner's own options parse to on their own.
// Only for unprivileged mode, since privileged mode grants host access anyway.
func sanitizeOptionsHostConfig(logger *logging.Logger, hostConfig, trusted *container.HostConfig) {
func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *container.HostConfig) {
resetOption(logger, "--pid", &hostConfig.PidMode, trusted.PidMode)
resetOption(logger, "--ipc", &hostConfig.IpcMode, trusted.IpcMode)
resetOption(logger, "--uts", &hostConfig.UTSMode, trusted.UTSMode)
@@ -1066,10 +1154,6 @@ func sanitizeOptionsHostConfig(logger *logging.Logger, hostConfig, trusted *cont
// systempaths=unconfined lands in these two rather than in SecurityOpt
resetOption(logger, "--security-opt", &hostConfig.MaskedPaths, trusted.MaskedPaths)
resetOption(logger, "--security-opt", &hostConfig.ReadonlyPaths, trusted.ReadonlyPaths)
resetOption(logger, "--log-driver", &hostConfig.LogConfig.Type, trusted.LogConfig.Type) // drivers run in the daemon, egressing from its network
resetOption(logger, "--log-opt", &hostConfig.LogConfig.Config, trusted.LogConfig.Config)
resetOption(logger, "--oom-score-adj", &hostConfig.OomScoreAdj, trusted.OomScoreAdj) // -1000 exempts it from the host OOM killer
resetOption(logger, "--oom-kill-disable", &hostConfig.OomKillDisable, trusted.OomKillDisable)
// a driver mounts what it likes, e.g. local with device= binds any host path, which
// valid_volumes never gets to see
@@ -1085,7 +1169,7 @@ func sanitizeOptionsHostConfig(logger *logging.Logger, hostConfig, trusted *cont
// resetOption puts a field back to the runner's own value. It compares the values rather than
// the flags, so a field that more than one option feeds cannot slip through.
func resetOption[T any](logger *logging.Logger, option string, field *T, trusted T) {
func resetOption[T any](logger logrus.FieldLogger, option string, field *T, trusted T) {
if reflect.DeepEqual(*field, trusted) {
return
}
@@ -1111,9 +1195,9 @@ func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
logger := common.Logger(ctx)
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
if len(cr.input.ValidVolumes) > 0 {
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
// sanitize binds
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
for _, bind := range hostConfig.Binds {
@@ -1127,47 +1211,37 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
sanitizedBinds = append(sanitizedBinds, bind)
continue
}
if matcher.isValid(parsed.Source, mount.Type(parsed.Type), parsed.ReadOnly) {
if matcher.needsNoCopy(parsed.Source, mount.Type(parsed.Type)) && (parsed.Volume == nil || !parsed.Volume.NoCopy) {
bind += ",nocopy"
}
if matcher.isValid(parsed.Source, mount.Type(parsed.Type)) {
sanitizedBinds = append(sanitizedBinds, bind)
} else {
logger.Warnf("[%s] is not a valid volume, will be ignored", parsed.Source)
}
}
hostConfig.Binds = sanitizedBinds
// sanitize mounts
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
for _, mt := range hostConfig.Mounts {
if matcher.isValid(mt.Source, mt.Type) {
sanitizedMounts = append(sanitizedMounts, mt)
} else {
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
}
}
hostConfig.Mounts = sanitizedMounts
} else {
for _, bind := range hostConfig.Binds {
logger.Warnf("[%s] is not a valid volume, will be ignored", bind)
}
hostConfig.Binds = []string{}
}
// sanitize mounts
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
for _, mt := range hostConfig.Mounts {
readOnly := mt.ReadOnly && (mt.BindOptions == nil || !mt.BindOptions.ReadOnlyNonRecursive) // bind-recursive=writable leaves submounts writable
if matcher.isValid(mt.Source, mt.Type, readOnly) {
if matcher.needsNoCopy(mt.Source, mt.Type) {
mt.VolumeOptions = cmp.Or(mt.VolumeOptions, &mount.VolumeOptions{})
mt.VolumeOptions.NoCopy = true
}
sanitizedMounts = append(sanitizedMounts, mt)
} else {
for _, mt := range hostConfig.Mounts {
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
}
hostConfig.Binds = []string{}
hostConfig.Mounts = []mount.Mount{}
}
hostConfig.Mounts = sanitizedMounts
return config, hostConfig
}
func SanitizeBinds(ctx context.Context, validVolumes, binds []string) []string {
cr := &containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}
_, hostConfig := cr.sanitizeConfig(ctx, nil, &container.HostConfig{Binds: binds})
return hostConfig.Binds
}
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
func bindTarget(bind string) string {
parsed, err := loader.ParseVolume(bind)
@@ -1197,39 +1271,28 @@ func overlayVolumes(dst, src *container.HostConfig) {
}
type validVolumeMatcher struct {
allowAll bool
allowAllReadOnly bool
named []validVolumeRule
host []validVolumeRule
}
type validVolumeRule struct {
pattern string
readOnly bool
allowAll bool
named []glob.Glob
host []glob.Glob
}
func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolumeMatcher {
logger := common.Logger(ctx)
ret := validVolumeMatcher{
named: make([]validVolumeRule, 0, len(validVolumes)),
host: make([]validVolumeRule, 0, len(validVolumes)),
named: make([]glob.Glob, 0, len(validVolumes)),
host: make([]glob.Glob, 0, len(validVolumes)),
}
for _, v := range validVolumes {
v, readOnly := strings.CutSuffix(v, ":ro")
if v == "**" {
if readOnly {
ret.allowAllReadOnly = true
} else {
ret.allowAll = true
}
ret.allowAll = true
continue
}
if !isHostVolumePattern(v) {
if doublestar.ValidatePattern(v) {
ret.named = append(ret.named, validVolumeRule{pattern: v, readOnly: readOnly})
if g, err := glob.Compile(v); err != nil {
logger.Errorf("create glob from %s error: %v", v, err)
} else {
logger.Errorf("invalid volume pattern %s", v)
ret.named = append(ret.named, g)
}
continue
}
@@ -1238,44 +1301,40 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
logger.Errorf("normalize volume pattern %s error: %v", v, err)
continue
}
if doublestar.ValidatePathPattern(normalized) {
ret.host = append(ret.host, validVolumeRule{pattern: normalized, readOnly: readOnly})
if g, err := glob.Compile(normalized); err != nil {
logger.Errorf("create glob from %s error: %v", normalized, err)
} else {
logger.Errorf("invalid volume pattern %s", normalized)
ret.host = append(ret.host, g)
}
}
return ret
}
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type, readOnly bool) bool {
if m.allowAll || (m.allowAllReadOnly && readOnly) {
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type) bool {
if m.allowAll {
return true
}
if sourceType == mount.TypeTmpfs && source == "" { // backed by memory, exposes nothing from the host, same as --tmpfs
return true
}
rules, match := m.named, doublestar.MatchUnvalidated
if isHostVolumeSource(source, sourceType) {
normalized, err := normalizeHostVolumePath(source)
if err != nil {
return false
}
rules, match, source = m.host, doublestar.PathMatchUnvalidated, normalized
for _, g := range m.host {
if g.Match(normalized) {
return true
}
}
return false
}
for _, rule := range rules {
if (readOnly || !rule.readOnly) && match(rule.pattern, source) {
for _, g := range m.named {
if g.Match(source) {
return true
}
}
return false
}
// needsNoCopy reports a volume only `:ro` rules allow, as dockerd fills an empty volume from the image even when mounted read-only.
func (m validVolumeMatcher) needsNoCopy(source string, sourceType mount.Type) bool {
return sourceType == mount.TypeVolume && !m.isValid(source, sourceType, false)
}
func isHostVolumePattern(pattern string) bool {
return filepath.IsAbs(pattern) ||
strings.HasPrefix(pattern, "."+string(filepath.Separator)) ||
+95 -220
View File
@@ -5,7 +5,6 @@
package container
import (
"archive/tar"
"bufio"
"bytes"
"context"
@@ -19,8 +18,6 @@ import (
"testing"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/pkg/stdcopy"
@@ -28,33 +25,50 @@ import (
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
client := &mockDockerClient{}
client.On("ContainerInspect", mock.Anything, "amd64-job", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Env: []string{
"PATH=/opt/node/x64/bin:/usr/bin:/bin", "CONFLICT_VAR=container-value", "SOME_RANDOM_VAR=",
"ANOTHER_ONE=old-value", "ANOTHER_ONE=BUT_I_HAVE_VALUE", "LITERAL='quoted'=$HOME\nsecond line",
}}}}, nil)
func TestDocker(t *testing.T) {
requireDocker(t)
ctx := context.Background()
client, err := GetDockerClient(ctx)
require.NoError(t, err)
defer client.Close()
dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{
ContextDir: "testdata",
ImageTag: "envmergetest",
})
err = dockerBuild(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
cr := &containerReference{
cli: client,
input: &NewContainerInput{
Image: "envmergetest",
},
}
env := map[string]string{
"PATH": "/custom/bin",
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin",
"RANDOM_VAR": "WITH_VALUE",
"ANOTHER_VAR": "",
"CONFLICT_VAR": "step-value",
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
}
require.NoError(t, (&containerReference{cli: client, id: "amd64-job"}).UpdateFromImageEnv(&env)(t.Context()))
envExecutor := cr.extractFromImageEnv(&env)
err = envExecutor(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, map[string]string{
"PATH": "/custom/bin:/opt/node/x64/bin:/usr/bin:/bin",
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin:/this/path/does/not/exists/anywhere:/this/either",
"RANDOM_VAR": "WITH_VALUE",
"ANOTHER_VAR": "",
"SOME_RANDOM_VAR": "",
"ANOTHER_ONE": "BUT_I_HAVE_VALUE",
"CONFLICT_VAR": "step-value",
"LITERAL": "'quoted'=$HOME\nsecond line",
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
}, env)
}
@@ -123,11 +137,6 @@ func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.Netw
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
}
func (m *mockDockerClient) NetworkCreate(ctx context.Context, name string, opts mobyclient.NetworkCreateOptions) (mobyclient.NetworkCreateResult, error) {
args := m.Called(ctx, name, opts)
return args.Get(0).(mobyclient.NetworkCreateResult), args.Error(1)
}
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
@@ -166,7 +175,7 @@ func (m *mockConn) Close() (err error) {
}
func TestDockerExecAbort(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
conn := &mockConn{}
@@ -207,7 +216,7 @@ func TestDockerExecAbort(t *testing.T) {
}
func TestDockerExecFailure(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
conn := &mockConn{}
@@ -254,7 +263,7 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
// attach() streaming goroutine has drained and flushed the container's output,
// so a final line without a trailing newline is not lost.
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
@@ -303,7 +312,7 @@ func TestDockerAttachFlushesTrailingLine(t *testing.T) {
}
func TestDockerWaitFailure(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
statusCh := make(chan container.WaitResponse, 1)
statusCh <- container.WaitResponse{StatusCode: 2}
@@ -336,49 +345,42 @@ func TestDockerWaitFailure(t *testing.T) {
// A remove that raced the daemon's AutoRemove teardown is not a failure and must not
// be logged as one.
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
removeFailure := errors.New("driver failed to remove root filesystem")
removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}
killOpts := mobyclient.ContainerKillOptions{Signal: "SIGKILL"}
for _, tc := range []struct {
name string
err error
wantWait bool
waitErr error
wantErr error
name string
err error
wantWait bool
wantFailure bool
}{
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress"), wantWait: true},
{name: "wait canceled", err: cerrdefs.ErrConflict, wantWait: true, waitErr: context.Canceled, wantErr: context.Canceled},
{name: "removed during wait", err: cerrdefs.ErrConflict, wantWait: true, waitErr: cerrdefs.ErrNotFound},
{name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")},
{name: "removed cleanly", err: nil},
{name: "real failure", err: removeFailure, wantErr: removeFailure},
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantFailure: true},
} {
t.Run(tc.name, func(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
client := &mockDockerClient{}
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
if tc.wantWait {
removed := make(chan container.WaitResponse, 1)
waitErrors := make(chan error, 1)
if tc.waitErr == nil {
removed <- container.WaitResponse{}
} else {
waitErrors <- tc.waitErr
}
removed <- container.WaitResponse{}
client.On("ContainerWait", mock.Anything, "abc", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionRemoved}).
Return(mobyclient.ContainerWaitResult{Result: removed, Error: waitErrors})
Return(mobyclient.ContainerWaitResult{Result: removed})
}
cr := &containerReference{id: "abc", cli: client}
require.ErrorIs(t, cr.remove()(ctx), tc.wantErr)
if tc.wantErr != nil {
require.NoError(t, cr.remove()(ctx))
// a failure keeps the id, so a later Remove() can retry it
if tc.wantFailure {
assert.Equal(t, "abc", cr.id)
assert.Len(t, hook.AllEntries(), 1)
} else {
assert.Empty(t, cr.id)
assert.Empty(t, hook.AllEntries())
}
assert.Empty(t, hook.AllRecords())
client.AssertExpectations(t)
})
}
@@ -388,7 +390,7 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
// create() lost its reply, must still be removed rather than leaking with its network. It
// was never started here, so it is not worth a kill of its own.
func TestRemoveWithoutIDUsesName(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
Return(mobyclient.ContainerRemoveResult{}, nil)
@@ -401,7 +403,7 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
// find() must drop a stale cached id so later Copy/Exec don't hit the
// daemon with a torn-down container.
func TestFindRevalidatesStaleID(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
boom := errors.New("daemon unreachable")
newCR := func(id string) (*containerReference, *mockDockerClient) {
@@ -459,7 +461,7 @@ func TestFindRevalidatesStaleID(t *testing.T) {
// Every daemon entry point fails fast with a clear, container-named
// error when no live cr.id is known.
func TestRejectsMissingContainer(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
@@ -469,7 +471,7 @@ func TestRejectsMissingContainer(t *testing.T) {
assert.Contains(t, err.Error(), `container "job-1" does not exist`, op)
}
check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
check("copyDir", cr.copyDir("/var/run/act", "/src", false, false)(ctx))
check("copyDir", cr.copyDir("/var/run/act", "/src", false)(ctx))
check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
_, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
check("GetContainerArchive", err)
@@ -484,45 +486,25 @@ func TestRejectsMissingContainer(t *testing.T) {
check("Inspect after removal", err)
}
// End-to-end: a stale cr.id is cleared, repopulated from name lookup,
// and the Copy completes against the fresh id.
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
for _, testcase := range []struct {
name, destination, extractionRoot, entry string
directory bool
}{
{name: "files", destination: "/var/run/act", extractionRoot: "/var/run/act", entry: "x"},
{name: "action below act volume", directory: true, destination: "/var/run/act/actions/checkout/", extractionRoot: "/var/run/act", entry: "actions/checkout/x"},
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
} {
t.Run(testcase.name, func(t *testing.T) {
ctx := t.Context()
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
Return(mobyclient.ContainerListResult{Items: []container.Summary{
{ID: "fresh", Names: []string{"/job-1"}},
}}, nil)
client.On("CopyToContainer", ctx, "fresh", mock.Anything).
Run(func(args mock.Arguments) {
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
assert.Equal(t, testcase.extractionRoot, opts.DestinationPath)
header, err := tar.NewReader(opts.Content).Next()
require.NoError(t, err)
assert.Equal(t, testcase.entry, header.Name)
}).Return(mobyclient.CopyToContainerResult{}, nil).Once()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
Return(mobyclient.ContainerListResult{Items: []container.Summary{
{ID: "fresh", Names: []string{"/job-1"}},
}}, nil)
client.On("CopyToContainer", ctx, "fresh", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act"
})).Return(mobyclient.CopyToContainerResult{}, nil)
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
if testcase.directory {
source := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
} else {
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
}
assert.Equal(t, "fresh", cr.id)
client.AssertExpectations(t)
})
}
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
require.NoError(t, cr.Copy("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
assert.Equal(t, "fresh", cr.id)
client.AssertExpectations(t)
}
// Type assert containerReference implements ExecutionsEnvironment
@@ -530,12 +512,10 @@ var _ ExecutionsEnvironment = &containerReference{}
func TestCheckVolumes(t *testing.T) {
testCases := []struct {
desc string
validVolumes []string
binds []string
expectedBinds []string
mounts []mount.Mount
expectedMounts []mount.Mount
desc string
validVolumes []string
binds []string
expectedBinds []string
}{
{
desc: "match all volumes",
@@ -573,14 +553,11 @@ func TestCheckVolumes(t *testing.T) {
"shared_volume",
"/home/test/data",
"/etc/conf.d/*.json",
"/opt/**",
},
binds: []string{
"shared_volume:/shared_volume",
"/home/test/data:/test_data",
"/etc/conf.d/base.json:/config/base.json",
"/etc/conf.d/sub/nested.json:/config/nested.json",
"/opt/app/cache:/cache",
"sql_data:/sql_data",
"/secrets/keys:/keys",
},
@@ -588,44 +565,21 @@ func TestCheckVolumes(t *testing.T) {
"shared_volume:/shared_volume",
"/home/test/data:/test_data",
"/etc/conf.d/base.json:/config/base.json",
"/opt/app/cache:/cache",
},
},
{
desc: "read-only rules allow only read-only mounts and copy nothing into volumes",
validVolumes: []string{"/etc/ssl:ro", "data:ro", "shared"},
binds: []string{"/etc/ssl:/ssl:ro", "/etc/ssl:/ssl", "data:/data:ro", "data:/data", "shared:/shared:ro"},
expectedBinds: []string{"/etc/ssl:/ssl:ro", "data:/data:ro,nocopy", "shared:/shared:ro"},
mounts: []mount.Mount{
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true, BindOptions: &mount.BindOptions{ReadOnlyNonRecursive: true}},
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true},
},
expectedMounts: []mount.Mount{
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true, VolumeOptions: &mount.VolumeOptions{NoCopy: true}},
},
},
{
desc: "read-only wildcard keeps writable rules writable",
validVolumes: []string{"**:ro", "shared"},
binds: []string{"/etc/ssl:/ssl:ro", "data:/data", "shared:/shared"},
expectedBinds: []string{"/etc/ssl:/ssl:ro", "shared:/shared"},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
ValidVolumes: tc.validVolumes,
},
}
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts})
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds})
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts)) // every drop is warned about
assert.Len(t, hook.AllEntries(), len(tc.binds)-len(tc.expectedBinds)) // every drop is warned about
})
}
}
@@ -653,33 +607,33 @@ func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
t.Setenv("RUNNER_SECRET", "s3cr3t")
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
_, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.ErrorContains(t, err, "not allowed in a workflow")
// the runner reading its own files is what those options are for
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
_, _, err = cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
_, _, err = cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
}
// a bare name is no longer resolved from the runner's environment, for either source
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
NetworkMode: "bridge",
RunnerOptions: "--env RUNNER_SECRET",
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
}}
config, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
config, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
}
func TestSanitizeOptionsHostConfig(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
// every field the sanitizer resets, so a reset dropped in a refactor fails here
hostConfig := &container.HostConfig{
@@ -701,9 +655,6 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
DeviceCgroupRules: []string{"a *:* rwm"},
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}},
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
LogConfig: container.LogConfig{Type: "syslog", Config: map[string]string{"syslog-address": "tcp://169.254.169.254:514"}},
OomScoreAdj: -1000,
OomKillDisable: new(true),
}
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{})
@@ -712,18 +663,17 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
}
// mergeOptions merges both option sources into a bare container, returning the result and its log.
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *logtest.Handler) {
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *test.Hook) {
t.Helper()
logger, hook := logtest.NewLogger()
logger, hook := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
RunnerOptions: runnerOptions,
WorkflowOptions: workflowOptions,
NetworkMode: "bridge",
UsernsMode: "private",
ValidVolumes: []string{"**"},
}}
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{
Privileged: privileged,
UsernsMode: container.UsernsMode("private"),
NetworkMode: container.NetworkMode("bridge"),
@@ -737,9 +687,7 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
"--security-opt apparmor=unconfined --volumes-from other --isolation process " +
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1 " +
"--log-driver=syslog --log-opt syslog-address=tcp://169.254.169.254:514 " +
"--oom-score-adj=-1000 --oom-kill-disable"
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's
// own options alone describe it, field for field
@@ -766,8 +714,8 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
}
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
logger := logging.New(logtest.New())
ctx := common.WithLogger(t.Context(), logger)
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
base := t.TempDir()
allowed := filepath.Join(base, "allowed")
@@ -802,22 +750,6 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
assert.Empty(t, hostConf.Binds)
}
func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
hostConfig, _ := mergeOptions(t, "", "--mount type=tmpfs,dst=/var/lib/mysql", false)
require.Len(t, hostConfig.Mounts, 1)
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
logger, hook := logtest.NewLogger()
_, sanitized := (&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{Mounts: []mount.Mount{
hostConfig.Mounts[0],
{Type: mount.TypeVolume, Target: "/anonymous"},
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
}})
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
assert.Len(t, hook.AllRecords(), 2)
}
}
func TestContainerInfoFromInspect(t *testing.T) {
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
@@ -869,42 +801,15 @@ func TestContainerInfoFromInspect(t *testing.T) {
assert.Equal(t, "abc123", info.ID)
assert.Equal(t, HealthNone, info.Health)
})
t.Run("reports the mount sources by container path", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
Mounts: []container.MountPoint{
{Type: mount.TypeVolume, Name: "job", Source: "/var/lib/docker/volumes/job/_data", Destination: "/workspace/owner/repo"},
{Type: mount.TypeBind, Source: "/var/run/docker.sock", Destination: "/var/run/docker.sock"},
{Type: mount.TypeVolume, Name: "cache", Source: "/var/lib/docker/volumes/cache/_data", Destination: "/cache"},
{Type: mount.TypeBind, Source: "/custom/resolv.conf", Destination: "/etc/resolv.conf"},
},
HostConfig: &container.HostConfig{
Tmpfs: map[string]string{"/workspace/owner/repo//tmp/": "size=1m"},
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "cache", Target: "/cache/", VolumeOptions: &mount.VolumeOptions{Subpath: "project"}}},
},
})
assert.Equal(t, map[string]string{
"/workspace/owner/repo": "/var/lib/docker/volumes/job/_data",
"/workspace/owner/repo/tmp": "",
"/var/run/docker.sock": "/var/run/docker.sock",
"/cache": "/var/lib/docker/volumes/cache/_data/project",
"/etc/resolv.conf": "/custom/resolv.conf",
"/etc/hosts": "",
"/etc/hostname": "",
}, info.Mounts)
})
}
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
logger := logging.New(logtest.New())
ctx := common.WithLogger(t.Context(), logger)
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
NetworkMode: "bridge",
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
WorkflowOptions: "--volume /unapproved:/var/run/docker.sock",
ValidVolumes: []string{"/host/tools"},
NetworkMode: "bridge",
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
},
}
@@ -920,46 +825,16 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
warnings := func(runnerOptions, workflowOptions string) int {
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
return len(hook.AllRecords())
return len(hook.AllEntries())
}
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
assert.Zero(t, warnings("", "--shm-size 1g"))
assert.Equal(t, 1, warnings("--network host", ""))
assert.Equal(t, 1, warnings("", "--privileged"))
}
func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
for _, entrypoint := range [][]string{nil, {"sleep", "3600"}} {
for _, optionEntrypoint := range []string{"/sbin/dumb-init", ""} {
logger, hook := logtest.NewLogger()
cr := &containerReference{
input: &NewContainerInput{
NetworkMode: "bridge",
Entrypoint: entrypoint,
RunnerOptions: "--entrypoint=runner-entrypoint",
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
},
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
if len(entrypoint) > 0 {
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
} else {
assert.Equal(t, []string{optionEntrypoint}, opts.Config.Entrypoint)
}
require.NotNil(t, opts.HostConfig.Init)
assert.True(t, *opts.HostConfig.Init)
return mobyclient.ContainerCreateResult{ID: "created"}, nil
}},
}
require.NoError(t, cr.create(nil, nil)(common.WithLogger(t.Context(), logger)))
assert.Equal(t, "created", cr.id)
assert.Equal(t, len(entrypoint) > 0, len(hook.AllRecords()) == 1)
}
}
}
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
}}
+1 -42
View File
@@ -9,7 +9,6 @@ package container
import (
"context"
"errors"
"net/netip"
"runtime"
"time"
@@ -18,10 +17,6 @@ import (
"github.com/moby/moby/api/types/system"
)
func IsolatedCacheContainer(context.Context, netip.Addr, string) (string, error) {
return "", nil
}
// ImageExistsLocally returns a boolean indicating if an image with the
// requested name, tag and architecture exists in the local docker image store
func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool, error) {
@@ -34,18 +29,6 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
return false, errors.New("Unsupported Operation")
}
func RemoveDockerJobResources(_ context.Context, _ string) error {
return nil
}
func NewDockerProxy(_ context.Context, _ string) *DockerProxy {
return nil
}
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
return nil, errors.New("Unsupported Operation")
}
// NewDockerBuildExecutor function to create a run executor for the container
func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
return func(ctx context.Context) error {
@@ -91,30 +74,6 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
}
}
func NewDockerNetworkConnectExecutor(_, _ string) common.Executor {
return func(_ context.Context) error {
return nil
}
}
func NewDockerNetworkDisconnectExecutor(_, _ string) common.Executor {
return func(_ context.Context) error {
return nil
}
}
func RemoveOrphanNetworks(_ context.Context, _, _ string, _ time.Time) error {
return nil
}
func SanitizeBinds(_ context.Context, _, binds []string) []string {
return binds
}
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
return nil
}
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
return nil
}
+27 -57
View File
@@ -8,67 +8,40 @@ package container
import (
"context"
"errors"
"fmt"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/client"
)
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
for _, volumeName := range volumeNames {
if _, err := cli.VolumeCreate(ctx, client.VolumeCreateOptions{
Name: volumeName,
Labels: runnerLabels(runnerUUID),
}); err != nil {
return err
}
}
return nil
}
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
}
defer cli.Close()
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID).Add("dangling", "true"),
})
if err != nil {
return err
}
var errs []error
for _, item := range volumes.Items {
created, err := time.Parse(time.RFC3339, item.CreatedAt)
// an unreadable or recent timestamp may belong to a job that is still starting up
if err != nil || created.After(createdBefore) || item.Labels[runnerUUIDLabel] != runnerUUID {
continue
}
if _, err := cli.VolumeRemove(ctx, item.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", item.Name, err))
continue
}
common.Logger(ctx).Infof("removed docker volume %s left behind by an earlier job", item.Name)
}
return errors.Join(errs...)
}
func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executor {
return func(ctx context.Context) error {
common.Logger(ctx).Debugf("docker volume rm %s", volumeName)
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
list, err := cli.VolumeList(ctx, client.VolumeListOptions{})
if err != nil {
return err
}
for _, vol := range list.Items {
if vol.Name == volumeName {
return removeExecutor(volumeName, force)(ctx)
}
}
// Volume not found - do nothing
return nil
}
}
func removeExecutor(volume string, force bool) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
logger.Debugf("docker volume rm %s", volume)
if common.Dryrun(ctx) {
return nil
@@ -80,10 +53,7 @@ func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executo
}
defer cli.Close()
_, err = cli.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: force})
if cerrdefs.IsNotFound(err) { // already gone is the outcome we wanted
return nil
}
_, err = cli.VolumeRemove(ctx, volume, client.VolumeRemoveOptions{Force: force})
return err
}
}
+2 -1
View File
@@ -4,6 +4,7 @@
package container
import (
"context"
"testing"
mobyclient "github.com/moby/moby/client"
@@ -14,7 +15,7 @@ import (
// so the daemon has to be pinged explicitly here to decide whether to skip.
func requireDocker(t *testing.T) {
t.Helper()
ctx := t.Context()
ctx := context.Background()
cli, err := GetDockerClient(ctx)
if err != nil {
t.Skipf("skipping: docker client unavailable: %v", err)
+18 -7
View File
@@ -27,6 +27,9 @@ import (
"gitea.com/gitea/runner/internal/pkg/process"
"github.com/creack/pty"
"github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"golang.org/x/term"
)
@@ -89,7 +92,7 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec
}
}
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
srcPrefix := filepath.Dir(srcPath)
@@ -97,16 +100,24 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGi
srcPrefix += string(filepath.Separator)
}
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
var ignorer gitignore.Matcher
if useGitIgnore {
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
if err != nil {
logger.Debugf("Error loading .gitignore: %v", err)
}
ignorer = gitignore.NewMatcher(ps)
}
fc := &filecollector.FileCollector{
SrcPath: srcPath,
SrcPrefix: srcPrefix,
SkipGitDir: skipGitDir,
UseGitIgnore: useGitIgnore,
Ignorer: ignorer,
SrcPath: srcPath,
SrcPrefix: srcPrefix,
Handler: &filecollector.CopyCollector{
DstDir: destPath,
},
}
return filepath.Walk(srcPath, fc.CollectFiles(ctx))
return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
}
}
@@ -140,7 +151,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
SrcPrefix: srcPrefix,
Handler: tc,
}
err = filepath.Walk(srcPath, fc.CollectFiles(ctx))
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
if err != nil {
return nil, err
}
+28 -33
View File
@@ -18,8 +18,8 @@ import (
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -33,36 +33,27 @@ func TestActionPlatformNames(t *testing.T) {
}
func TestCopyDir(t *testing.T) {
for _, test := range []struct {
name string
useGitIgnore, skipGitDir bool
}{
{name: "git dir kept for a workspace copy"},
{name: "git dir skipped for an action copy", skipGitDir: true},
{name: "unfiltered outside a git repository", useGitIgnore: true},
} {
t.Run(test.name, func(t *testing.T) {
src := filepath.Join(t.TempDir(), "action")
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
dest := filepath.Join(t.TempDir(), "dest")
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
assert.FileExists(t, filepath.Join(dest, "action.yml"))
if test.skipGitDir {
assert.NoDirExists(t, filepath.Join(dest, ".git"))
} else {
assert.FileExists(t, filepath.Join(dest, ".git", "objects", "pack"))
}
})
dir := t.TempDir()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
ToolCache: filepath.Join(dir, "tool_cache"),
ActPath: filepath.Join(dir, "act_path"),
StdOut: os.Stdout,
Workdir: path.Join("testdata", "scratch"),
}
_ = os.MkdirAll(e.Path, 0o700)
_ = os.MkdirAll(e.TmpDir, 0o700)
_ = os.MkdirAll(e.ToolCache, 0o700)
_ = os.MkdirAll(e.ActPath, 0o700)
err := e.CopyDir(e.Workdir, e.Path, true)(ctx)
assert.NoError(t, err)
}
func TestGetContainerArchive(t *testing.T) {
dir := t.TempDir()
ctx := t.Context()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
@@ -97,7 +88,7 @@ func TestHostEnvironmentExecExitCode(t *testing.T) {
t.Skip("uses POSIX shell")
}
dir := t.TempDir()
ctx := t.Context()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
@@ -148,7 +139,7 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
err := e.Exec(
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
map[string]string{"PATH": os.Getenv("PATH")}, "", "",
)(t.Context())
)(context.Background())
require.NoError(t, err)
got := strings.TrimSpace(strings.ReplaceAll(buf.String(), "\r", ""))
assert.Equal(t, tc.expect, got)
@@ -157,7 +148,8 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
}
func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
ctx := common.WithLogger(t.Context(), logging.Default())
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
miscRoot := filepath.Join(base, "misc")
path := filepath.Join(miscRoot, "hostexecutor")
@@ -179,7 +171,8 @@ func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
}
func TestHostEnvironmentRemoveCleansWorkdirWhenOwned(t *testing.T) {
ctx := common.WithLogger(t.Context(), logging.Default())
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
miscRoot := filepath.Join(base, "misc")
path := filepath.Join(miscRoot, "hostexecutor")
@@ -220,7 +213,7 @@ func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
removeAll = orig
})
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
err := removeAllWithContext(ctx, t.TempDir())
@@ -237,7 +230,8 @@ func TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp(t *testing.T) {
hostCleanupTimeout = 100 * time.Millisecond
t.Cleanup(func() { hostCleanupTimeout = orig })
ctx := common.WithLogger(t.Context(), logging.Default())
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
path := filepath.Join(base, "misc", "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
@@ -290,7 +284,8 @@ func TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval(t *testing.T) {
removeAll = origRemoveAll
})
ctx := common.WithLogger(t.Context(), logging.Default())
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
path := filepath.Join(base, "misc", "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
-576
View File
@@ -1,576 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"bufio"
"bytes"
"cmp"
"context"
"crypto/rand"
"errors"
"fmt"
"io"
"maps"
"net/http"
"net/url"
"os"
"path"
"regexp"
"slices"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/kballard/go-shellquote"
"github.com/spf13/pflag"
)
// KubernetesOptions configures the pod a job runs in.
type KubernetesOptions struct {
Kubeconfig string
Namespace string
PodTemplates []map[string]any // merged in order, a later one winning
MaxLifetime time.Duration
ForcePull bool
RunnerUUID string
}
// KubernetesPod runs a job container and its service containers as one pod.
type KubernetesPod struct {
options KubernetesOptions
client *kubeClient
name, secretName string
job *kubernetesContainer
services []*kubernetesContainer
arch, path string
}
type kubernetesContainer struct {
LinuxContainerEnvironmentExtensions
pod *KubernetesPod
id string // the workflow's service id, name is its DNS label form
name string
input *NewContainerInput
probe map[string]any
}
type containerStatus struct {
Name string
Ready bool
State struct {
Waiting *struct{ Reason, Message string }
Running *struct{}
Terminated *struct {
ExitCode int
Message string
}
}
}
type podStatus struct {
Spec struct{ Containers []struct{} }
Status struct {
Phase, Reason, Message string
Conditions []struct{ Type, Status, Message string }
InitContainerStatuses, ContainerStatuses []containerStatus
}
}
func NewKubernetesPod(options KubernetesOptions) (*KubernetesPod, error) {
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
if err != nil {
return nil, fmt.Errorf("connect to Kubernetes: %w", err)
}
return &KubernetesPod{options: options, client: client}, nil
}
// JobContainer returns the container of the pod the job's steps run in.
func (p *KubernetesPod) JobContainer(input *NewContainerInput) ExecutionsEnvironment {
p.job = &kubernetesContainer{pod: p, name: "job", input: input}
return p.job
}
var invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
// KubernetesServiceName returns the container name and hostname of service id, made a DNS label as both must be.
func KubernetesServiceName(id string) string {
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
return strings.TrimRight(name[:min(63, len(name))], "-")
}
// ServiceContainer returns a container of the pod the job reaches at hostname KubernetesServiceName(id).
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
service := &kubernetesContainer{pod: p, id: id, name: KubernetesServiceName(id), input: input}
p.services = append(p.services, service)
return service
}
func (p *KubernetesPod) create(ctx context.Context) error {
prefix := strings.Trim(strings.ToLower(p.job.input.Name), "-")
prefix = cmp.Or(strings.TrimRight(prefix[:min(40, len(prefix))], "-"), "gitea-runner") + "-"
name := prefix + strings.ToLower(rand.Text()[:8]) // chosen here, so cleanup knows it even when a create response is lost
labels := map[string]any{"app.kubernetes.io/managed-by": "gitea-runner"}
if p.options.RunnerUUID != "" {
labels[runnerUUIDLabel] = p.options.RunnerUUID
}
metadata := map[string]any{"name": name, "labels": labels}
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
var runner struct{ Metadata struct{ Name, UID string } }
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
} else {
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
}
}
secretData := map[string]string{}
manifest, err := p.manifest(name, secretData, metadata)
if err != nil {
return err
}
for _, service := range p.services {
if !strings.EqualFold(service.id, service.name) {
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
}
}
if len(secretData) > 0 {
p.secretName = name // named like the pod, RemoveOrphanKubernetesResources finds it by that
if err := p.client.do(ctx, http.MethodPost, p.client.path("secrets"), map[string]any{"metadata": metadata, "stringData": secretData}, nil); err != nil {
return fmt.Errorf("create secret: %w", err)
}
}
p.name = name
if err := p.client.do(ctx, http.MethodPost, p.client.path("pods"), manifest, nil); err != nil {
return fmt.Errorf("create pod: %w", err)
}
return nil
}
func (p *KubernetesPod) manifest(secret string, secretData map[string]string, metadata map[string]any) (any, error) {
template := map[string]any{"spec": map[string]any{
"automountServiceAccountToken": false,
"enableServiceLinks": false,
"terminationGracePeriodSeconds": 0, // the job container's tail ignores SIGTERM
}}
for _, layer := range p.options.PodTemplates {
template, _ = mergeManifest(template, layer, "").(map[string]any)
}
templateSpec, _ := template["spec"].(map[string]any)
templateContainers, _ := templateSpec["containers"].([]any)
templateVolumes, _ := templateSpec["volumes"].([]any)
var serviceTemplate any
if index := slices.IndexFunc(templateContainers, func(container any) bool { return manifestField(container, "name") == "$services" }); index >= 0 {
serviceTemplate = templateContainers[index]
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
}
var containers, hostnames, volumes []any
names := map[string]bool{}
for _, container := range templateContainers {
names[manifestField(container, "name")] = true
}
for _, c := range append([]*kubernetesContainer{p.job}, p.services...) {
if c != p.job && (c.name == "" || names[c.name]) {
return nil, fmt.Errorf("service %s maps to container %q, which is empty or named like another container of the pod", c.id, c.name)
}
names[c.name] = true
spec, err := c.spec(secret, secretData)
if err != nil {
return nil, err
}
if c == p.job {
containers = append(containers, spec)
continue
}
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
hostnames = append(hostnames, c.name)
}
for _, name := range []string{"workspace", "act"} {
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
volumes = append(volumes, map[string]any{"name": name, "emptyDir": map[string]any{}})
}
}
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
if len(hostnames) > 0 {
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
}
if p.options.MaxLifetime > 0 {
spec["activeDeadlineSeconds"] = max(1, int64(p.options.MaxLifetime.Seconds()))
}
return mergeManifest(template, map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}, ""), nil
}
// mergeManifest merges over onto base like a strategic merge patch: maps key by key, map list items by name or volumeMounts by mountPath, env entries, volumes and other values replaced.
func mergeManifest(base, over any, field string) any {
switch over := over.(type) {
case map[string]any:
merged := map[string]any{}
if base, ok := base.(map[string]any); ok {
maps.Copy(merged, base)
}
for key, value := range over {
merged[key] = mergeManifest(merged[key], value, key)
}
return merged
case []any:
key := cmp.Or(map[string]string{"volumeMounts": "mountPath"}[field], "name")
baseList, _ := base.([]any)
merged := slices.Clone(baseList)
for _, item := range over {
if _, ok := item.(map[string]any); !ok {
return over
}
name := manifestField(item, key)
index := slices.IndexFunc(merged, func(existing any) bool { return name != "" && manifestField(existing, key) == name })
switch {
case index < 0:
merged = append(merged, item)
case field == "env" || field == "volumes": // each holds one source, merging two makes a pod the API rejects
merged[index] = item
default:
merged[index] = mergeManifest(merged[index], item, "")
}
}
return merged
}
return over
}
func manifestField(item any, key string) string {
fields, _ := item.(map[string]any)
value, _ := fields[key].(string)
return value
}
func (c *kubernetesContainer) spec(secret string, secretData map[string]string) (map[string]any, error) {
env := []any{}
for index, entry := range c.input.Env {
key, value, _ := strings.Cut(entry, "=")
if c == c.pod.job {
env = append(env, map[string]any{"name": key, "value": value})
continue
}
secretKey := fmt.Sprintf("%s.%d", c.name, index)
secretData[secretKey] = value
env = append(env, map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": secret, "key": secretKey}}})
}
spec := map[string]any{
"name": c.name, "image": c.input.Image, "command": escapeExpansion(c.input.Entrypoint), "args": escapeExpansion(c.input.Cmd),
"workingDir": c.input.WorkingDir, "env": env,
}
if c.pod.options.ForcePull {
spec["imagePullPolicy"] = "Always"
}
if c == c.pod.job {
spec["volumeMounts"] = []any{
map[string]any{"name": "workspace", "mountPath": c.input.WorkingDir},
map[string]any{"name": "act", "mountPath": c.GetActPath()},
}
}
if err := c.parseOptions(); err != nil {
return nil, fmt.Errorf("service %s: %w", c.name, err)
}
if c.probe != nil {
spec["startupProbe"] = c.probe
}
return spec, nil
}
// escapeExpansion keeps kubelet from expanding $(VAR) and $$ in command and args, which docker passes as is.
func escapeExpansion(args []string) []string {
args = slices.Clone(args)
for index := range args {
args[index] = strings.ReplaceAll(args[index], "$", "$$")
}
return args
}
// parseOptions turns docker health options, the only ones a pod supports, into a startup probe, which like docker gives up after the retries.
func (c *kubernetesContainer) parseOptions() error {
args, err := shellquote.Split(c.input.WorkflowOptions)
if err != nil {
return err
}
flags := pflag.NewFlagSet("options", pflag.ContinueOnError)
flags.SetOutput(io.Discard)
command := flags.String("health-cmd", "", "")
interval := flags.Duration("health-interval", 0, "")
timeout := flags.Duration("health-timeout", 0, "")
retries := flags.Int("health-retries", 0, "")
startPeriod := flags.Duration("health-start-period", 0, "")
if err := flags.Parse(args); err != nil {
return fmt.Errorf("options %q: %w, only --health-* options are supported on Kubernetes", c.input.WorkflowOptions, err)
}
if *command == "" {
return nil
}
period := max(1, int(cmp.Or(*interval, 30*time.Second).Seconds()))
c.probe = map[string]any{
"exec": map[string]any{"command": []string{"/bin/sh", "-c", *command}},
"periodSeconds": period,
"timeoutSeconds": max(1, int(cmp.Or(*timeout, 30*time.Second).Seconds())),
"failureThreshold": max(1, cmp.Or(*retries, 3)) + int(startPeriod.Seconds())/period, // docker does not count failures in the start period
}
return nil
}
func (p *KubernetesPod) get(ctx context.Context) (*podStatus, error) {
var pod podStatus
return &pod, p.client.do(ctx, http.MethodGet, p.client.path("pods/"+p.name), nil, &pod)
}
func (p *KubernetesPod) waitStarted(ctx context.Context) error {
var unscheduled string
for delay := 100 * time.Millisecond; ; delay = min(2*delay, time.Second) {
pod, err := p.get(ctx)
if err != nil {
return err
}
if pod.Status.Phase == "Failed" {
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
}
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
if waiting := status.State.Waiting; waiting != nil {
switch waiting.Reason {
case "ErrImagePull", "ImagePullBackOff", "InvalidImageName", "ErrImageNeverPull", "CreateContainerConfigError", "CreateContainerError", "RunContainerError":
return fmt.Errorf("container %s of pod %s: %s: %s", status.Name, p.name, waiting.Reason, waiting.Message)
}
started = false
}
}
if started {
return nil
}
for _, condition := range pod.Status.Conditions {
if condition.Type == "PodScheduled" && condition.Status == "False" && condition.Message != unscheduled {
unscheduled = condition.Message
common.Logger(ctx).Infof("Waiting for pod %s to be scheduled: %s", p.name, unscheduled)
}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(delay):
}
}
}
func (c *kubernetesContainer) Create(_, _ []string) common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
return c.pod.create(ctx)
}
}
func (c *kubernetesContainer) Start(_ bool) common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
if err := c.pod.waitStarted(ctx); err != nil {
return err
}
var stdout bytes.Buffer
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard); err != nil {
return err
}
machine, path, _ := strings.Cut(stdout.String(), "\n")
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
return nil
}
}
func (*kubernetesContainer) Pull(bool) common.Executor {
return func(context.Context) error { return nil }
}
var (
shellName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
bashReadonly = []string{"BASHOPTS", "BASH_VERSINFO", "EUID", "PPID", "SHELLOPTS", "UID"} // bash as sh cannot export these
)
// Exec feeds the command to sh on stdin, which exports the env so it stays out of the request URL and, where sh can export a name, out of process arguments.
func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
return func(ctx context.Context) error {
script, args := "", []string{"env", "--"}
for name, value := range env {
if shellName.MatchString(name) && !slices.Contains(bashReadonly, name) {
script += "command export " + shellquote.Join(name+"="+value) + "\n"
} else {
args = append(args, name+"="+value)
}
}
script += "exec " + shellquote.Join(append(args, command...)...)
if workdir != "" {
script = "cd " + shellquote.Join(workdir) + " || exit\n" + script
}
defer common.FlushWriter(c.input.Stdout)
defer common.FlushWriter(c.input.Stderr)
return c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh"}, strings.NewReader(script), c.input.Stdout, c.input.Stderr)
}
}
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
reader, writer := io.Pipe()
defer reader.Close()
go func() { writer.CloseWithError(write(writer)) }()
var stderr bytes.Buffer
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
}
return nil
}
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
return func(ctx context.Context) error {
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
}
}
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
return func(ctx context.Context) error {
return c.extract(ctx, destPath, func(writer io.Writer) error {
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
})
}
}
// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
reader, writer := io.Pipe()
go func() {
var stderr bytes.Buffer
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
if err != nil {
err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
}
writer.CloseWithError(err)
}()
buffered := bufio.NewReader(reader)
if _, err := buffered.Peek(1); err != nil {
reader.Close()
return nil, err
}
return struct {
io.Reader
io.Closer
}{buffered, reader}, nil
}
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(c, srcPath, env)
}
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
return func(context.Context) error {
(*env)["PATH"] = cmp.Or((*env)["PATH"], c.pod.path)
return nil
}
}
func (c *kubernetesContainer) Inspect(ctx context.Context) (*Info, error) {
if c.pod.name == "" {
return nil, fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
}
pod, err := c.pod.get(ctx)
if isNotFound(err) {
return nil, fmt.Errorf("pod %s %w", c.pod.name, ErrContainerNotFound)
} else if err != nil {
return nil, err
}
info := &Info{ID: c.pod.name + "/" + c.name, State: "created", Health: HealthNone, Ports: map[string]string{}}
for port := range c.input.ExposedPorts {
info.Ports[port.Port()] = port.Port() // services share the pod's network
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name != c.name {
continue
}
if status.State.Running != nil {
info.State = StateRunning
} else if terminated := status.State.Terminated; terminated != nil {
info.State, info.ExitCode, info.HealthOutput = "exited", terminated.ExitCode, terminated.Message
}
if c.probe != nil {
switch {
case status.Ready:
info.Health = HealthHealthy
case info.State == "exited":
info.Health = HealthUnhealthy
default:
info.Health = HealthStarting
}
}
}
return info, nil
}
func (c *kubernetesContainer) DumpLogs(ctx context.Context) error {
if c.pod.name == "" {
return fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
}
defer common.FlushWriter(c.input.Stdout)
return c.pod.client.do(ctx, http.MethodGet, c.pod.client.path("pods/"+c.pod.name+"/log?container="+c.name), nil, c.input.Stdout)
}
func (c *kubernetesContainer) Remove() common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
return errors.Join(c.pod.client.delete(ctx, "pods", c.pod.name), c.pod.client.delete(ctx, "secrets", c.pod.secretName))
}
}
// RemoveOrphanKubernetesResources removes this runner's job pods created before createdBefore and the Secrets named like them, as listing Secrets would reveal all of the namespace's.
func RemoveOrphanKubernetesResources(ctx context.Context, options KubernetesOptions, createdBefore time.Time) error {
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
if err != nil {
return fmt.Errorf("connect to Kubernetes: %w", err)
}
defer client.http.CloseIdleConnections()
var pods struct {
Items []struct {
Metadata struct {
Name string
CreationTimestamp time.Time
}
}
}
if err := client.do(ctx, http.MethodGet, client.path("pods?labelSelector="+url.QueryEscape(runnerUUIDLabel+"="+options.RunnerUUID)), nil, &pods); err != nil {
return fmt.Errorf("list pods: %w", err)
}
var errs []error
for _, item := range pods.Items {
if !item.Metadata.CreationTimestamp.Before(createdBefore) {
continue
}
if err := client.delete(ctx, "secrets", item.Metadata.Name); err != nil {
errs = append(errs, err) // leave the pod for the next cleanup, it is the only way back to its Secret
continue
}
errs = append(errs, client.delete(ctx, "pods", item.Metadata.Name))
}
return errors.Join(errs...)
}
func (c *kubernetesContainer) Close() common.Executor {
return func(context.Context) error {
c.pod.client.http.CloseIdleConnections()
return nil
}
}
func (c *kubernetesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
oldStdout, oldStderr := c.input.Stdout, c.input.Stderr
c.input.Stdout, c.input.Stderr = stdout, stderr
return oldStdout, oldStderr
}
func (c *kubernetesContainer) GetRunnerContext(_ context.Context) map[string]any {
return map[string]any{"os": "Linux", "arch": c.pod.arch, "temp": "/tmp", "tool_cache": DefaultToolCache}
}
-400
View File
@@ -1,400 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"bytes"
"cmp"
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/coder/websocket"
"go.yaml.in/yaml/v4"
)
const (
serviceAccountDir = "/var/run/secrets/kubernetes.io/serviceaccount"
execProtocol = "v5.channel.k8s.io"
)
type kubeClient struct {
server, namespace, token string
tokenFile string // re-read on every request, service account tokens rotate
inCluster bool
http *http.Client
}
type apiError struct {
code int
message string
}
func (e *apiError) Error() string { return e.message }
func isNotFound(err error) bool {
var apiErr *apiError
return errors.As(err, &apiErr) && apiErr.code == http.StatusNotFound
}
func newKubeClient(kubeconfig, namespace string) (*kubeClient, error) {
client := &kubeClient{namespace: namespace}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}
if host := os.Getenv("KUBERNETES_SERVICE_HOST"); host != "" && kubeconfig == "" {
if err := client.loadInCluster(host, transport.TLSClientConfig); err != nil {
return nil, err
}
} else if err := client.loadKubeconfig(kubeconfig, transport); err != nil {
return nil, err
}
client.http = &http.Client{Transport: transport}
return client, nil
}
func (c *kubeClient) loadInCluster(host string, tlsConfig *tls.Config) error {
c.server = "https://" + net.JoinHostPort(host, os.Getenv("KUBERNETES_SERVICE_PORT"))
c.tokenFile = serviceAccountDir + "/token"
ca, err := os.ReadFile(serviceAccountDir + "/ca.crt")
if err != nil {
return err
}
tlsConfig.RootCAs = x509.NewCertPool()
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
if c.namespace == "" {
namespace, err := os.ReadFile(serviceAccountDir + "/namespace")
if err != nil {
return err
}
c.namespace = strings.TrimSpace(string(namespace))
}
c.inCluster = true
return nil
}
func (c *kubeClient) loadKubeconfig(path string, transport *http.Transport) error {
if paths := filepath.SplitList(os.Getenv("KUBECONFIG")); path == "" && len(paths) > 0 {
path = paths[0]
}
if path == "" {
home, err := os.UserHomeDir()
if err != nil {
return err
}
path = filepath.Join(home, ".kube", "config")
}
content, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("no in-cluster service account and no kubeconfig: %w", err)
}
var config struct {
CurrentContext string `yaml:"current-context"`
Contexts []struct {
Name string
Context struct{ Cluster, User, Namespace string }
}
Clusters []struct {
Name string
Cluster struct {
Server string
TLSServerName string `yaml:"tls-server-name"`
InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify"`
ProxyURL string `yaml:"proxy-url"`
CertificateAuthority string `yaml:"certificate-authority"`
CertificateAuthorityData string `yaml:"certificate-authority-data"`
}
}
Users []struct {
Name string
User struct {
Token string
TokenFile string `yaml:"tokenFile"`
ClientCertificate string `yaml:"client-certificate"`
ClientCertificateData string `yaml:"client-certificate-data"`
ClientKey string `yaml:"client-key"`
ClientKeyData string `yaml:"client-key-data"`
Exec any
AuthProvider any `yaml:"auth-provider"`
}
}
}
if err := yaml.Unmarshal(content, &config); err != nil {
return fmt.Errorf("parse kubeconfig %s: %w", path, err)
}
var current struct{ Cluster, User, Namespace string }
for _, entry := range config.Contexts {
if entry.Name == config.CurrentContext {
current = entry.Context
}
}
if current.Cluster == "" {
return fmt.Errorf("kubeconfig %s has no current context", path)
}
c.namespace = cmp.Or(c.namespace, current.Namespace, "default")
resolve := func(file string) string {
if file == "" || filepath.IsAbs(file) {
return file
}
return filepath.Join(filepath.Dir(path), file)
}
read := func(data, file string) ([]byte, error) {
if data != "" {
return base64.StdEncoding.DecodeString(data)
}
if file == "" {
return nil, nil
}
return os.ReadFile(resolve(file))
}
tlsConfig := transport.TLSClientConfig
for _, cluster := range config.Clusters {
if cluster.Name != current.Cluster {
continue
}
c.server = strings.TrimSuffix(cluster.Cluster.Server, "/")
tlsConfig.ServerName, tlsConfig.InsecureSkipVerify = cluster.Cluster.TLSServerName, cluster.Cluster.InsecureSkipTLSVerify
if cluster.Cluster.ProxyURL != "" {
proxy, err := url.Parse(cluster.Cluster.ProxyURL)
if err != nil {
return fmt.Errorf("kubeconfig %s: %w", path, err)
}
transport.Proxy = http.ProxyURL(proxy)
}
ca, err := read(cluster.Cluster.CertificateAuthorityData, cluster.Cluster.CertificateAuthority)
if err != nil {
return err
}
if ca != nil {
tlsConfig.RootCAs = x509.NewCertPool()
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
}
}
for _, user := range config.Users {
if user.Name != current.User {
continue
}
if user.User.Exec != nil || user.User.AuthProvider != nil {
return fmt.Errorf("kubeconfig %s: user %q authenticates through a credential plugin, which the runner does not support, use a token or client certificate", path, user.Name)
}
c.token, c.tokenFile = user.User.Token, resolve(user.User.TokenFile)
cert, err := read(user.User.ClientCertificateData, user.User.ClientCertificate)
if err != nil {
return err
}
key, err := read(user.User.ClientKeyData, user.User.ClientKey)
if err != nil {
return err
}
if cert != nil {
pair, err := tls.X509KeyPair(cert, key)
if err != nil {
return err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
}
if c.server == "" {
return fmt.Errorf("kubeconfig %s names no server for context %q", path, config.CurrentContext)
}
return nil
}
func (c *kubeClient) authorize(header http.Header) error {
token := c.token
if c.tokenFile != "" {
content, err := os.ReadFile(c.tokenFile)
if err != nil {
return err
}
token = strings.TrimSpace(string(content))
}
if token != "" {
header.Set("Authorization", "Bearer "+token)
}
return nil
}
func (c *kubeClient) path(resource string) string {
return "/api/v1/namespaces/" + c.namespace + "/" + resource
}
func (c *kubeClient) do(ctx context.Context, method, path string, in, out any) error {
var body io.Reader
if in != nil {
content, err := json.Marshal(in)
if err != nil {
return err
}
body = bytes.NewReader(content)
}
req, err := http.NewRequestWithContext(ctx, method, c.server+path, body)
if err != nil {
return err
}
if err := c.authorize(req.Header); err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusMultipleChoices {
return statusError(resp)
}
if out == nil {
out = io.Discard // draining lets the connection be reused
}
if writer, ok := out.(io.Writer); ok {
_, err = io.Copy(writer, resp.Body)
return err
}
return json.NewDecoder(resp.Body).Decode(out)
}
func (c *kubeClient) delete(ctx context.Context, resource, name string) error {
if name == "" {
return nil
}
if err := c.do(ctx, http.MethodDelete, c.path(resource+"/"+name), nil, nil); err != nil && !isNotFound(err) {
return fmt.Errorf("delete %s %s: %w", resource, name, err)
}
return nil
}
func statusError(resp *http.Response) error {
content, _ := io.ReadAll(resp.Body)
var status struct{ Message string }
if json.Unmarshal(content, &status) != nil || status.Message == "" {
status.Message = strings.TrimSpace(string(content))
}
return &apiError{code: resp.StatusCode, message: status.Message}
}
func (c *kubeClient) exec(ctx context.Context, pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
query := url.Values{"container": {containerName}, "command": command, "stdout": {"true"}, "stderr": {"true"}}
if stdin != nil {
query.Set("stdin", "true")
}
header := http.Header{}
if err := c.authorize(header); err != nil {
return err
}
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
conn, resp, err := websocket.Dial(ctx, c.server+c.path("pods/"+pod+"/exec?")+query.Encode(), &websocket.DialOptions{
HTTPClient: c.http,
HTTPHeader: header,
Subprotocols: []string{execProtocol},
})
if err != nil {
if resp != nil && resp.StatusCode >= http.StatusMultipleChoices {
defer resp.Body.Close()
err = statusError(resp)
}
return fmt.Errorf("exec in pod %s: %w", pod, err)
}
defer func() { _ = conn.CloseNow() }()
if conn.Subprotocol() != execProtocol {
return fmt.Errorf("the Kubernetes API server does not speak %s, which needs Kubernetes 1.30 or later", execProtocol)
}
conn.SetReadLimit(-1)
if stdin != nil {
go func() {
if err := sendStdin(ctx, conn, stdin); err != nil {
cancel(err)
}
}()
}
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if conn.Ping(ctx) != nil {
return
}
}
}
}()
for {
_, frame, err := conn.Read(ctx)
if err != nil {
return cmp.Or(context.Cause(ctx), fmt.Errorf("exec in pod %s ended without an exit status: %w", pod, err))
}
if len(frame) < 2 {
continue
}
switch frame[0] {
case 1:
_, err = stdout.Write(frame[1:])
case 2:
_, err = stderr.Write(frame[1:])
case 3:
return cmp.Or(context.Cause(ctx), execStatus(frame[1:]))
}
if err != nil {
return err
}
}
}
func sendStdin(ctx context.Context, conn *websocket.Conn, stdin io.Reader) error {
frame := make([]byte, 1+32*1024) // frame[0] is the stdin channel, 0
for {
n, err := stdin.Read(frame[1:])
if n > 0 {
if err := conn.Write(ctx, websocket.MessageBinary, frame[:1+n]); err != nil {
return err
}
}
if errors.Is(err, io.EOF) {
return conn.Write(ctx, websocket.MessageBinary, []byte{255, 0}) // v5 close signal for stdin
}
if err != nil {
return err
}
}
}
func execStatus(content []byte) error {
var status struct {
Status, Message string
Details struct {
Causes []struct{ Reason, Message string }
}
}
if err := json.Unmarshal(content, &status); err != nil {
return fmt.Errorf("decode exec status: %w", err)
}
if status.Status == "Success" {
return nil
}
for _, cause := range status.Details.Causes {
if code, err := strconv.Atoi(cause.Message); cause.Reason == "ExitCode" && err == nil {
return ExitCodeError(code)
}
}
return fmt.Errorf("exec failed: %s", status.Message)
}
-300
View File
@@ -1,300 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/coder/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type podManifest struct {
Metadata struct{ Labels map[string]string }
Spec struct {
AutomountServiceAccountToken bool
EnableServiceLinks bool
HostAliases []struct{ Hostnames []string }
Volumes []map[string]any
Containers []struct {
Name, ImagePullPolicy string
Resources map[string]any
StartupProbe *struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }
}
}
}
type fakeCluster struct {
mu sync.Mutex
requests map[string][]string
}
func (f *fakeCluster) record(request, content string) {
f.mu.Lock()
defer f.mu.Unlock()
f.requests[request] = append(f.requests[request], content)
}
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
cluster := &fakeCluster{requests: map[string][]string{}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer test-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
cluster.record(r.Method+" "+r.URL.Path, string(body))
pods, secrets := "/api/v1/namespaces/ci/pods", "/api/v1/namespaces/ci/secrets"
switch {
case r.Method == http.MethodGet && r.URL.Path == pods:
cluster.record("list", r.URL.Query().Get("labelSelector"))
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
case strings.HasSuffix(r.URL.Path, "/exec"):
cluster.record("exec", r.URL.RawQuery)
serveLocalExec(t, w, r)
case strings.HasSuffix(r.URL.Path, "/log"):
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
_, _ = io.WriteString(w, podStatus)
case r.Method == http.MethodPost && (r.URL.Path == pods || r.URL.Path == secrets), r.Method == http.MethodDelete:
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(kubeconfig), "token"), []byte("test-token\n"), 0o600))
require.NoError(t, os.WriteFile(kubeconfig, fmt.Appendf(nil, `
current-context: test
contexts: [{name: test, context: {cluster: test, user: test, namespace: ci}}]
clusters: [{name: test, cluster: {server: "http://kubernetes.invalid", proxy-url: %q}}]
users: [{name: test, user: {tokenFile: token}}]
`, server.URL), 0o600))
return kubeconfig, cluster
}
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
if err != nil {
t.Error(err)
return
}
defer func() { _ = conn.CloseNow() }()
conn.SetReadLimit(-1)
command := r.URL.Query()["command"]
var stdin bytes.Buffer
for r.URL.Query().Get("stdin") == "true" {
_, frame, err := conn.Read(r.Context())
if err != nil {
t.Error(err)
return
}
if frame[0] == 255 {
break
}
stdin.Write(frame[1:])
}
cmd := exec.Command(command[0], command[1:]...)
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
var stdout, stderr bytes.Buffer
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
status := `{"status":"Success"}`
var exitErr *exec.ExitError
if err := cmd.Run(); errors.As(err, &exitErr) {
status = fmt.Sprintf(`{"status":"Failure","reason":"NonZeroExitCode","details":{"causes":[{"reason":"ExitCode","message":"%d"}]}}`, exitErr.ExitCode())
} else {
assert.NoError(t, err)
}
for _, frame := range [][]byte{append([]byte{1}, stdout.Bytes()...), append([]byte{2}, stderr.Bytes()...), append([]byte{3}, status...)} {
_ = conn.Write(r.Context(), websocket.MessageBinary, frame)
}
}
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("uses POSIX shell")
}
ctx := t.Context()
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
{"name":"job","ready":true,"state":{"running":{}}},
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
workdir, err := filepath.EvalSymlinks(t.TempDir())
require.NoError(t, err)
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
{"spec": map[string]any{
"automountServiceAccountToken": true,
"containers": []any{
map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"cpu": "2"}}},
map[string]any{
"name": "$services", "imagePullPolicy": "Always", "resources": map[string]any{"limits": map[string]any{"memory": "256Mi"}},
"env": []any{map[string]any{"name": "POSTGRES_PASSWORD", "value": "template"}},
},
},
}},
{"spec": map[string]any{
"volumes": []any{map[string]any{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}}},
"containers": []any{map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}}},
}},
}}
pod, err := NewKubernetesPod(options)
require.NoError(t, err)
var jobLog, serviceLog bytes.Buffer
job := pod.JobContainer(&NewContainerInput{Name: "Job-Name", Image: "node", WorkingDir: workdir, Stdout: &jobLog, Stderr: &jobLog})
service := pod.ServiceContainer("My_DB", &NewContainerInput{
Image: "postgres",
Cmd: []string{"postgres", "-D", "$(PGDATA)"},
Env: []string{"POSTGRES_PASSWORD=service-secret"},
WorkflowOptions: "--health-cmd pg_isready --health-interval 2s --health-start-period 4s",
Stdout: &serviceLog,
})
require.NoError(t, job.Create(nil, nil)(ctx))
var manifest podManifest
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
assert.Equal(t, "gitea-runner", manifest.Metadata.Labels["app.kubernetes.io/managed-by"])
assert.Equal(t, "runner-uuid", manifest.Metadata.Labels[runnerUUIDLabel])
assert.True(t, manifest.Spec.AutomountServiceAccountToken)
assert.False(t, manifest.Spec.EnableServiceLinks)
assert.Equal(t, []string{"my-db"}, manifest.Spec.HostAliases[0].Hostnames)
require.Len(t, manifest.Spec.Containers, 2)
assert.Equal(t, map[string]any{"limits": map[string]any{"cpu": "2", "memory": "1Gi"}}, manifest.Spec.Containers[0].Resources)
assert.Empty(t, manifest.Spec.Containers[0].ImagePullPolicy)
assert.Equal(t, "my-db", manifest.Spec.Containers[1].Name)
assert.Equal(t, "Always", manifest.Spec.Containers[1].ImagePullPolicy)
assert.Equal(t, map[string]any{"limits": map[string]any{"memory": "256Mi"}}, manifest.Spec.Containers[1].Resources)
assert.Equal(t, []map[string]any{
{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}},
{"name": "act", "emptyDir": map[string]any{}},
}, manifest.Spec.Volumes)
assert.Equal(t, struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }{2, 30, 5}, *manifest.Spec.Containers[1].StartupProbe)
assert.NotContains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], "service-secret")
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], "service-secret")
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], `"com.gitea.runner.uuid":"runner-uuid"`)
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `{"name":"POSTGRES_PASSWORD","valueFrom":{"secretKeyRef":{"key":"my-db.0","name":"`+pod.secretName+`"}}}`)
assert.Equal(t, []any{"new"}, mergeManifest([]any{"old"}, []any{"new"}, ""))
mount := func(path string) any { return map[string]any{"name": "config", "mountPath": path} }
assert.Equal(t, []any{mount("/a"), mount("/b")}, mergeManifest([]any{mount("/a")}, []any{mount("/b")}, "volumeMounts"))
tmpfs := map[string]any{"name": "workspace", "emptyDir": map[string]any{"medium": "Memory"}}
ephemeral := map[string]any{"name": "workspace", "ephemeral": map[string]any{}}
assert.Equal(t, []any{ephemeral}, mergeManifest([]any{tmpfs}, []any{ephemeral}, "volumes"))
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
require.NoError(t, job.Start(false)(ctx))
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
envArgs, err := os.ReadFile(filepath.Join(workdir, "env-args"))
require.NoError(t, err)
assert.NotContains(t, string(envArgs), "hunter2")
require.NoError(t, job.Exec([]string{"printenv", "INPUT_WHO-TO-GREET"}, stepEnv, "", "")(ctx))
require.NoError(t, job.Exec([]string{"printenv", "UID"}, stepEnv, "", "")(ctx))
require.NoError(t, job.Exec([]string{"pwd", "-P"}, nil, "", workdir)(ctx))
assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
assert.Contains(t, jobLog.String(), "failing\n")
for _, query := range cluster.requests["exec"] {
assert.NotContains(t, query, "hunter2")
}
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
require.NoError(t, err)
reader := tar.NewReader(archive)
header, err := reader.Next()
require.NoError(t, err)
content, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "event.json", header.Name)
assert.JSONEq(t, `{"ref":"main"}`, string(content))
require.NoError(t, archive.Close())
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
require.Error(t, err)
actionDir := filepath.Join(t.TempDir(), "action")
require.NoError(t, os.MkdirAll(actionDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
require.NoError(t, err)
assert.Equal(t, "main()", string(script))
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
pathEnv := map[string]string{}
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
info, err := service.Inspect(ctx)
require.NoError(t, err)
assert.Equal(t, HealthHealthy, info.Health)
require.NoError(t, service.DumpLogs(ctx))
assert.Equal(t, "log of my-db", serviceLog.String())
require.NoError(t, job.Remove()(ctx))
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/"+pod.name], 1)
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/"+pod.secretName], 1)
require.NoError(t, RemoveOrphanKubernetesResources(ctx, options, time.Now()))
assert.Equal(t, []string{runnerUUIDLabel + "=runner-uuid"}, cluster.requests["list"])
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/old"], 1)
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/old"], 1)
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/new"])
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
}
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
template := map[string]any{"spec": map[string]any{"containers": []any{map[string]any{"name": "sidecar"}}}}
for _, test := range []struct {
services []string
options, err string
}{
{[]string{"db"}, "--hostname db", "only --health-* options are supported"},
{[]string{"sidecar"}, "", "named like another container"},
{[]string{"my_db", "My-DB"}, "", `service My-DB maps to container "my-db"`},
} {
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci", PodTemplates: []map[string]any{template}})
require.NoError(t, err)
job := pod.JobContainer(&NewContainerInput{Image: "node"})
for _, service := range test.services {
pod.ServiceContainer(service, &NewContainerInput{Image: "postgres", WorkflowOptions: test.options})
}
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
}
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
require.NoError(t, err)
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
require.NoError(t, job.Create(nil, nil)(t.Context()))
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
plugin := filepath.Join(t.TempDir(), "kubeconfig")
require.NoError(t, os.WriteFile(plugin, []byte(`
current-context: eks
contexts: [{name: eks, context: {cluster: eks, user: eks}}]
clusters: [{name: eks, cluster: {server: "https://eks.example"}}]
users: [{name: eks, user: {exec: {command: aws}}}]
`), 0o600))
_, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: plugin})
require.ErrorContains(t, err, "credential plugin")
}
@@ -11,7 +11,7 @@ import (
"runtime"
"strings"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
)
type LinuxContainerEnvironmentExtensions struct{}
@@ -21,13 +21,13 @@ type LinuxContainerEnvironmentExtensions struct{}
// For use in docker volumes and binds
func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
logging.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
return ""
}
abspath, err := filepath.Abs(path)
if err != nil {
logging.Error(err)
log.Error(err)
return ""
}
@@ -11,8 +11,7 @@ import (
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
@@ -28,7 +27,7 @@ func TestContainerPath(t *testing.T) {
if runtime.GOOS == "windows" {
cwd, err := os.Getwd()
if err != nil {
logging.Error(err)
log.Error(err)
}
rootDrive := os.Getenv("SystemDrive")
@@ -50,7 +49,7 @@ func TestContainerPath(t *testing.T) {
} else {
cwd, err := os.Getwd()
if err != nil {
logging.Error(err)
log.Error(err)
}
for _, v := range []containerPathJob{
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
+73
View File
@@ -0,0 +1,73 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"context"
"encoding/json"
"errors"
"fmt"
"os/exec"
"strings"
"time"
)
type macOSVM struct {
Name string `json:"name"`
State string `json:"state"`
Running bool `json:"running"`
AccessedAt string `json:"accessed_at"`
}
// CleanupOrphanMacOSVMs deletes local macOS VMs left behind by this runner that
// have not been accessed since cutoff.
func CleanupOrphanMacOSVMs(ctx context.Context, executorPath, uuid string, cutoff time.Time) error {
if uuid == "" {
return nil
}
output, err := exec.CommandContext(ctx, executorPath, "list", "--format", "json").Output()
if err != nil {
return fmt.Errorf("list macOS VMs: %w", err)
}
var vms []macOSVM
if err := json.Unmarshal(output, &vms); err != nil {
return fmt.Errorf("parse macOS VM list: %w", err)
}
var errs []error
for _, name := range macOSVMsToDelete(vms, uuid, cutoff) {
if err := exec.CommandContext(ctx, executorPath, "destroy", name).Run(); err != nil {
errs = append(errs, fmt.Errorf("destroy macOS VM %s: %w", name, err))
}
}
return errors.Join(errs...)
}
func macOSVMsToDelete(vms []macOSVM, uuid string, cutoff time.Time) []string {
prefix := "GITEA-MACOS-VM-" + uuid + "-"
var stale []string
for _, vm := range vms {
if vm.Running || !strings.HasPrefix(vm.Name, prefix) {
continue
}
accessed, err := parseMacOSVMTime(vm.AccessedAt)
if err != nil || accessed.After(cutoff) {
continue
}
stale = append(stale, vm.Name)
}
return stale
}
func parseMacOSVMTime(raw string) (time.Time, error) {
if raw == "" {
return time.Time{}, errors.New("empty access time")
}
if parsed, err := time.Parse(time.RFC3339Nano, raw); err == nil {
return parsed, nil
}
return time.Parse(time.RFC3339, raw)
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMacOSVMsToDelete(t *testing.T) {
cutoff := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
vms := []macOSVM{
{Name: "GITEA-MACOS-VM-runner-1-stale", AccessedAt: "2026-09-04T10:00:00Z"},
{Name: "GITEA-MACOS-VM-runner-1-running", AccessedAt: "2026-09-04T10:00:00Z", Running: true},
{Name: "GITEA-MACOS-VM-runner-1-fresh", AccessedAt: "2026-09-05T11:00:00Z"},
{Name: "GITEA-MACOS-VM-runner-2-stale", AccessedAt: "2026-09-04T10:00:00Z"},
}
assert.Equal(t, []string{"GITEA-MACOS-VM-runner-1-stale"}, macOSVMsToDelete(vms, "runner-1", cutoff))
}
func TestParseMacOSVMTime(t *testing.T) {
parsed, err := parseMacOSVMTime("2026-09-05T10:00:00Z")
require.NoError(t, err)
assert.Equal(t, time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC), parsed)
parsed, err = parseMacOSVMTime("2026-09-05T10:00:00.123456789Z")
require.NoError(t, err)
assert.Equal(t, time.Date(2026, 9, 5, 10, 0, 0, 123456789, time.UTC), parsed)
_, err = parseMacOSVMTime("")
require.Error(t, err)
}
+450
View File
@@ -0,0 +1,450 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"maps"
"os/exec"
"path"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/filecollector"
"github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
)
const (
defaultMacOSVMBootTimeout = 5 * time.Minute
macOSVMReadyPollInterval = 500 * time.Millisecond
macOSVMStopWaitTimeout = 10 * time.Second
)
// MacOSVMEnvironment runs workflow steps in an ephemeral macOS VM.
type MacOSVMEnvironment struct {
Path string // guest scratch directory
TmpDir string // guest temp directory
ToolCache string // guest tool cache directory
Workdir string // host-side workspace path
GuestWorkdir string // guest-side workspace path
ActPath string // guest act directory
Image string
VMName string
ExecutorPath string
CPU int
Memory int
BootTimeout time.Duration
Stdout io.Writer
Stderr io.Writer
executorPath string
running bool
runCmd *exec.Cmd
runDone chan struct{}
runErr error
}
// MacOSVMEnvironmentInput is the input for NewMacOSVMEnvironment.
type MacOSVMEnvironmentInput struct {
Path string
TmpDir string
ToolCache string
Workdir string
GuestWorkdir string
ActPath string
Image string
VMName string
ExecutorPath string
CPU int
Memory int
BootTimeout time.Duration
Stdout io.Writer
Stderr io.Writer
}
func NewMacOSVMEnvironment(input MacOSVMEnvironmentInput) (*MacOSVMEnvironment, error) {
if input.Image == "" {
return nil, errors.New("macOS VM image is required")
}
if input.VMName == "" {
return nil, errors.New("macOS VM name is required")
}
if input.BootTimeout <= 0 {
input.BootTimeout = defaultMacOSVMBootTimeout
}
if input.ExecutorPath == "" {
input.ExecutorPath = "runner-executor-macos"
}
return &MacOSVMEnvironment{
Path: input.Path,
TmpDir: input.TmpDir,
ToolCache: input.ToolCache,
Workdir: input.Workdir,
GuestWorkdir: input.GuestWorkdir,
ActPath: input.ActPath,
Image: input.Image,
VMName: input.VMName,
ExecutorPath: input.ExecutorPath,
CPU: input.CPU,
Memory: input.Memory,
BootTimeout: input.BootTimeout,
Stdout: input.Stdout,
Stderr: input.Stderr,
executorPath: input.ExecutorPath,
}, nil
}
var _ ExecutionsEnvironment = &MacOSVMEnvironment{}
func (e *MacOSVMEnvironment) Create(_, _ []string) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
// A previous run can leave the deterministic job VM behind. Destroy it first.
_ = e.runExecutor(ctx, nil, io.Discard, io.Discard, "destroy", e.VMName)
args := []string{"provision", e.Image, e.VMName}
if e.CPU > 0 {
args = append(args, "--cpu", strconv.Itoa(e.CPU))
}
if e.Memory > 0 {
args = append(args, "--memory", strconv.Itoa(e.Memory))
}
return e.runExecutor(ctx, nil, e.Stdout, e.Stderr, args...)
}
}
func (e *MacOSVMEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
for _, file := range files {
dest := path.Join(destPath, file.Name)
script := `mkdir -p "$1" && cat > "$2" && chmod "$3" "$2"`
mode := file.Mode
if mode == 0 {
mode = 0o644
}
args := []string{"exec", "--stdin", e.VMName, "/bin/sh", "-c", script, "sh", path.Dir(dest), dest, strconv.FormatInt(mode, 8)}
if err := e.runExecutor(ctx, strings.NewReader(file.Body), e.Stdout, e.Stderr, args...); err != nil {
return err
}
}
return nil
}
}
func (e *MacOSVMEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
pipeReader, pipeWriter := io.Pipe()
writeDone := make(chan error, 1)
go func() {
err := e.writeTar(ctx, pipeWriter, srcPath, useGitIgnore)
_ = pipeWriter.CloseWithError(err)
writeDone <- err
}()
script := `export COPYFILE_DISABLE=1; mkdir -p "$1" && /usr/bin/tar -xf - -C "$1"`
cmdErr := e.runExecutor(ctx, pipeReader, e.Stdout, e.Stderr, "exec", "--stdin", e.VMName, "/bin/sh", "-c", script, "sh", destPath)
_ = pipeReader.Close()
writeErr := <-writeDone
return errors.Join(cmdErr, writeErr)
}
}
func (e *MacOSVMEnvironment) writeTar(ctx context.Context, writer io.Writer, srcPath string, useGitIgnore bool) error {
tarWriter := tar.NewWriter(writer)
defer tarWriter.Close()
srcPrefix := filepath.Dir(srcPath)
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
srcPrefix += string(filepath.Separator)
}
var ignorer gitignore.Matcher
if useGitIgnore {
patterns, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
if err != nil {
common.Logger(ctx).Debugf("Error loading .gitignore: %v", err)
}
ignorer = gitignore.NewMatcher(patterns)
}
collector := &filecollector.FileCollector{
Ignorer: ignorer,
SrcPath: srcPath,
SrcPrefix: srcPrefix,
Handler: &filecollector.TarCollector{TarWriter: tarWriter},
}
return filepath.Walk(srcPath, collector.CollectFiles(ctx, []string{}))
}
func (e *MacOSVMEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
if common.Dryrun(ctx) {
return nil, errors.New("DRYRUN is not supported in GetContainerArchive")
}
cmd := exec.CommandContext(ctx, e.executorPath, "exec", e.VMName, "/usr/bin/env", "COPYFILE_DISABLE=1", "/usr/bin/tar", "-cf", "-", "-C", "/", strings.TrimPrefix(srcPath, "/"))
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
cmd.Stderr = e.Stderr
if err := cmd.Start(); err != nil {
return nil, err
}
return &macOSVMArchiveReader{ReadCloser: stdout, cmd: cmd}, nil
}
type macOSVMArchiveReader struct {
io.ReadCloser
cmd *exec.Cmd
}
func (r *macOSVMArchiveReader) Close() error {
closeErr := r.ReadCloser.Close()
return errors.Join(closeErr, r.cmd.Wait())
}
func (e *MacOSVMEnvironment) Inspect(ctx context.Context) (*Info, error) {
if common.Dryrun(ctx) {
return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
}
state := "exited"
if e.running {
state = StateRunning
}
return &Info{ID: e.VMName, State: state, Health: HealthNone, Ports: map[string]string{}}, nil
}
func (e *MacOSVMEnvironment) DumpLogs(context.Context) error {
return nil
}
func (e *MacOSVMEnvironment) Pull(forcePull bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
return nil
}
}
func (e *MacOSVMEnvironment) Start(bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
cmd := exec.Command(e.executorPath, "start", "--no-graphics", e.VMName)
cmd.Stdout = e.Stdout
cmd.Stderr = e.Stderr
if err := cmd.Start(); err != nil {
return err
}
e.runCmd = cmd
e.runDone = make(chan struct{})
go func() {
e.runErr = cmd.Wait()
close(e.runDone)
}()
e.running = true
return e.waitReady(ctx)
}
}
func (e *MacOSVMEnvironment) waitReady(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, e.BootTimeout)
defer cancel()
var lastErr error
for {
if err := e.runExecutor(ctx, nil, io.Discard, io.Discard, "exec", e.VMName, "/usr/bin/true"); err == nil {
return nil
} else {
lastErr = err
}
select {
case <-ctx.Done():
return fmt.Errorf("macOS VM %s did not become ready: %w", e.VMName, lastErr)
case <-e.runDone:
if e.runErr != nil {
return fmt.Errorf("macOS VM start exited before the VM became ready: %w", e.runErr)
}
return errors.New("macOS VM start exited before the VM became ready")
case <-time.After(macOSVMReadyPollInterval):
}
}
}
func (e *MacOSVMEnvironment) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
err := e.runExecutor(ctx, nil, e.Stdout, e.Stderr, macOSVMExecArgs(e.VMName, command, env, e.resolveWorkdir(workdir))...)
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return ExitCodeError(exitErr.ExitCode())
}
return err
}
}
func macOSVMExecArgs(vmName string, command []string, env map[string]string, workdir string) []string {
args := []string{"exec", vmName, "/usr/bin/env"}
for _, key := range slices.Sorted(maps.Keys(env)) {
args = append(args, key+"="+env[key])
}
args = append(args, "/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", workdir)
return append(args, command...)
}
func (e *MacOSVMEnvironment) resolveWorkdir(workdir string) string {
switch {
case workdir == "":
return e.GuestWorkdir
case strings.HasPrefix(workdir, "/"):
return workdir
default:
return path.Join(e.GuestWorkdir, workdir)
}
}
func (e *MacOSVMEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(e, srcPath, env)
}
func (e *MacOSVMEnvironment) UpdateFromImageEnv(*map[string]string) common.Executor {
return func(context.Context) error {
return nil
}
}
func (e *MacOSVMEnvironment) Remove() common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
cleanCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
if e.runCmd != nil && e.runDone != nil {
select {
case <-e.runDone:
default:
if err := e.runExecutor(cleanCtx, nil, e.Stdout, e.Stderr, "stop", e.VMName, "--timeout", "20"); err != nil {
common.Logger(ctx).Debugf("macOS VM stop failed for VM %s: %v", e.VMName, err)
}
select {
case <-e.runDone:
case <-time.After(macOSVMStopWaitTimeout):
if e.runCmd.Process != nil {
_ = e.runCmd.Process.Kill()
}
common.Logger(ctx).Warnf("timed out waiting for macOS VM start to exit for VM %s", e.VMName)
}
}
}
e.running = false
if err := e.runExecutor(cleanCtx, nil, e.Stdout, e.Stderr, "destroy", e.VMName); err != nil {
common.Logger(ctx).Warnf("failed to destroy macOS VM %s: %v", e.VMName, err)
}
return nil
}
}
func (e *MacOSVMEnvironment) Close() common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
if e.runCmd != nil && e.runCmd.Process != nil {
_ = e.runCmd.Process.Kill()
}
e.runCmd = nil
return nil
}
}
func (e *MacOSVMEnvironment) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
oldOut := e.Stdout
oldErr := e.Stderr
e.Stdout = stdout
e.Stderr = stderr
return oldOut, oldErr
}
func (e *MacOSVMEnvironment) ToContainerPath(hostPath string) string {
if filepath.Clean(hostPath) == filepath.Clean(e.Workdir) {
return e.GuestWorkdir
}
if rel, err := filepath.Rel(e.Workdir, hostPath); err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return path.Join(e.GuestWorkdir, filepath.ToSlash(rel))
}
return filepath.ToSlash(hostPath)
}
func (e *MacOSVMEnvironment) GetActPath() string {
return e.ActPath
}
func (*MacOSVMEnvironment) GetPathVariableName() string {
return "PATH"
}
func (*MacOSVMEnvironment) DefaultPathVariable() string {
return "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"
}
func (*MacOSVMEnvironment) JoinPathVariable(paths ...string) string {
return strings.Join(paths, ":")
}
func (e *MacOSVMEnvironment) GetRunnerContext(context.Context) map[string]any {
return map[string]any{
"os": "macOS",
"arch": goArchToActionArch(runtime.GOARCH),
"temp": e.TmpDir,
"tool_cache": e.ToolCache,
}
}
func (*MacOSVMEnvironment) IsEnvironmentCaseInsensitive() bool {
return false
}
func (e *MacOSVMEnvironment) runExecutor(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, args ...string) error {
if stdout == nil {
stdout = e.Stdout
}
if stderr == nil {
stderr = e.Stderr
}
cmd := exec.CommandContext(ctx, e.executorPath, args...)
cmd.Stdin = stdin
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("macOS VM executor %s: %w", args[0], err)
}
return nil
}
@@ -0,0 +1,97 @@
//go:build macos_vm_integration
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestMacOSVMEnvironmentIntegration(t *testing.T) {
if runtime.GOOS != "darwin" {
t.Skip("macOS VMs run on macOS")
}
if _, err := exec.LookPath("runner-executor-macos"); err != nil {
t.Skip("runner-executor-macos executable not found")
}
image := os.Getenv("MACOS_VM_TEST_IMAGE")
if image == "" {
t.Skip("set MACOS_VM_TEST_IMAGE to a macOS VM image to run this test")
}
ctx := context.Background()
vmName := fmt.Sprintf("gitea-runner-test-%d", time.Now().UnixNano())
base := "/Users/admin/runner"
if os.Getenv("RUNNER_EXECUTOR_MACOS_DEV_MODE") == "1" {
base = t.TempDir()
}
env, err := NewMacOSVMEnvironment(MacOSVMEnvironmentInput{
Path: base + "/scratch/" + vmName,
TmpDir: base + "/scratch/" + vmName + "/tmp",
ToolCache: base + "/scratch/" + vmName + "/tool_cache",
Workdir: t.TempDir(),
GuestWorkdir: base + "/workspace/" + vmName,
ActPath: base + "/scratch/" + vmName + "/act",
Image: image,
VMName: vmName,
ExecutorPath: "runner-executor-macos",
BootTimeout: 10 * time.Minute,
Stdout: io.Discard,
Stderr: io.Discard,
})
require.NoError(t, err)
t.Cleanup(func() {
_ = env.Remove()(context.Background())
})
require.NoError(t, env.Create(nil, nil)(ctx))
require.NoError(t, env.Start(false)(ctx))
if os.Getenv("RUNNER_EXECUTOR_MACOS_DEV_MODE") == "1" {
require.NoError(t, os.MkdirAll(env.GuestWorkdir, 0o755))
}
require.NoError(t, env.Exec([]string{"/bin/mkdir", "-p", env.GuestWorkdir}, nil, "", "/")(ctx))
require.NoError(t, env.Copy("/tmp", &FileEntry{
Name: "gitea-runner-macos.txt",
Mode: 0o644,
Body: "hello from macos-vm",
})(ctx))
archive, err := env.GetContainerArchive(ctx, "/tmp/gitea-runner-macos.txt")
require.NoError(t, err)
body := readFirstTarFile(t, archive)
require.NoError(t, archive.Close())
require.Equal(t, "hello from macos-vm", body)
var output bytes.Buffer
oldOut, oldErr := env.ReplaceLogWriter(&output, &output)
defer env.ReplaceLogWriter(oldOut, oldErr)
require.NoError(t, env.Exec([]string{"/usr/bin/uname", "-s"}, nil, "", "")(ctx))
require.Contains(t, output.String(), "Darwin")
require.NoError(t, env.Remove()(ctx))
}
func readFirstTarFile(t *testing.T, archive io.Reader) string {
t.Helper()
reader := tar.NewReader(archive)
header, err := reader.Next()
require.NoError(t, err)
require.Contains(t, header.Name, "gitea-runner-macos.txt")
body, err := io.ReadAll(reader)
require.NoError(t, err)
return string(body)
}
@@ -0,0 +1,91 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"context"
"io"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewMacOSVMEnvironment(t *testing.T) {
env, err := NewMacOSVMEnvironment(MacOSVMEnvironmentInput{
Image: "ghcr.io/cirruslabs/macos-sonoma-base:latest",
VMName: "job-vm",
})
require.NoError(t, err)
assert.Equal(t, "ghcr.io/cirruslabs/macos-sonoma-base:latest", env.Image)
assert.Equal(t, "job-vm", env.VMName)
assert.Equal(t, defaultMacOSVMBootTimeout, env.BootTimeout)
assert.Equal(t, "runner-executor-macos", env.executorPath)
_, err = NewMacOSVMEnvironment(MacOSVMEnvironmentInput{VMName: "job-vm"})
require.Error(t, err)
_, err = NewMacOSVMEnvironment(MacOSVMEnvironmentInput{Image: "macos"})
require.Error(t, err)
}
func TestMacOSVMExecArgs(t *testing.T) {
args := macOSVMExecArgs("vm", []string{"bash", "-c", "echo hi"}, map[string]string{"B": "two", "A": "one"}, "/Users/admin/work")
assert.Equal(t, []string{
"exec", "vm", "/usr/bin/env",
"A=one", "B=two",
"/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", "/Users/admin/work",
"bash", "-c", "echo hi",
}, args)
args = macOSVMExecArgs("vm", []string{"true"}, nil, "")
assert.Equal(t, []string{
"exec", "vm", "/usr/bin/env",
"/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", "",
"true",
}, args)
}
func TestMacOSVMEnvironmentResolveWorkdir(t *testing.T) {
env := &MacOSVMEnvironment{GuestWorkdir: "/Users/admin/work"}
assert.Equal(t, "/Users/admin/work", env.resolveWorkdir(""))
assert.Equal(t, "/Users/admin/work/sub", env.resolveWorkdir("sub"))
assert.Equal(t, "/tmp", env.resolveWorkdir("/tmp"))
}
func TestMacOSVMEnvironmentToContainerPath(t *testing.T) {
env := &MacOSVMEnvironment{Workdir: "/workspace/owner/repo", GuestWorkdir: "/Users/admin/work/owner/repo"}
assert.Equal(t, "/Users/admin/work/owner/repo", env.ToContainerPath("/workspace/owner/repo"))
assert.Equal(t, "/Users/admin/work/owner/repo/sub", env.ToContainerPath("/workspace/owner/repo/sub"))
assert.Equal(t, "/elsewhere", env.ToContainerPath("/elsewhere"))
}
func TestMacOSVMEnvironmentGetRunnerContext(t *testing.T) {
env := &MacOSVMEnvironment{
TmpDir: "/Users/admin/scratch/tmp",
ToolCache: "/Users/admin/scratch/tool_cache",
}
ctx := env.GetRunnerContext(context.Background())
assert.Equal(t, "macOS", ctx["os"])
assert.Equal(t, goArchToActionArch(runtime.GOARCH), ctx["arch"])
assert.Equal(t, "/Users/admin/scratch/tmp", ctx["temp"])
assert.Equal(t, "/Users/admin/scratch/tool_cache", ctx["tool_cache"])
}
func TestMacOSVMEnvironmentReplaceLogWriter(t *testing.T) {
oldOut := io.Discard
oldErr := io.Discard
env := &MacOSVMEnvironment{Stdout: oldOut, Stderr: oldErr}
newOut, newErr := env.ReplaceLogWriter(io.Discard, io.Discard)
assert.Equal(t, oldOut, newOut)
assert.Equal(t, oldErr, newErr)
}
func TestMacOSVMEnvironmentDefaultPathVariable(t *testing.T) {
env := &MacOSVMEnvironment{}
assert.Equal(t, "PATH", env.GetPathVariableName())
assert.Equal(t, "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin", env.DefaultPathVariable())
assert.Equal(t, "/a:/b", env.JoinPathVariable("/a", "/b"))
assert.False(t, env.IsEnvironmentCaseInsensitive())
}
+11 -10
View File
@@ -5,6 +5,7 @@ package container
import (
"bufio"
"context"
"os"
"path/filepath"
"strings"
@@ -27,7 +28,7 @@ func TestParseEnvFileSingleLine(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\nBAZ=qux\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "qux", env["BAZ"])
}
@@ -38,7 +39,7 @@ func TestParseEnvFileMultiLine(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "line1\nline2", env["FOO"])
}
@@ -49,7 +50,7 @@ func TestParseEnvFileLargeValueWithinLimit(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, big, env["FOO"])
}
@@ -60,7 +61,7 @@ func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
err := parseEnvFile(e, envPath, &env)(t.Context())
err := parseEnvFile(e, envPath, &env)(context.Background())
require.ErrorIs(t, err, bufio.ErrTooLong)
assert.Contains(t, err.Error(), "reading env file")
}
@@ -72,7 +73,7 @@ func TestParseEnvFileBlankLines(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte("\nFOO=bar\n\n \nBAZ=qux\n\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "qux", env["BAZ"])
}
@@ -83,12 +84,12 @@ func TestParseEnvFileMultiLineKeepsBlankLines(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\n\nline2\nEOF\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "line1\n\nline2", env["FOO"])
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n\nline2\nEOF\n"), 0o600))
env = map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "\nline2", env["FOO"])
}
@@ -98,7 +99,7 @@ func TestParseEnvFileUTF8BOM(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, content, 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
}
@@ -120,7 +121,7 @@ func TestParseEnvFileUTF16(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, content, 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "line1", env["MULTI"])
})
@@ -132,7 +133,7 @@ func TestParseEnvFileMissingDelimiter(t *testing.T) {
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
env := map[string]string{}
err := parseEnvFile(e, envPath, &env)(t.Context())
err := parseEnvFile(e, envPath, &env)(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "delimiter")
}
-55
View File
@@ -1,55 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"context"
"io"
"path/filepath"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/filecollector"
)
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
tw := tar.NewWriter(w)
for _, file := range files {
common.Logger(ctx).Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
if err := tw.WriteHeader(&tar.Header{
Name: file.Name,
Mode: file.Mode,
Size: int64(len(file.Body)),
Uid: uid,
Gid: gid,
}); err != nil {
return err
}
if _, err := tw.Write([]byte(file.Body)); err != nil {
return err
}
}
return tw.Close()
}
// writeDirTar prefixes entries with dstPath for extraction at root.
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
logger := common.Logger(ctx)
tw := tar.NewWriter(w)
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
SrcPath: srcPath,
SrcPrefix: srcPrefix,
SkipGitDir: skipGitDir,
UseGitIgnore: useGitIgnore,
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
}).CollectFiles(ctx)); err != nil {
return err
}
return tw.Close()
}
+5
View File
@@ -0,0 +1,5 @@
FROM scratch
ENV PATH="/this/path/does/not/exists/anywhere:/this/either"
ENV SOME_RANDOM_VAR=""
ENV ANOTHER_ONE="BUT_I_HAVE_VALUE"
ENV CONFLICT_VAR="I_EXIST_ONLY_HERE"
+50 -41
View File
@@ -16,10 +16,10 @@ import (
"path/filepath"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/git"
"golang.org/x/text/unicode/norm"
git "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/go-git/go-git/v5/plumbing/format/index"
)
type Handler interface {
@@ -94,62 +94,66 @@ func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string
}
type FileCollector struct {
SrcPath string
SrcPrefix string
SkipGitDir bool
UseGitIgnore bool
Handler Handler
Ignorer gitignore.Matcher
SrcPath string
SrcPrefix string
Handler Handler
}
// foldPath compares git's spelling of a path with the filesystem's, which may differ in case or Unicode normalization.
func foldPath(name string) string {
return strings.ToLower(norm.NFC.String(name))
}
func (fc *FileCollector) gitFiles(ctx context.Context) map[string]bool {
if !fc.UseGitIgnore {
return nil
}
files, err := git.ListFiles(ctx, fc.SrcPath)
func openGitIndex(path string) (*index.Index, error) {
repo, err := git.PlainOpen(path)
if err != nil {
common.Logger(ctx).Debugf("Copying %s without applying .gitignore: %v", fc.SrcPath, err)
return nil
return nil, err
}
keep := map[string]bool{".": true}
for _, file := range files {
for file = foldPath(file); !keep[file]; file = path.Dir(file) {
keep[file] = true
}
}
return keep
return repo.Storer.Index()
}
func (fc *FileCollector) CollectFiles(ctx context.Context) filepath.WalkFunc {
keep := fc.gitFiles(ctx)
func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc {
i, _ := openGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...)))
return func(file string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if ctx.Err() != nil {
if ctx != nil && ctx.Err() != nil {
return errors.New("copy cancelled")
}
if fc.SkipGitDir && fi.Name() == ".git" {
if fi.IsDir() {
return filepath.SkipDir
}
sansPrefix := strings.TrimPrefix(file, fc.SrcPrefix)
split := strings.Split(sansPrefix, string(filepath.Separator))
// The root folders should be skipped, submodules only have the last path component set to "." by filepath.Walk
if fi.IsDir() && len(split) > 0 && split[len(split)-1] == "." {
return nil
}
if keep != nil {
rel, _ := filepath.Rel(fc.SrcPath, file)
if rel = filepath.ToSlash(rel); !keep[foldPath(rel)] && !strings.Contains("/"+rel+"/", "/.git/") {
if fi.IsDir() {
var entry *index.Entry
if i != nil {
entry, err = i.Entry(strings.Join(split[len(submodulePath):], "/"))
} else {
err = index.ErrEntryNotFound
}
if err != nil && fc.Ignorer != nil && fc.Ignorer.Match(split, fi.IsDir()) {
if fi.IsDir() {
if i != nil {
ms, err := i.Glob(strings.Join(append(split[len(submodulePath):], "**"), "/"))
if err != nil || len(ms) == 0 {
return filepath.SkipDir
}
} else {
return filepath.SkipDir
}
} else {
return nil
}
}
path := filepath.ToSlash(strings.TrimPrefix(file, fc.SrcPrefix))
if err == nil && entry.Mode == filemode.Submodule {
err = filepath.Walk(file, fc.CollectFiles(ctx, split))
if err != nil {
return err
}
return filepath.SkipDir
}
path := filepath.ToSlash(sansPrefix)
// return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update)
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
linkName, err := os.Readlink(file)
if err != nil {
@@ -160,12 +164,17 @@ func (fc *FileCollector) CollectFiles(ctx context.Context) filepath.WalkFunc {
return nil
}
// open file
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
defer context.AfterFunc(ctx, func() { _ = f.Close() })()
if ctx != nil {
stop := context.AfterFunc(ctx, func() { _ = f.Close() })
defer stop()
}
return fc.Handler.WriteFile(path, fi, "", f)
}
+85 -52
View File
@@ -9,78 +9,106 @@ import (
"bytes"
"context"
"io"
"maps"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
git "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func gitCmd(t *testing.T, args ...string) {
t.Helper()
out, err := exec.Command("git", args...).CombinedOutput()
require.NoError(t, err, string(out))
}
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
func TestIgnoredTrackedfile(t *testing.T) {
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
gitCmd(t, "init", "-q", dir)
}
for name, body := range map[string]string{
".gitignore": ".*\nign/\n",
".env": "",
"Renamed": "",
"ign/tracked": "",
"ign/untracked": "",
"sub/.gitignore": "dropped\n",
"sub/dropped": "",
"sub/kept": "",
"nested/kept": "",
} {
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
}
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
for _, gitlink := range []string{"sub", "uninitialized"} {
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
}
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
if runtime.GOOS != "windows" {
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
expected["test.env"] = ".env"
}
repo, err := git.PlainInit(repoDir, false)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte(".*\n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
worktree, err := repo.Worktree()
require.NoError(t, err)
_, err = worktree.Add(".gitignore")
require.NoError(t, err)
var archive bytes.Buffer
tw := tar.NewWriter(&archive)
fc := &FileCollector{SrcPath: repoDir, SrcPrefix: repoDir + string(filepath.Separator), UseGitIgnore: true, Handler: &TarCollector{TarWriter: tw}}
require.NoError(t, filepath.Walk(repoDir, fc.CollectFiles(t.Context())))
require.NoError(t, tw.Close())
files := map[string]string{}
tr := tar.NewReader(&archive)
for header, err := tr.Next(); err != io.EOF; header, err = tr.Next() {
require.NoError(t, err)
files[header.Name] = header.Linkname
patterns, err := gitignore.ReadPatterns(worktree.Filesystem, nil)
require.NoError(t, err)
ignorer := gitignore.NewMatcher(patterns)
fc := &FileCollector{
Ignorer: ignorer,
SrcPath: repoDir,
SrcPrefix: repoDir + string(filepath.Separator),
Handler: &TarCollector{
TarWriter: tw,
},
}
assert.Contains(t, files, ".git/HEAD")
maps.DeleteFunc(files, func(name, _ string) bool { return strings.Contains("/"+name, "/.git/") })
assert.Equal(t, expected, files)
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
assert.NoError(t, err, "successfully collect files")
require.NoError(t, tw.Close())
tr := tar.NewReader(&archive)
h, err := tr.Next()
assert.NoError(t, err, "tar must not be empty") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, ".gitignore", h.Name)
_, err = tr.Next()
assert.ErrorIs(t, err, io.EOF, "tar must only contain one element")
}
func TestSymlinks(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires elevated privileges on Windows")
}
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
repo, err := git.PlainInit(repoDir, false)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
worktree, err := repo.Worktree()
require.NoError(t, err)
_, err = worktree.Add("test.env")
require.NoError(t, err)
var archive bytes.Buffer
tw := tar.NewWriter(&archive)
fc := &FileCollector{
SrcPath: repoDir,
SrcPrefix: repoDir + string(filepath.Separator),
Handler: &TarCollector{
TarWriter: tw,
},
}
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
assert.NoError(t, err, "successfully collect files")
require.NoError(t, tw.Close())
tr := tar.NewReader(&archive)
h, err := tr.Next()
files := map[string]tar.Header{}
for err == nil {
files[h.Name] = *h
h, err = tr.Next()
}
assert.Equal(t, ".env", files[".env"].Name)
assert.Equal(t, "test.env", files["test.env"].Name)
assert.Equal(t, ".env", files["test.env"].Linkname)
assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF")
}
// Regression for https://gitea.com/gitea/runner/issues/876 and /941:
// re-copying an action directory must overwrite a pre-existing read-only
// file (e.g. a git pack .idx at mode 0444) instead of failing with EACCES
// on macOS or "Access is denied" on Windows.
func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
dst := t.TempDir()
target := filepath.Join(dst, "sub", "pack.idx")
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
require.NoError(t, os.WriteFile(target, []byte("old"), 0o444))
fi, err := os.Stat(target)
src := filepath.Join(t.TempDir(), "pack.idx")
require.NoError(t, os.WriteFile(src, []byte("new"), 0o444))
fi, err := os.Stat(src)
require.NoError(t, err)
cc := &CopyCollector{DstDir: dst}
@@ -91,6 +119,8 @@ func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
assert.Equal(t, "new", string(got))
}
// Without the destination removal, os.Symlink fails with EEXIST when the
// path already holds a regular file from an earlier copy of the action.
func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires elevated privileges on Windows")
@@ -99,8 +129,11 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
target := filepath.Join(dst, "link")
require.NoError(t, os.WriteFile(target, []byte("stale"), 0o644))
fi, err := os.Lstat(target)
require.NoError(t, err)
cc := &CopyCollector{DstDir: dst}
require.NoError(t, cc.WriteFile("link", nil, "target", nil))
require.NoError(t, cc.WriteFile("link", fi, "target", nil))
resolved, err := os.Readlink(target)
require.NoError(t, err)
@@ -110,7 +143,7 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
func TestFileCollectorCancellationAndWalkError(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
walk := (&FileCollector{}).CollectFiles(ctx)
walk := (&FileCollector{}).CollectFiles(ctx, nil)
err := walk("file", nil, nil)
require.EqualError(t, err, "copy cancelled")
+7 -31
View File
@@ -12,7 +12,6 @@ import (
"context"
"fmt"
"strings"
"sync"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/git"
@@ -26,26 +25,6 @@ var (
findGithubRepo = git.FindGithubRepo
)
type pinnedKey struct{}
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
func WithPinnedCheckout(ctx context.Context) context.Context {
if ctx.Value(pinnedKey{}) != nil {
return ctx
}
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
}
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
cache, ok := ctx.Value(pinnedKey{}).(*sync.Map)
if !ok {
return lookup()
}
cached, _ := cache.LoadOrStore(key, sync.OnceValues(lookup))
lookupOnce, _ := cached.(func() (string, error))
return lookupOnce()
}
// SetRef resolves the ref of the context from its event payload, falling back
// to the ref checked out in repoPath.
func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
@@ -72,9 +51,9 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
}
if ghc.Ref == "" {
ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) })
ref, err := findGitRef(ctx, repoPath)
if err != nil {
logger.Warnf("unable to get git ref: %v", err)
logger.Warningf("unable to get git ref: %v", err)
} else {
logger.Debugf("using github ref: %s", ref)
ghc.Ref = ref
@@ -116,12 +95,9 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
}
if ghc.Sha == "" {
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
_, revision, err := findGitRevision(ctx, repoPath)
return revision, err
})
_, sha, err := findGitRevision(ctx, repoPath)
if err != nil {
logger.Warnf("unable to get git revision: %v", err)
logger.Warningf("unable to get git revision: %v", err)
} else {
ghc.Sha = sha
}
@@ -130,11 +106,11 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
// SetRepositoryAndOwner resolves the repository of the context from the git
// remote in repoPath when it is not set yet, and derives its owner.
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, githubInstance, repoPath string) {
if ghc.Repository == "" {
repo, err := findGithubRepo(ctx, repoPath)
repo, err := findGithubRepo(ctx, repoPath, githubInstance)
if err != nil {
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v, repoPath: %v): %v", githubInstance, repoPath, err)
return
}
ghc.Repository = repo
+6 -20
View File
@@ -9,14 +9,13 @@ import (
"errors"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.dev/actionslib/pkg/model"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestSetRef(t *testing.T) {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
oldFindGitRef := findGitRef
oldFindGitRevision := findGitRevision
@@ -105,7 +104,7 @@ func TestSetRef(t *testing.T) {
Event: table.event,
}
SetRef(t.Context(), ghc, "/some/dir")
SetRef(context.Background(), ghc, "/some/dir")
ghc.SetRefTypeAndName()
assert.Equal(t, table.ref, ghc.Ref)
@@ -123,14 +122,14 @@ func TestSetRef(t *testing.T) {
Event: map[string]any{},
}
SetRef(t.Context(), ghc, "/some/dir")
SetRef(context.Background(), ghc, "/some/dir")
assert.Equal(t, "refs/heads/master", ghc.Ref)
})
}
func TestSetSha(t *testing.T) {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
oldFindGitRef := findGitRef
oldFindGitRevision := findGitRevision
@@ -210,22 +209,9 @@ func TestSetSha(t *testing.T) {
Event: table.event,
}
SetSha(t.Context(), ghc, "/some/dir")
SetSha(context.Background(), ghc, "/some/dir")
assert.Equal(t, table.sha, ghc.Sha)
})
}
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
ctx := WithPinnedCheckout(t.Context())
for _, head := range []string{"first", "moved"} {
findGitRevision = func(context.Context, string) (string, string, error) { return "", head, nil }
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
ghc := &model.GithubContext{Event: map[string]any{}}
SetSha(ctx, ghc, "/some/dir")
SetRef(ctx, ghc, "/some/dir")
assert.Equal(t, "first", ghc.Sha)
assert.Equal(t, "refs/heads/first", ghc.Ref)
}
})
}
+65 -53
View File
@@ -6,7 +6,6 @@ package runner
import (
"context"
"crypto/rand"
"crypto/sha256"
"embed"
"encoding/hex"
@@ -14,7 +13,6 @@ import (
"fmt"
"io"
"io/fs"
"maps"
"os"
"path"
"path/filepath"
@@ -145,18 +143,18 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
containerActionDirCopy += `/`
}
unlock, err := git.AcquireCloneLock(ctx, actionDir)
if err != nil {
return err
}
defer unlock()
defer git.AcquireCloneLock(actionDir)()
if !rc.Config.NoActionPatch {
// A concurrent job's prepare resets this directory, so patch under the copy's lock.
defer patchActions(ctx, actionScriptPaths(filepath.Join(actionDir, actionPath), step.getActionModel()))()
patchActions(ctx, actionScriptPaths(filepath.Join(actionDir, actionPath), step.getActionModel()))
}
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", rc.Config.UseGitIgnore, true)(ctx)
if err := removeGitIgnore(ctx, actionDir); err != nil {
return err
}
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", rc.Config.UseGitIgnore)(ctx)
}
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
@@ -241,6 +239,23 @@ func nodeActionCommand(script string) []string {
return []string{"node", "--preserve-symlinks-main", script}
}
// https://github.com/nektos/act/issues/228#issuecomment-629709055
// files in .gitignore are not copied in a Docker container
// this causes issues with actions that ignore other important resources
// such as `node_modules` for example
func removeGitIgnore(ctx context.Context, directory string) error {
gitIgnorePath := path.Join(directory, ".gitignore")
if _, err := os.Stat(gitIgnorePath); err == nil {
// .gitignore exists
common.Logger(ctx).Debugf("Removing %s before docker cp", gitIgnorePath)
err := os.Remove(gitIgnorePath)
if err != nil {
return err
}
}
return nil
}
// dockerActionImageTag derives the local docker image tag used when an action
// is built from a Dockerfile.
//
@@ -271,30 +286,11 @@ func dockerActionImageTag(repository, actionName string, localAction bool) strin
return strings.ToLower(image)
}
// One image per step and job: its stages share a build, and concurrent jobs never run each other's.
func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, actionName string, localAction bool) string {
job := rc.topLevelRunContext()
if job.dockerActionImages == nil {
job.dockerActionImages = map[*model.Step]string{}
}
image, ok := job.dockerActionImages[step.getStepModel()]
if !ok {
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
job.dockerActionImages[step.getStepModel()] = image
}
}
return image
}
// TODO: break out parts of function to reduce complexicity
func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, basedir string, localAction bool, stage stepStage) error {
logger := common.Logger(ctx)
rc := step.getRunContext()
action := step.getActionModel()
if rc.kubernetes {
return errKubernetesDocker
}
var prepImage common.Executor
var image string
@@ -304,7 +300,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
// Apply forcePull only for prebuild docker images
forcePull = rc.Config.ForcePull
} else {
image = rc.dockerActionImage(ctx, step, actionName, localAction)
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction)
contextDir, fileName := filepath.Split(filepath.Join(basedir, action.Runs.Image))
anyArchExists, err := ContainerImageExistsLocally(ctx, image, "any")
@@ -349,11 +345,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
// Held across the whole build: the daemon drains contextDir lazily.
inner := prepImage
prepImage = func(ctx context.Context) error {
unlock, err := git.AcquireCloneLock(ctx, actionDir)
if err != nil {
return err
}
defer unlock()
defer git.AcquireCloneLock(actionDir)()
return inner(ctx)
}
}
@@ -361,7 +353,12 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
}
}
cmd, err := shellquote.Split(step.getStepModel().With["args"])
eval := rc.NewActionInputsExpressionEvaluator(ctx, step)
args, err := eval.Interpolate(ctx, step.getStepModel().With["args"])
if err != nil {
return fmt.Errorf("unable to interpolate with.args: %w", err)
}
cmd, err := shellquote.Split(args)
if err != nil {
return err
}
@@ -378,11 +375,14 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
}
}
}
entrypoint, err := dockerEntrypoint(step, stage)
entrypoint, err := dockerEntrypoint(ctx, step, eval, stage)
if err != nil {
return err
}
stepContainer, err := newStepContainer(ctx, step, image, cmd, entrypoint, rc.Config.ContainerOptions)
if err != nil {
return err
}
stepContainer := newStepContainer(ctx, step, image, cmd, entrypoint, rc.Config.ContainerOptions)
return common.NewPipelineExecutor(
prepImage,
stepContainer.Pull(forcePull),
@@ -394,7 +394,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
// dockerEntrypoint returns the entrypoint the action's image runs with for the given
// stage. Only the main stage honours the `entrypoint` input.
func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
func dockerEntrypoint(ctx context.Context, step actionStep, eval *expressionEvaluator, stage stepStage) ([]string, error) {
runs := step.getActionModel().Runs
var entrypoint string
@@ -406,7 +406,11 @@ func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
default:
entrypoint = runs.Entrypoint
if entrypoint == "" {
if fields := strings.Fields(step.getStepModel().With["entrypoint"]); len(fields) > 0 {
withEntrypoint, err := eval.Interpolate(ctx, step.getStepModel().With["entrypoint"])
if err != nil {
return nil, fmt.Errorf("unable to interpolate with.entrypoint: %w", err)
}
if fields := strings.Fields(withEntrypoint); len(fields) > 0 {
return fields, nil
}
}
@@ -421,6 +425,7 @@ func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
// evalDockerEnv returns an evaluator bound to the environment it installed.
func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expressionEvaluator, error) {
rc := step.getRunContext()
stepModel := step.getStepModel()
var err error
inputs := make(map[string]string)
@@ -431,33 +436,40 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
return nil, fmt.Errorf("unable to interpolate the default of input %s: %w", k, err)
}
}
mergeIntoMap(step, step.getEnv(), inputs, step.getStepModel().With)
for k, v := range stepModel.With {
if inputs[k], err = eval.Interpolate(ctx, v); err != nil {
return nil, fmt.Errorf("unable to interpolate with.%s: %w", k, err)
}
}
mergeIntoMap(step, step.getEnv(), inputs)
env := make(map[string]string, len(action.Runs.Env)+len(*step.getEnv()))
mergeIntoMap(step, &env, action.Runs.Env, *step.getEnv())
*step.getEnv() = env
ee := rc.NewActionInputsExpressionEvaluator(ctx, step)
runsEnv := make(map[string]string, len(action.Runs.Env))
for k, v := range action.Runs.Env {
if runsEnv[k], err = ee.Interpolate(ctx, v); err != nil {
for k, v := range *step.getEnv() {
if (*step.getEnv())[k], err = ee.Interpolate(ctx, v); err != nil {
return nil, fmt.Errorf("unable to interpolate env %s: %w", k, err)
}
}
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
return ee, nil
}
func newStepContainer(ctx context.Context, step step, image string, cmd, entrypoint []string, runnerOptions string) container.Container {
func newStepContainer(ctx context.Context, step step, image string, cmd, entrypoint []string, runnerOptions string) (container.Container, error) {
rc := step.getRunContext()
logWriter := rc.commandLogWriter(ctx)
envList := make([]string, 0)
for k, v := range *step.getEnv() {
if rc.IsHostEnv() && strings.EqualFold(k, rc.JobContainer.GetPathVariableName()) && v == rc.JobContainer.DefaultPathVariable() { // the runner's own PATH stays out, as on GitHub
continue
}
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
}
envList = append(envList, rc.runnerEnv(ctx)...)
binds, mounts := rc.GetBindsAndMounts(ctx)
binds, mounts, err := rc.GetBindsAndMounts()
if err != nil {
return nil, err
}
networkMode := "container:" + rc.jobContainerName()
if rc.IsHostEnv() {
networkMode = "default"
@@ -481,7 +493,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
AutoRemove: true,
ValidVolumes: rc.validVolumes(),
AllocatePTY: rc.Config.AllocatePTY,
})
}), nil
}
func populateEnvsFromSavedState(env *map[string]string, step actionStep, rc *RunContext) {
@@ -668,17 +680,17 @@ func shouldRunPostStep(step actionStep) common.Conditional {
stepResult := stepResults[step.getStepModel().ID]
if stepResult == nil {
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; step was not executed", step.getStepModel())
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; step was not executed", step.getStepModel())
return false
}
if stepResult.Conclusion == model.StepStatusSkipped {
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; main step was skipped", step.getStepModel())
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; main step was skipped", step.getStepModel())
return false
}
if step.getActionModel() == nil {
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s': no action model available", step.getStepModel())
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s': no action model available", step.getStepModel())
return false
}
+1 -4
View File
@@ -29,7 +29,7 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
}
}
ee := parent.NewStepExpressionEvaluator(ctx, step)
ee := parent.NewActionInputsExpressionEvaluator(ctx, step)
for inputID, input := range step.getActionModel().Inputs {
envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_")
@@ -104,9 +104,6 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action
Parent: parent,
EventJSON: parent.EventJSON,
platformImage: parent.platformImage,
kubernetes: parent.kubernetes,
jobIndex: parent.jobIndex,
jobTotal: parent.jobTotal,
}
compositerc.setCompositeActionEnv(env)
compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx)
+3 -54
View File
@@ -12,21 +12,18 @@ import (
"gitea.dev/actionslib/pkg/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v4"
)
func TestCompositeActionParity(t *testing.T) {
t.Run("inherits contexts without leaking inputs", func(t *testing.T) {
ctx := t.Context()
strategy := &model.Strategy{MaxParallelString: "3", RawMatrix: yaml.Node{Kind: yaml.MappingNode}}
strategy := &model.Strategy{MaxParallel: 3}
parent := &RunContext{
Config: &Config{},
Matrix: map[string]any{"os": "linux"},
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Name: "workflow", Jobs: map[string]*model.Job{"job": {Strategy: strategy}}}},
JobContainer: &jobContainerMock{},
platformImage: "-self-hosted",
jobIndex: 1,
jobTotal: 4,
}
composite, err := newCompositeRunContext(ctx, parent, &stepActionRemote{
Step: &model.Step{With: map[string]string{"SHARED": "outer"}},
@@ -39,9 +36,9 @@ func TestCompositeActionParity(t *testing.T) {
assert.Same(t, strategy, composite.Run.Job().Strategy)
assert.True(t, composite.IsHostEnv())
interpolated, err := composite.NewExpressionEvaluator(ctx).Interpolate(ctx,
"${{ matrix.os }}|${{ strategy.max-parallel }}|${{ strategy.job-index }}|${{ inputs.shared }}")
"${{ matrix.os }}|${{ strategy.max-parallel }}|${{ inputs.shared }}")
require.NoError(t, err)
assert.Equal(t, "linux|3|1|outer", interpolated)
assert.Equal(t, "linux|3|outer", interpolated)
assert.NotContains(t, composite.Env, "INPUT_SHARED")
nestedEnv := composite.GetEnv()
@@ -120,51 +117,3 @@ func TestAppendUniqueMasksNoExponentialGrowth(t *testing.T) {
assert.Equal(t, []string{"secret"}, parentMasks)
}
func TestCompositeStepIfReadsJobStatusExceptInItsOwnMainSteps(t *testing.T) {
failStep := func(rc *RunContext) {
rc.StepResults["failed"] = &model.StepResult{Conclusion: model.StepStatusFailure}
}
tests := []struct {
name string
setup func(job, composite *RunContext)
nested bool
stage stepStage
want map[string]bool
}{
{"post-if reads the failed job", func(job, _ *RunContext) { failStep(job) }, false, stepStagePost, map[string]bool{"success()": false, "failure()": true, "job.status == 'failure'": true}},
{"main if ignores the failed job", func(job, _ *RunContext) { failStep(job) }, false, stepStageMain, map[string]bool{"success()": true, "failure()": false}},
{"main if reads the composite's own failure", func(_, composite *RunContext) { failStep(composite) }, false, stepStageMain, map[string]bool{"success()": false, "failure()": true}},
{"nested main if ignores the outer composite's failure", func(_, composite *RunContext) { failStep(composite) }, true, stepStageMain, map[string]bool{"success()": true}},
{"main if sees the cancelled job", func(job, _ *RunContext) { job.markCancelled() }, false, stepStageMain, map[string]bool{"cancelled()": true, "success()": false}},
{"post-if sees the cancelled job", func(job, _ *RunContext) { job.markCancelled() }, false, stepStagePost, map[string]bool{"cancelled()": true, "success()": false}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
job := makeTestRC(t, "runs-on: ubuntu-latest")
composite := newTestCompositeRunContext(t, job)
rc := composite
if tt.nested {
rc = newTestCompositeRunContext(t, composite)
}
tt.setup(job, composite)
step := &stepRun{RunContext: rc}
for expr, want := range tt.want {
enabled, err := isStepEnabled(t.Context(), expr, step, tt.stage)
require.NoError(t, err)
assert.Equal(t, want, enabled, expr)
}
})
}
}
func newTestCompositeRunContext(t *testing.T, parent *RunContext) *RunContext {
composite, err := newCompositeRunContext(t.Context(), parent, &stepActionRemote{
Step: &model.Step{},
RunContext: parent,
action: &model.Action{},
env: map[string]string{},
}, "/action")
require.NoError(t, err)
return composite
}
+21 -31
View File
@@ -132,7 +132,7 @@ runs:
return nil
}
action, err := readActionImpl(t.Context(), tt.step, "actionDir", "actionPath", readFile, writeFile)
action, err := readActionImpl(context.Background(), tt.step, "actionDir", "actionPath", readFile, writeFile)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, action)
@@ -143,7 +143,6 @@ runs:
}
func TestActionRunner(t *testing.T) {
t.Chdir(t.TempDir())
table := []struct {
name string
step actionStep
@@ -225,10 +224,10 @@ func TestActionRunner(t *testing.T) {
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
cm := &containerMock{}
cm.On("CopyDir", "/var/run/act/actions/dir/", "dir/", false, true).Return(func(ctx context.Context) error { return nil })
cm.On("CopyDir", "/var/run/act/actions/dir/", "dir/", false).Return(func(ctx context.Context) error { return nil })
envMatcher := mock.MatchedBy(func(env map[string]string) bool {
for k, v := range tt.expectedEnv {
@@ -243,7 +242,7 @@ func TestActionRunner(t *testing.T) {
tt.step.getRunContext().JobContainer = cm
err := runActionImpl(tt.step, "dir", mustNewRemoteAction(t, "org/repo/path@ref"))(ctx)
err := runActionImpl(tt.step, "dir", newRemoteAction("org/repo/path@ref"))(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
cm.AssertExpectations(t)
@@ -290,7 +289,7 @@ func TestNewStepContainerDoesNotUseDockerSecrets(t *testing.T) {
ContainerNewContainer = origContainerNewContainer
}()
ctx := t.Context()
ctx := context.Background()
rc := &RunContext{
Name: "job",
Config: &Config{
@@ -317,7 +316,7 @@ func TestNewStepContainerDoesNotUseDockerSecrets(t *testing.T) {
step.On("getStepModel").Return(&model.Step{ID: "action"})
step.On("getEnv").Return(&env)
newStepContainer(ctx, step, "registry.example.com/action:tag", nil, nil, "")
_, _ = newStepContainer(ctx, step, "registry.example.com/action:tag", nil, nil, "")
// DOCKER_USERNAME/DOCKER_PASSWORD should not be injected as pull credentials for docker action containers.
assert.Empty(t, captured.Username)
@@ -327,7 +326,7 @@ func TestNewStepContainerDoesNotUseDockerSecrets(t *testing.T) {
}
func TestMaybeCopyToActionDirHoldsCloneLock(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
actionDir := t.TempDir()
@@ -338,7 +337,7 @@ func TestMaybeCopyToActionDirHoldsCloneLock(t *testing.T) {
copyEntered := make(chan struct{})
cm := &containerMock{}
cm.On("CopyDir", "/var/run/act/actions/", actionDir+"/", false, true).Return(func(ctx context.Context) error {
cm.On("CopyDir", "/var/run/act/actions/", actionDir+"/", false).Return(func(ctx context.Context) error {
close(copyEntered)
<-releaseCopy
return nil
@@ -367,10 +366,7 @@ func TestMaybeCopyToActionDirHoldsCloneLock(t *testing.T) {
peerAcquired := make(chan struct{})
go func() {
unlock, err := git.AcquireCloneLock(t.Context(), actionDir)
if !assert.NoError(t, err) {
return
}
unlock := git.AcquireCloneLock(actionDir)
close(peerAcquired)
unlock()
}()
@@ -404,9 +400,7 @@ func TestMaybeCopyToActionDirHoldsCloneLock(t *testing.T) {
func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
actionDir := t.TempDir()
unlock, err := git.AcquireCloneLock(t.Context(), actionDir)
require.NoError(t, err)
unlockOnce := sync.OnceFunc(unlock)
unlockOnce := sync.OnceFunc(git.AcquireCloneLock(actionDir))
defer unlockOnce()
innerEntered := make(chan struct{})
@@ -447,7 +441,7 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
env: map[string]string{},
}
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
@@ -467,11 +461,18 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
case <-innerEntered:
case err := <-done:
t.Fatalf("execAsDocker returned without entering inner: %v", err)
case <-time.After(time.Second):
t.Fatal("inner build executor not entered after lock released")
}
cancel()
releaseOnce()
<-done
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("execAsDocker did not return after inner was released and ctx was canceled")
}
}
func TestDockerActionImageTag(t *testing.T) {
@@ -519,17 +520,6 @@ func TestDockerActionImageTag(t *testing.T) {
dockerActionImageTag("owner/repo", "./", true),
dockerActionImageTag("owner/repo", "./sub", true),
)
workflow := &model.Workflow{Jobs: map[string]*model.Job{"job1": {}}}
job := newTestRC(workflow, nil)
step := &stepActionRemote{Step: &model.Step{}, RunContext: job}
image := job.dockerActionImage(t.Context(), step, "abc123", false)
assert.True(t, strings.HasPrefix(image, "act-abc123-dockeraction:latest-"))
assert.Equal(t, image, (&RunContext{Parent: job}).dockerActionImage(t.Context(), step, "abc123", false))
assert.NotEqual(t, image, job.dockerActionImage(t.Context(), &stepActionRemote{Step: &model.Step{}, RunContext: job}, "abc123", false))
assert.NotEqual(t, image, newTestRC(workflow, nil).dockerActionImage(t.Context(), step, "abc123", false))
job.dockerActionImage(common.WithDryrun(t.Context(), true), &stepActionRemote{Step: &model.Step{}, RunContext: job}, "abc123", false)
assert.Len(t, job.dockerActionImages, 2)
}
func TestExecAsDockerStageEntrypoint(t *testing.T) {
@@ -619,7 +609,7 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
cm.On("Start", true).Return(func(context.Context) error { return nil })
cm.On("Close").Return(func(context.Context) error { return nil })
require.NoError(t, execAsDocker(t.Context(), step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
require.NoError(t, execAsDocker(context.Background(), step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
require.NotNil(t, input)
assert.Equal(t, tc.wantCmd, input.Cmd)
assert.Equal(t, tc.wantEntrypoint, input.Entrypoint)
@@ -634,7 +624,7 @@ func TestDockerActionHasPreAndPostStep(t *testing.T) {
newStep := func(runs model.ActionRuns) actionStep {
return &stepActionRemote{action: &model.Action{Runs: runs}}
}
ctx := t.Context()
ctx := context.Background()
assert.False(t, hasPreStep(newStep(model.ActionRuns{Using: "docker", Image: "Dockerfile"}))(ctx))
assert.False(t, hasPostStep(newStep(model.ActionRuns{Using: "docker", Image: "Dockerfile"}))(ctx))
+11 -14
View File
@@ -14,7 +14,6 @@ import (
"gitea.dev/actionslib/pkg/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace"
"go.yaml.in/yaml/v4"
)
@@ -62,7 +61,7 @@ func TestCancelledJobStatusEnablesAlwaysAndCancelledSteps(t *testing.T) {
enabled, err := interp.Evaluate("always()", exprparser.DefaultStatusCheckSuccess)
require.NoError(t, err)
assert.Equal(t, true, enabled, "`if: always()` step must run on a cancelled job")
setJobResult(t.Context(), rc, rc, true)
setJobResult(context.Background(), rc, rc, true)
assert.Equal(t, "cancelled", rc.Run.Job().Result)
}
@@ -76,7 +75,7 @@ func TestMainStepsExecutorRunsAlwaysStepsAfterCancel(t *testing.T) {
"job1": createJob(t, `runs-on: ubuntu-latest`, ""),
})
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var ran []string
@@ -110,7 +109,7 @@ func TestMainStepsExecutorMarksFailedOnTimeoutBetweenSteps(t *testing.T) {
"job1": createJob(t, `runs-on: ubuntu-latest`, ""),
})
ctx := newControllableDeadlineContext(t.Context())
ctx := newControllableDeadlineContext(context.Background())
var ran []string
var laterStepCtxErr error
@@ -149,7 +148,7 @@ func TestStepsExecutorRunsMainStepsAfterPreCancel(t *testing.T) {
"job1": createJob(t, `runs-on: ubuntu-latest`, ""),
})
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var ran []string
@@ -200,7 +199,7 @@ func TestStepsExecutorRunsMainStepsAfterPreFailure(t *testing.T) {
},
}
err := newStepsExecutor(rc, preSteps, steps)(t.Context())
err := newStepsExecutor(rc, preSteps, steps)(context.Background())
require.ErrorIs(t, err, assert.AnError, "the pre-step error is propagated")
assert.Equal(t, []string{"pre1", "always-step"}, ran, "the main always() step runs after a pre-step failure")
@@ -215,7 +214,7 @@ func TestPreStepFailureAffectsMainStepIfStatus(t *testing.T) {
rc := createIfTestRunContext(map[string]*model.Job{
"job1": createJob(t, `runs-on: ubuntu-latest`, ""),
})
ctx := common.WithJobErrorContainer(t.Context())
ctx := common.WithJobErrorContainer(context.Background())
reportStepError(ctx, rc, assert.AnError)
@@ -250,8 +249,7 @@ func TestPreStepFailureAffectsMainStepIfStatus(t *testing.T) {
// SetJobError dereferences a nil map and panics. The post context must therefore be detached
// from cancellation (so the steps run) yet still carry a usable error container.
func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) {
jobSpan := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
cancelled, cancel := context.WithCancel(trace.ContextWithSpanContext(common.WithJobErrorContainer(t.Context()), jobSpan))
cancelled, cancel := context.WithCancel(common.WithJobErrorContainer(context.Background()))
cancel()
require.ErrorIs(t, cancelled.Err(), context.Canceled)
@@ -260,7 +258,6 @@ func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) {
// Detached from cancellation, so the post steps actually run.
require.NoError(t, postCtx.Err(), "post context must not be cancelled")
assert.Equal(t, jobSpan, trace.SpanContextFromContext(postCtx), "post steps must stay in the job's trace")
// A failing post step records its error instead of panicking.
require.NotPanics(t, func() {
@@ -273,7 +270,7 @@ func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) {
// job-error container (via context.WithoutCancel), so the timeout failure and any post-step error
// survive into the post phase and the job is still reported as failed.
func TestPostStepsContextDeadlinePreservesJobError(t *testing.T) {
base := common.WithJobErrorContainer(t.Context())
base := common.WithJobErrorContainer(context.Background())
common.SetJobError(base, assert.AnError)
expired, cancel := context.WithDeadline(base, time.Now().Add(-time.Hour))
defer cancel()
@@ -292,14 +289,14 @@ func TestReportStepErrorTreatsCancelAsInterruption(t *testing.T) {
rc := &RunContext{}
// stray read cancellation while the job context is live: ignored, not a failure
live := common.WithJobErrorContainer(t.Context())
live := common.WithJobErrorContainer(context.Background())
reportStepError(live, rc, context.Canceled)
require.NoError(t, common.JobError(live))
assert.False(t, rc.jobFailed)
assert.False(t, rc.jobCancelled)
// genuine job cancellation: recorded as cancelled, still not a failure
cancelled, cancel := context.WithCancel(common.WithJobErrorContainer(t.Context()))
cancelled, cancel := context.WithCancel(common.WithJobErrorContainer(context.Background()))
cancel()
reportStepError(cancelled, rc, context.Canceled)
require.NoError(t, common.JobError(cancelled))
@@ -307,7 +304,7 @@ func TestReportStepErrorTreatsCancelAsInterruption(t *testing.T) {
assert.True(t, rc.jobCancelled)
// a real error still fails the job
failed := common.WithJobErrorContainer(t.Context())
failed := common.WithJobErrorContainer(context.Background())
reportStepError(failed, rc, assert.AnError)
require.ErrorIs(t, common.JobError(failed), assert.AnError)
assert.True(t, rc.jobFailed)
+2 -2
View File
@@ -117,7 +117,7 @@ func (rc *RunContext) refuseUnsecureCommand(ctx context.Context, command string)
}
// The step executor logs the failure itself, so keep this line's wording distinct.
common.Logger(ctx).With(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(fmt.Sprintf(
common.Logger(ctx).WithField(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(fmt.Sprintf(
"The `%s` command is disabled: it can set the environment of every later step from untrusted output. "+
"Write to $GITHUB_ENV or $GITHUB_PATH instead, or set ACTIONS_ALLOW_UNSECURE_COMMANDS to allow it",
command)))
@@ -159,7 +159,7 @@ func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg
if fromCommand {
message = "Can't update NODE_OPTIONS environment variable using ::set-env:: command."
}
common.Logger(ctx).With(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(message))
common.Logger(ctx).WithField(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(message))
return
}
common.Logger(ctx).Infof("::set-env:: %s=%s", name, arg)
+19 -18
View File
@@ -6,14 +6,15 @@ package runner
import (
"bytes"
"context"
"io"
"os"
"testing"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.dev/actionslib/pkg/model"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -25,8 +26,8 @@ func unsecureRC() *RunContext {
func TestSetEnv(t *testing.T) {
a := assert.New(t)
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
rc := unsecureRC()
handler := rc.commandHandler(ctx)
@@ -36,7 +37,7 @@ func TestSetEnv(t *testing.T) {
rc.setEnvFile(ctx, map[string]string{"name": "node_options"}, "--require env.js")
a.NotContains(rc.Env, "NODE_OPTIONS")
a.NotContains(rc.Env, "node_options")
entries := hook.AllRecords()
entries := hook.AllEntries()
require.Len(t, entries, 3)
a.Equal("##[error]Can't update NODE_OPTIONS environment variable using ::set-env:: command.", entries[1].Message)
a.Equal("##[error]Can't store NODE_OPTIONS output parameter using '$GITHUB_ENV' command.", entries[2].Message)
@@ -44,7 +45,7 @@ func TestSetEnv(t *testing.T) {
func TestStopCommandsKeepsSuppressedLinesInLog(t *testing.T) {
a := assert.New(t)
ctx := t.Context()
ctx := context.Background()
rc := unsecureRC()
handler := rc.commandHandler(ctx)
@@ -67,7 +68,7 @@ func TestStopCommandsKeepsSuppressedLinesInLog(t *testing.T) {
func TestSetOutput(t *testing.T) {
a := assert.New(t)
ctx := t.Context()
ctx := context.Background()
rc := new(RunContext)
rc.StepResults = make(map[string]*model.StepResult)
handler := rc.commandHandler(ctx)
@@ -107,7 +108,7 @@ func TestSetOutput(t *testing.T) {
func TestAddpath(t *testing.T) {
a := assert.New(t)
ctx := t.Context()
ctx := context.Background()
rc := unsecureRC()
handler := rc.commandHandler(ctx)
@@ -119,10 +120,10 @@ func TestAddpath(t *testing.T) {
}
func TestStopCommands(t *testing.T) {
logger, hook := logtest.NewLogger()
logger, hook := test.NewNullLogger()
a := assert.New(t)
ctx := common.WithLogger(t.Context(), logger)
ctx := common.WithLogger(context.Background(), logger)
rc := unsecureRC()
handler := rc.commandHandler(ctx)
@@ -136,7 +137,7 @@ func TestStopCommands(t *testing.T) {
a.Equal("abcd", rc.Env["x"])
messages := make([]string, 0)
for _, entry := range hook.AllRecords() {
for _, entry := range hook.AllEntries() {
messages = append(messages, entry.Message)
}
@@ -148,7 +149,7 @@ func TestStopCommands(t *testing.T) {
func TestStopCommandsResumesOnCommandNamedToken(t *testing.T) {
a := assert.New(t)
rc := unsecureRC()
handler := rc.commandHandler(t.Context())
handler := rc.commandHandler(context.Background())
handler("::stop-commands::add-mask\n")
handler("::set-env name=x::suppressed\n")
@@ -161,7 +162,7 @@ func TestStopCommandsResumesOnCommandNamedToken(t *testing.T) {
func TestAddpathADO(t *testing.T) {
a := assert.New(t)
ctx := t.Context()
ctx := context.Background()
rc := unsecureRC()
handler := rc.commandHandler(ctx)
@@ -173,17 +174,17 @@ func TestAddpathADO(t *testing.T) {
}
func TestAddmask(t *testing.T) {
logger, hook := logtest.NewLogger()
logger, hook := test.NewNullLogger()
a := assert.New(t)
ctx := t.Context()
ctx := context.Background()
loggerCtx := common.WithLogger(ctx, logger)
rc := new(RunContext)
handler := rc.commandHandler(loggerCtx)
handler("::ADD-MASK::my::secret")
a.Equal("***", hook.LastRecord().Message)
a.Equal("***", hook.LastEntry().Message)
a.Equal([]string{"my::secret"}, rc.Masks)
}
@@ -230,7 +231,7 @@ func TestAddmaskUsemask(t *testing.T) {
}
re := captureOutput(t, func() {
ctx := t.Context()
ctx := context.Background()
ctx = WithJobLogger(ctx, "0", "testjob", config, &rc.Masks, map[string]any{})
handler := rc.commandHandler(ctx)
@@ -247,7 +248,7 @@ func TestSaveState(t *testing.T) {
StepResults: map[string]*model.StepResult{},
}
ctx := t.Context()
ctx := context.Background()
handler := rc.commandHandler(ctx)
handler("::save-state name=state-name::state-value\n")
@@ -280,7 +281,7 @@ func TestUnsecureCommands(t *testing.T) {
a := assert.New(t)
rc := &RunContext{Env: tt.jobEnv}
rc.setCurrentStepEnv(tt.stepEnv)
handler := rc.commandHandler(t.Context())
handler := rc.commandHandler(context.Background())
handler("::set-env name=x::valz\n")
handler("::add-path::/opt/bin\n")
+2 -2
View File
@@ -57,8 +57,8 @@ func (cm *containerMock) Copy(destPath string, files ...*container.FileEntry) co
return args.Get(0).(func(context.Context) error)
}
func (cm *containerMock) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
args := cm.Called(destPath, srcPath, useGitIgnore, skipGitDir)
func (cm *containerMock) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
args := cm.Called(destPath, srcPath, useGitIgnore)
return args.Get(0).(func(context.Context) error)
}
-110
View File
@@ -1,110 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package runner
import (
"context"
"crypto/rand"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.com/gitea/runner/act/container"
"gitea.dev/actionslib/pkg/model"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDockerProxyMountedJob(t *testing.T) {
mode := os.Getenv("ACT_TEST_DOCKER_PROXY")
if mode == "" {
t.Skip("set ACT_TEST_DOCKER_PROXY=proxy or direct to verify mounted Docker access")
}
require.Contains(t, []string{"proxy", "direct"}, mode)
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
defer cancel()
dockerClient, err := container.GetDockerClient(ctx)
require.NoError(t, err)
t.Cleanup(func() { assert.NoError(t, dockerClient.Close()) })
require.True(t, strings.HasPrefix(dockerClient.DaemonHost(), "unix://"), "mounted Docker coverage requires a Unix daemon socket")
fixtureDir, err := filepath.Abs("testdata/docker-proxy")
require.NoError(t, err)
resourceName := "gitea-proxy-test-" + strings.ToLower(rand.Text())
runner, err := New(&Config{
Workdir: fixtureDir,
ActionCacheDir: t.TempDir(),
EventName: "push",
PlatformPicker: mapPlatformPicker(platforms),
ContainerNamePrefix: resourceName,
ContainerDaemonSocket: dockerClient.DaemonHost(),
ContainerMaxLifetime: 2 * time.Minute,
ContainerOptions: "-v /usr/local/bin/docker:/usr/local/bin/docker:ro -v /usr/local/libexec/docker/cli-plugins:/usr/local/libexec/docker/cli-plugins:ro",
ValidVolumes: []string{"/usr/local/bin/docker", "/usr/local/libexec/docker/cli-plugins"},
Env: map[string]string{"PROXY_TEST_RESOURCE": resourceName, "PROXY_TEST_MODE": mode, "PROXY_TEST_IMAGE": baseImage, "COMPOSE_PROJECT_NAME": resourceName},
})
require.NoError(t, err)
planner, err := model.NewWorkflowPlanner(filepath.Join(fixtureDir, "push.yml"), true)
require.NoError(t, err)
plan, err := planner.PlanEvent("push")
if mode == "direct" {
plan, err = planner.PlanJob("proxy")
}
require.NoError(t, err)
runContext, err := runner.newRunContext(ctx, plan.Stages[0].Runs[0], nil)
require.NoError(t, err)
jobName := runContext.jobContainerName()
jobNetwork, _ := runContext.networkNameForGitea()
t.Cleanup(func() {
cleanCtx, cleanCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cleanCancel()
if _, err := dockerClient.ContainerRemove(cleanCtx, jobName, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); !cerrdefs.IsNotFound(err) {
assert.NoError(t, err)
}
for _, name := range []string{resourceName, jobNetwork} {
if _, err := dockerClient.NetworkRemove(cleanCtx, name, client.NetworkRemoveOptions{}); !cerrdefs.IsNotFound(err) {
assert.NoError(t, err)
}
}
for _, name := range []string{resourceName, jobName, jobName + "-env"} {
if _, err := dockerClient.VolumeRemove(cleanCtx, name, client.VolumeRemoveOptions{}); !cerrdefs.IsNotFound(err) {
assert.NoError(t, err)
}
}
})
hook := logtest.New()
require.NoError(t, runner.NewPlanExecutor(plan)(common.WithLoggerHandler(ctx, hook)))
var messages []string
for _, entry := range hook.AllRecords() {
messages = append(messages, strings.TrimSpace(entry.Message))
}
require.Contains(t, messages, "docker proxy post verified")
if mode == "proxy" {
require.Contains(t, messages, "docker binds verified")
}
_, err = dockerClient.ContainerInspect(ctx, jobName, client.ContainerInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err), "job container survived cleanup: %v", err)
_, err = dockerClient.NetworkInspect(ctx, resourceName, client.NetworkInspectOptions{})
if mode == "proxy" {
assert.True(t, cerrdefs.IsNotFound(err), "labelled network survived cleanup: %v", err)
} else {
require.NoError(t, err)
}
_, err = dockerClient.VolumeInspect(ctx, resourceName, client.VolumeInspectOptions{})
if mode == "proxy" {
assert.True(t, cerrdefs.IsNotFound(err), "labelled volume survived cleanup: %v", err)
} else {
require.NoError(t, err)
}
}
+98 -54
View File
@@ -13,6 +13,7 @@ import (
"reflect"
"regexp"
"strings"
"sync"
"time"
"gitea.com/gitea/runner/act/common"
@@ -35,9 +36,24 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map
var workflowCallResult map[string]*model.WorkflowCallResult
// todo: cleanup EvaluationEnvironment creation
using := make(map[string]exprparser.Needs)
strategy := make(map[string]any)
if rc.Run != nil {
strategy = exprparser.StrategyContext(rc.Run.Job().Strategy, rc.jobIndex, rc.jobTotal)
job := rc.Run.Job()
if job != nil && job.Strategy != nil {
strategy["fail-fast"] = job.Strategy.FailFast
strategy["max-parallel"] = job.Strategy.MaxParallel
}
jobs := rc.Run.Workflow.Jobs
jobNeeds := rc.Run.Job().Needs()
for _, needs := range jobNeeds {
using[needs] = exprparser.Needs{
Outputs: jobs[needs].Outputs,
Result: jobs[needs].NeedsResult(),
}
}
// only setup jobs context in case of workflow_call
// and existing expression evaluator (this means, jobs are at
@@ -45,7 +61,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map
if rc.caller != nil && rc.ExprEval != nil {
workflowCallResult = map[string]*model.WorkflowCallResult{}
for jobName, job := range rc.Run.Workflow.Jobs {
for jobName, job := range jobs {
result := model.WorkflowCallResult{
Outputs: map[string]string{},
}
@@ -70,7 +86,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map
Vars: getWorkflowVars(ctx, rc),
Strategy: strategy,
Matrix: rc.Matrix,
Needs: exprparser.NeedsContext(rc.Run),
Needs: using,
Inputs: inputs,
HashFiles: getHashFilesFunction(ctx, rc),
}
@@ -89,26 +105,44 @@ var hashfiles string
// NewStepExpressionEvaluator creates a new evaluator with the `inputs` of the enclosing workflow or composite action
func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext())
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs)
}
// NewActionInputsExpressionEvaluator creates a new evaluator with the step's own with: values as `inputs`
func (rc *RunContext) NewActionInputsExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext())
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()))
}
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext) *ExpressionEvaluator {
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any) *ExpressionEvaluator {
// todo: cleanup EvaluationEnvironment creation
job := rc.Run.Job()
strategy := make(map[string]any)
if job.Strategy != nil {
strategy["fail-fast"] = job.Strategy.FailFast
strategy["max-parallel"] = job.Strategy.MaxParallel
}
jobs := rc.Run.Workflow.Jobs
jobNeeds := rc.Run.Job().Needs()
using := make(map[string]exprparser.Needs)
for _, needs := range jobNeeds {
using[needs] = exprparser.Needs{
Outputs: jobs[needs].Outputs,
Result: jobs[needs].NeedsResult(),
}
}
ee := &exprparser.EvaluationEnvironment{
Github: step.getGithubContext(ctx),
Env: *step.getEnv(),
Job: jobContext,
Job: rc.getJobContext(),
Steps: rc.getStepsContext(),
Secrets: getWorkflowSecrets(rc),
Vars: getWorkflowVars(ctx, rc),
Strategy: exprparser.StrategyContext(rc.Run.Job().Strategy, rc.jobIndex, rc.jobTotal),
Strategy: strategy,
Matrix: rc.Matrix,
Needs: exprparser.NeedsContext(rc.Run),
Needs: using,
// todo: should be unavailable
// but required to interpolate/evaluate the inputs in actions/composite
Inputs: getEvaluatorInputs(rc, stepInputs, rc.getGithubContext(ctx)),
@@ -246,44 +280,50 @@ func inputsFromEnv(env map[string]string) map[string]any {
}
func getEvaluatorInputs(rc *RunContext, stepInputs map[string]any, ghc *model.GithubContext) map[string]any {
if stepInputs != nil { // an action's own inputs, which workflow inputs must not shadow
return maps.Clone(stepInputs)
}
inputs := map[string]any{}
// only a called workflow's job itself takes the inputs Gitea resolved,
// composite actions copy its Config but have their own `inputs`
if rc.Config.WorkflowCallInputs != nil && rc.Parent == nil {
maps.Copy(inputs, rc.Config.WorkflowCallInputs)
return inputs
}
maps.Copy(inputs, rc.workflowCallInputs)
maps.Copy(inputs, stepInputs)
switch ghc.EventName {
case "workflow_dispatch":
if config := rc.Run.Workflow.WorkflowDispatchConfig(); config != nil {
for name, input := range config.Inputs {
if input.Type == "number" {
input.Type = "string" // boolean is the only typed dispatch input, as in Gitea
if ghc.EventName == "workflow_dispatch" {
config := rc.Run.Workflow.WorkflowDispatchConfig()
if config != nil && config.Inputs != nil {
for k, v := range config.Inputs {
value := nestedMapLookup(ghc.Event, "inputs", k)
if value == nil {
value = v.Default
}
setEvaluatorInput(inputs, ghc, name, model.WorkflowCallInput{Type: input.Type, Default: input.Default})
inputs[k] = coerceInputValue(value, v.Type)
}
}
case "workflow_call":
for name, input := range rc.Run.Workflow.WorkflowCallConfig().Inputs {
setEvaluatorInput(inputs, ghc, name, input)
}
if ghc.EventName == "workflow_call" {
config := rc.Run.Workflow.WorkflowCallConfig()
if config != nil && config.Inputs != nil {
for k, v := range config.Inputs {
value := nestedMapLookup(ghc.Event, "inputs", k)
if value == nil {
value = v.Default
}
inputs[k] = coerceInputValue(value, v.Type)
}
}
}
return inputs
}
func setEvaluatorInput(inputs map[string]any, ghc *model.GithubContext, name string, input model.WorkflowCallInput) {
value := model.NestedMapLookup(ghc.Event, "inputs", name)
if value == nil {
value, _ = input.DefaultValue()
// coerceInputValue converts an input value to the type declared by the workflow.
// The event payload carries natively typed JSON values on newer Gitea versions,
// while defaults and older servers provide strings.
func coerceInputValue(value any, inputType string) any {
if inputType != "boolean" {
return value
}
inputs[name] = model.CoerceInputValue(value, input.Type)
if b, ok := value.(bool); ok {
return b
}
return value == "true"
}
// resolveWorkflowCall evaluates the caller's with: and secrets: once, as the server does before dispatching a called workflow.
@@ -291,32 +331,36 @@ func (rc *RunContext) resolveWorkflowCall(ctx context.Context) error {
if rc.caller == nil {
return nil
}
logger := common.Logger(ctx)
callerJob := rc.caller.runContext.Run.Job()
callerEval := rc.caller.runContext.ExprEval.shared(ctx)
var callerWith map[string]any
if err := model.DecodeEvaluated("workflow inputs", callerJob.RawWith, callerEval.EvaluateYamlNode, &callerWith); err != nil {
return err
}
if !rc.Run.Workflow.IsWorkflowCall() {
logger.Warn("workflow_call key is not defined in the referenced workflow")
}
callerEval := rc.caller.runContext.ExprEval
calleeEval := sync.OnceValue(func() *expressionEvaluator { return rc.NewExpressionEvaluator(ctx) })
config := rc.Run.Workflow.WorkflowCallConfig()
var err error
if rc.workflowCallInputs, err = callerEval.ResolveWorkflowCallInputs(config, callerWith); err != nil {
logger.Warn(err)
rc.workflowCallInputs = make(map[string]any, len(config.Inputs))
for name, input := range config.Inputs {
value, eval, label := callerJob.With[name], callerEval, "input"
if value == nil {
value, eval, label = input.Default, calleeEval(), "the default of input"
}
if str, ok := value.(string); ok {
var err error
if value, err = eval.Interpolate(ctx, str); err != nil {
return fmt.Errorf("unable to interpolate %s %s: %w", label, name, err)
}
}
rc.workflowCallInputs[name] = coerceInputValue(value, input.Type)
}
if callerJob.InheritSecrets() {
rc.workflowCallSecrets = getWorkflowSecrets(rc.caller.runContext)
return nil
}
secrets := callerJob.Secrets()
if err := config.ValidateSecrets(secrets); err != nil {
logger.Warn(err)
if secrets == nil && callerJob.InheritSecrets() {
secrets = rc.caller.runContext.Config.Secrets
}
if rc.workflowCallSecrets, err = callerEval.EvaluateCallerSecrets(secrets); err != nil {
return fmt.Errorf("unable to interpolate %w", err)
rc.workflowCallSecrets = make(map[string]string, len(secrets))
for k, v := range secrets {
var err error
if rc.workflowCallSecrets[k], err = callerEval.Interpolate(ctx, v); err != nil {
return fmt.Errorf("unable to interpolate secret %s: %w", k, err)
}
}
return nil
}
+11 -127
View File
@@ -5,6 +5,7 @@
package runner
import (
"context"
"strings"
"testing"
@@ -81,14 +82,14 @@ func createRunContext(t *testing.T) *RunContext {
func TestEvaluateRunContext(t *testing.T) {
rc := createRunContext(t)
ee := rc.NewExpressionEvaluator(t.Context())
ee := rc.NewExpressionEvaluator(context.Background())
tables := []struct {
in string
out any
errMesg string
}{
{" 1 ", 1.0, ""},
{" 1 ", 1, ""},
// {"1 + 3", "4", ""},
// {"(1 + 3) * -2", "-8", ""},
{"'my text'", "my text", ""},
@@ -141,7 +142,7 @@ func TestEvaluateRunContext(t *testing.T) {
for _, table := range tables {
t.Run(table.in, func(t *testing.T) {
assertObject := assert.New(t)
out, err := ee.evaluate(t.Context(), table.in, exprparser.DefaultStatusCheckNone)
out, err := ee.evaluate(context.Background(), table.in, exprparser.DefaultStatusCheckNone)
if table.errMesg == "" {
assertObject.NoError(err, table.in) //nolint:testifylint // pre-existing issue from nektos/act
assertObject.Equal(table.out, out, table.in)
@@ -161,7 +162,7 @@ func TestEvaluateStep(t *testing.T) {
env: map[string]string{"INPUT_FORGED": "leaked"},
}
ee := rc.NewStepExpressionEvaluator(t.Context(), step)
ee := rc.NewStepExpressionEvaluator(context.Background(), step)
tables := []struct {
in string
@@ -183,7 +184,7 @@ func TestEvaluateStep(t *testing.T) {
for _, table := range tables {
t.Run(table.in, func(t *testing.T) {
assertObject := assert.New(t)
out, err := ee.evaluate(t.Context(), table.in, exprparser.DefaultStatusCheckNone)
out, err := ee.evaluate(context.Background(), table.in, exprparser.DefaultStatusCheckNone)
if table.errMesg == "" {
assertObject.NoError(err, table.in) //nolint:testifylint // pre-existing issue from nektos/act
assertObject.Equal(table.out, out, table.in)
@@ -223,7 +224,7 @@ func TestInterpolate(t *testing.T) {
},
},
}
ee := rc.NewExpressionEvaluator(t.Context())
ee := rc.NewExpressionEvaluator(context.Background())
tables := []struct {
in string
out string
@@ -265,18 +266,19 @@ func TestInterpolate(t *testing.T) {
{"${{ null }}", ""},
{"${{ fromJSON('[1,2]') }}", "Array"},
{"${{ fromJSON('{\"a\":1}') }}", "Object"},
{"${{ 1", "${{ 1"},
}
for _, table := range tables {
t.Run("interpolate", func(t *testing.T) {
out, err := ee.Interpolate(t.Context(), table.in)
out, err := ee.Interpolate(context.Background(), table.in)
require.NoError(t, err, table.in)
assert.Equal(t, table.out, out, table.in)
})
}
for _, in := range []string{"${{ 1) && (2 }}", "run ${{ 1) && (2 }} now", "${{ 1"} {
_, err := ee.Interpolate(t.Context(), in)
for _, in := range []string{"${{ 1) && (2 }}", "run ${{ 1) && (2 }} now"} {
_, err := ee.Interpolate(context.Background(), in)
assert.Error(t, err, in)
}
}
@@ -293,8 +295,6 @@ on:
name:
type: string
default: gitea
unset:
type: boolean
`,
"workflow_dispatch": `
on:
@@ -306,8 +306,6 @@ on:
name:
type: string
default: gitea
unset:
type: boolean
`,
}
@@ -359,123 +357,9 @@ on:
inputs := getEvaluatorInputs(rc, nil, ghc)
assert.Equal(t, table.flag, inputs["flag"])
assert.Equal(t, "gitea", inputs["name"])
assert.Equal(t, false, inputs["unset"])
})
}
}
t.Run("deferred workflow call inputs", func(t *testing.T) {
workflow, err := model.ReadWorkflow(strings.NewReader(workflows["workflow_call"] + `
jobs:
test: {}
call:
uses: ./reuse.yml
with: ${{ fromJSON(matrix.args) }}
`))
require.NoError(t, err)
workflow.GetJob("test")
job := workflow.GetJob("call")
rawWith := model.CloneYamlNode(job.RawWith)
for _, value := range []string{"true", "false"} {
t.Run(value, func(t *testing.T) {
t.Parallel()
parent, err := (&runnerImpl{config: &Config{}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "call"}, map[string]any{"args": `{"flag":` + value + `,"name":"${{ 'runner' }}"}`})
require.NoError(t, err)
child, err := (&runnerImpl{config: &Config{}, caller: &caller{runContext: parent}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
require.NoError(t, err)
assert.Equal(t, map[string]any{"flag": value == "true", "name": "${{ 'runner' }}", "unset": false}, child.workflowCallInputs)
assert.Equal(t, rawWith, job.RawWith)
assert.Nil(t, job.With)
})
}
})
}
func TestGetEvaluatorInputsResolvedByGitea(t *testing.T) {
wf, err := model.ReadWorkflow(strings.NewReader(`
on:
workflow_call:
inputs:
name: {type: string, default: callee}
workflow_dispatch:
inputs:
name: {type: string}
`))
require.NoError(t, err)
config := &Config{Workdir: ".", WorkflowCallInputs: map[string]any{"name": "caller", "target": "dispatch"}}
ghc := &model.GithubContext{EventName: "workflow_dispatch", Event: map[string]any{"inputs": map[string]any{"name": "dispatch"}}}
// the event's own inputs must not override those Gitea resolved
job := &RunContext{Config: config, Run: &model.Run{JobID: "job1", Workflow: wf}}
assert.Equal(t, map[string]any{"name": "caller", "target": "dispatch"}, getEvaluatorInputs(job, nil, ghc))
composite := &RunContext{Config: config, Run: &model.Run{JobID: "job1", Workflow: &model.Workflow{}}, Parent: job}
assert.Equal(t, map[string]any{"arg": "x"}, getEvaluatorInputs(composite, map[string]any{"arg": "x"}, ghc))
}
func TestActionInputsShadowDispatchInputs(t *testing.T) {
wf, err := model.ReadWorkflow(strings.NewReader(`
on:
workflow_dispatch:
inputs:
message: {type: string}
jobs:
job1: {}
`))
require.NoError(t, err)
rc := newTestRC(wf, nil)
rc.Config.EventName = "workflow_dispatch"
rc.EventJSON = `{"inputs":{"message":"workflow"}}`
step := &stepActionRemote{RunContext: rc, env: map[string]string{"INPUT_MESSAGE": "step"}}
assert.Equal(t, "step", rc.NewActionInputsExpressionEvaluator(t.Context(), step).InterpolateName(t.Context(), "${{ inputs.message }}"))
assert.Equal(t, "workflow", rc.NewStepExpressionEvaluator(t.Context(), step).InterpolateName(t.Context(), "${{ inputs.message }}"))
enabled, err := isStepEnabled(t.Context(), "inputs.message == 'workflow'", step, stepStageMain)
require.NoError(t, err)
assert.True(t, enabled)
step.Step, step.action = &model.Step{}, &model.Action{Inputs: map[string]model.Input{"fallback": {Default: "${{ inputs.message }}"}}}
env, err := evaluateCompositeInputAndEnv(t.Context(), rc, step)
require.NoError(t, err)
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
}
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) {
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
jobs:
call:
uses: ./reuse.yml
with: {flag: not-a-boolean}
secrets: {undeclared: "${{ secrets.token }}"}
inherit:
uses: ./reuse.yml
secrets: inherit
`))
require.NoError(t, err)
callee, err := model.ReadWorkflow(strings.NewReader(`
on:
workflow_call:
inputs:
flag: {type: boolean, default: true}
jobs:
test: {}
`))
require.NoError(t, err)
notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n"))
require.NoError(t, err)
config := &Config{Secrets: map[string]string{"token": "${{ github.token }}"}}
resolve := func(callerJobID string, workflow *model.Workflow) *RunContext {
parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil)
require.NoError(t, err)
child, err := (&runnerImpl{config: config, caller: &caller{runContext: parent}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
require.NoError(t, err)
return child
}
called := resolve("call", callee)
assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs)
assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets)
inherited := resolve("inherit", notCallable)
assert.Empty(t, inherited.workflowCallInputs)
assert.Equal(t, config.Secrets, inherited.workflowCallSecrets)
}
func TestJobNameMasksSecrets(t *testing.T) {
+2 -12
View File
@@ -4,7 +4,7 @@
package runner
import (
"os"
"context"
"os/exec"
"runtime"
"testing"
@@ -29,7 +29,7 @@ func requireLinuxDocker(t *testing.T) {
// so the daemon has to be pinged explicitly here to decide whether to skip.
func requireDocker(t *testing.T) {
t.Helper()
ctx := t.Context()
ctx := context.Background()
cli, err := container.GetDockerClient(ctx)
if err != nil {
t.Skipf("skipping: docker client unavailable: %v", err)
@@ -40,16 +40,6 @@ func requireDocker(t *testing.T) {
}
}
func requireDockerSharingFiles(t *testing.T) {
t.Helper()
requireDocker(t)
for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} {
if _, err := os.Stat(marker); err == nil {
t.Skip("skipping: inside a container, the docker daemon cannot bind-mount this process's files")
}
}
}
// requireHostTools skips the test unless every named executable is on PATH. Used by the
// self-hosted (host environment) suite, which runs steps directly on the host.
func requireHostTools(t *testing.T, tools ...string) {
+60 -97
View File
@@ -13,12 +13,12 @@ import (
"errors"
"fmt"
"io"
"maps"
"net/http"
"path"
"slices"
"strconv"
"strings"
"sync"
"time"
"unicode"
@@ -29,10 +29,7 @@ import (
"gitea.dev/actionslib/pkg/model"
)
const (
jobCleanupTimeout = time.Minute
maxJobSummaryBytes = 1024 * 1024
)
const maxJobSummaryBytes = 1024 * 1024
// jobSummaryTruncationMarker is appended to a summary that exceeded the size limit
// so the rendered output makes the truncation visible instead of silently cutting off.
@@ -92,14 +89,14 @@ func printPrepareActions(rc *RunContext, preparers []actionPreparer) common.Exec
return nil
}
rawLogger := common.Logger(ctx).With(rawOutputField, true)
rawLogger := common.Logger(ctx).WithField(rawOutputField, true)
rawLogger.Infof("Prepare all required actions")
for _, preparer := range preparers {
if err := preparer.prepareActionExecutor()(ctx); err != nil {
// No step has run yet, so the failure belongs to the job.
reportStepError(ctx, rc, err)
return setupError{err}
return err
}
reference, sha, ok := preparer.actionDownloadInfo()
if !ok {
@@ -115,11 +112,6 @@ func printPrepareActions(rc *RunContext, preparers []actionPreparer) common.Exec
}
}
// setupError is a failure of the job setup, such as an action that cannot be downloaded.
type setupError struct{ error }
func (e setupError) Unwrap() error { return e.error }
// printCompleteJobName closes the setup section the way actions/runner ends its "Set up job" step.
func printCompleteJobName(rc *RunContext) common.Executor {
return func(ctx context.Context) error {
@@ -131,7 +123,7 @@ func printCompleteJobName(rc *RunContext) common.Executor {
if name == "" && rc.Run != nil {
name = rc.Run.JobID
}
common.Logger(ctx).With(rawOutputField, true).Infof("Complete job name: %s", name)
common.Logger(ctx).WithField(rawOutputField, true).Infof("Complete job name: %s", name)
return nil
}
}
@@ -144,18 +136,20 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
preparers := make([]actionPreparer, 0)
var postExecutor common.Executor
steps = append(steps, func(ctx context.Context) error {
logger := common.Logger(ctx)
if len(info.matrix()) > 0 {
logger.Infof("Matrix: %v", info.matrix())
}
return nil
})
infoSteps := info.steps()
if len(infoSteps) == 0 {
return common.NewDebugExecutor("No steps found")
}
preSteps = append(preSteps, func(ctx context.Context) error {
if len(info.matrix()) > 0 {
common.Logger(ctx).Infof("Matrix: %v", info.matrix())
}
return nil
})
preSteps = append(preSteps, func(ctx context.Context) error {
// Have to be skipped for some Tests
if rc.Run == nil {
@@ -234,15 +228,14 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
// The setup section of the job log. The started hook goes first, so what it sets up is
// in place for the first action download and the first step.
// The pre steps come after it, as steps of their own.
preSteps = append(preSteps, rc.runJobStartedHook)
preSteps = append(preSteps, printPrepareActions(rc, preparers))
preSteps = append(preSteps, printCompleteJobName(rc))
preSteps = append(preSteps, stepPreSteps...)
preSteps = append(preSteps, printCompleteJobName(rc))
// Ahead of the teardown below, while the job environment is still up.
postExecutor = postExecutor.Finally(useJobStageLogger(StageComplete, rc.runJobCompletedHook))
postExecutor = postExecutor.Finally(useJobStageLogger(StageComplete, func(ctx context.Context) error {
postExecutor = postExecutor.Finally(rc.runJobCompletedHook)
postExecutor = postExecutor.Finally(func(ctx context.Context) error {
// swallowed: a bad output fails this job, it must not abandon the rest of the plan
if err := info.interpolateOutputs()(ctx); err != nil {
reportStepError(ctx, rc, err)
@@ -250,13 +243,13 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
reportStepError(ctx, rc, err)
}
return nil
}))
})
postExecutor = postExecutor.Finally(useJobStageLogger(StageComplete, func(ctx context.Context) error {
postExecutor = postExecutor.Finally(func(ctx context.Context) error {
jobError := common.JobError(ctx)
var err error
// always allow 1 min for stopping and removing the runner, even if we were cancelled
ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), jobCleanupTimeout)
ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), time.Minute)
defer cancel()
logger := common.Logger(ctx)
@@ -266,32 +259,30 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
logger.Errorf("##[error]%s", EscapeCommandData("Error while stop job container: "+err.Error()))
}
setJobResult(ctx, info, rc, jobError == nil)
rc.removeDockerActionImages(ctx)
return err
}))
})
return common.Executor(func(ctx context.Context) error {
if err := info.startContainer()(ctx); err != nil {
return err
}
return newStepsExecutor(rc, preSteps, steps).Finally(func(ctx context.Context) error {
stepsExecutor := newStepsExecutor(rc, preSteps, steps)
return common.NewPipelineExecutor(info.startContainer(), stepsExecutor.
Finally(func(ctx context.Context) error {
// Record an interrupt (backstop for interrupts that land outside the main
// step loop) so the post steps observe the cancelled/failed job status.
rc.markInterrupted(ctx.Err())
postCtx, cancel := postStepsContext(ctx)
defer cancel()
return postExecutor(postCtx)
})(ctx)
}).Finally(info.closeContainer())
}).
Finally(info.closeContainer()))
}
// postStepsContext derives the context used to run the job's post/cleanup steps from the
// finished main-pipeline context. Cleanup has to run even when the run was interrupted, so the
// returned context always carries a fresh bounded deadline and is never itself cancelled.
//
// - context.Canceled (server cancel): detach with WithoutCancel under a fresh job-error
// container, so a failing post step cannot turn the cancellation into a failure.
// - context.Canceled (server cancel): detach from the cancelled context via a fresh root so
// the post steps can run.
// - context.DeadlineExceeded (job timeout): detach the deadline with WithoutCancel, which
// keeps the original values — including the job-error container — so the timeout failure and
// any post-step error are preserved and the job is still reported as failed.
@@ -299,7 +290,10 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
func postStepsContext(ctx context.Context) (context.Context, context.CancelFunc) {
switch ctx.Err() {
case context.Canceled:
return context.WithTimeout(common.WithJobErrorContainer(context.WithoutCancel(ctx)), 5*time.Minute)
// The cancelled context is abandoned for a fresh root, which drops the job-error
// container installed at the job root. Re-attach a fresh one so a failing post step
// records its error via SetJobError instead of panicking on a nil container.
return context.WithTimeout(common.WithJobErrorContainer(common.WithLogger(context.Background(), common.Logger(ctx))), 5*time.Minute)
case context.DeadlineExceeded:
return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
default:
@@ -323,9 +317,6 @@ func newStepsExecutor(rc *RunContext, preSteps, steps []common.Executor) common.
mainExecutor := newMainStepsExecutor(rc, steps)
return func(ctx context.Context) error {
preErr := preExecutor(ctx)
if errors.As(preErr, new(setupError)) {
return preErr // as on GitHub, no step runs after the job setup failed
}
mainErr := mainExecutor(ctx)
if preErr != nil {
return preErr
@@ -419,56 +410,44 @@ func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success boo
jobResultMessage = "cancelled"
}
logger.With("jobResult", jobResult).Infof("Job %s", jobResultMessage)
logger.WithField("jobResult", jobResult).Infof("Job %s", jobResultMessage)
}
// evaluateJobEnv returns the workflow and job env evaluated, as far as it got on error, leaving rc.Env raw.
func evaluateJobEnv(ctx context.Context, rc *RunContext) (map[string]string, error) {
env := rc.GetEnv()
evaluated := maps.Clone(env)
evaluator := rc.NewExpressionEvaluatorWithEnv(ctx, evaluated)
var workflowEnv map[string]string
if rc.Run.Workflow.Env == nil {
if err := model.DecodeEvaluated("workflow env", rc.Run.Workflow.RawEnv, evaluator.shared(ctx).EvaluateYamlNode, &workflowEnv); err != nil {
return evaluated, err
}
}
if workflowEnv != nil {
evaluated = mergeMaps(workflowEnv, env)
evaluator = rc.NewExpressionEvaluatorWithEnv(ctx, evaluated)
}
var err error
for k, v := range env {
if evaluated[k], err = evaluator.Interpolate(ctx, v); err != nil {
return evaluated, fmt.Errorf("unable to interpolate env %s: %w", k, err)
}
}
return evaluated, nil
}
// evaluateJobEnvAndDefaults resolves the job's env, defaults.run and container env once, as GitHub does at job setup.
// evaluateJobEnvAndDefaults resolves the job's env and defaults.run once, as GitHub does at job setup.
func evaluateJobEnvAndDefaults(ctx context.Context, rc *RunContext) error {
env, err := evaluateJobEnv(ctx, rc)
rc.Env = env
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
if err != nil {
return err
var err error
for k, v := range rc.GetEnv() {
if rc.Env[k], err = rc.ExprEval.Interpolate(ctx, v); err != nil {
return fmt.Errorf("unable to interpolate env %s: %w", k, err)
}
}
var defaults model.Defaults
if err := model.DecodeEvaluated("defaults", rc.Run.Job().RawDefaults, rc.ExprEval.shared(ctx).EvaluateYamlNode, &defaults); err != nil {
return err
defaults := rc.Run.Job().Defaults.Run
if rc.jobRunDefaults.Shell, err = rc.ExprEval.Interpolate(ctx, defaults.Shell); err != nil {
return fmt.Errorf("unable to interpolate defaults.run.shell: %w", err)
}
rc.jobRunDefaults = defaults.Run
return model.DecodeEvaluated("container env", rc.containerEnv, rc.ExprEval.shared(ctx).EvaluateYamlNode, &rc.containerSpec.Env)
if rc.jobRunDefaults.WorkingDirectory, err = rc.ExprEval.Interpolate(ctx, defaults.WorkingDirectory); err != nil {
return fmt.Errorf("unable to interpolate defaults.run.working-directory: %w", err)
}
return nil
}
func setJobOutputs(ctx context.Context, rc *RunContext) error {
if rc.caller == nil {
return nil
}
callerOutputs, err := rc.NewExpressionEvaluator(ctx).shared(ctx).EvaluateWorkflowCallOutputs(rc.Run.Workflow.WorkflowCallConfig())
if err != nil {
return fmt.Errorf("unable to interpolate workflow %w", err)
outputs := rc.Run.Workflow.WorkflowCallConfig().Outputs
callerOutputs := make(map[string]string, len(outputs))
ee := sync.OnceValue(func() *expressionEvaluator { return rc.NewExpressionEvaluator(ctx) })
for k, v := range outputs {
value := v.Value
for range 2 { // two passes, the value resolves through a job output
var err error
if value, err = ee().Interpolate(ctx, value); err != nil {
return fmt.Errorf("unable to interpolate workflow output %s: %w", k, err)
}
}
callerOutputs[k] = value
}
// Matrix combinations of a reusable-workflow caller share the caller's *model.Job;
@@ -486,7 +465,7 @@ func applyJobTimeout(ctx context.Context, rc *RunContext, job *model.Job) (conte
if err != nil {
common.Logger(ctx).Errorf("An error occurred when attempting to determine the job timeout: %s", err)
} else if timeout != "" {
if timeoutMinutes, err := strconv.ParseInt(timeout, 10, 64); err == nil && timeoutMinutes > 0 {
if timeoutMinutes, err := strconv.ParseInt(timeout, 10, 64); err == nil {
return context.WithTimeout(ctx, time.Duration(timeoutMinutes)*time.Minute)
}
}
@@ -542,7 +521,7 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
"/jobs/" + strconv.FormatInt(jobID, 10) + "/steps/"
actPath := rc.JobContainer.GetActPath()
// Reuse a single client across all step uploads so connections can be pooled.
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout, Transport: rc.Config.instanceTransportFor(base)}
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout}
for i := range rc.Run.Job().Steps {
summaryPath := path.Join(actPath, "workflow", "step-summary-"+strconv.Itoa(i)+".md")
body, ok := readSingleFileFromContainerArchive(ctx, rc.JobContainer, summaryPath, maxJobSummaryBytes)
@@ -554,15 +533,6 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
}
}
// For Gitea
// instanceTransportFor returns InstanceTransport only for URLs on the registered address, as steps can rewrite the summary URL through GITHUB_ENV.
func (c Config) instanceTransportFor(rawURL string) http.RoundTripper {
if !strings.HasPrefix(rawURL, c.GitHubInstance+"/") {
return nil
}
return c.InstanceTransport
}
// extractJobIDFromRuntimeToken returns the JobID claim from an ACTIONS_RUNTIME_TOKEN JWT
// without verifying its signature. Returns 0 if the token is unparseable or has no JobID.
func extractJobIDFromRuntimeToken(token string) int64 {
@@ -691,13 +661,6 @@ func archiveEntryMatchesPath(entryName, requestedPath string) bool {
return entryName == requestedPath || entryName == path.Base(requestedPath)
}
// useJobStageLogger tags the log entries of a job level executor with the stage they belong to.
func useJobStageLogger(stage string, executor common.Executor) common.Executor {
return func(ctx context.Context) error {
return executor(common.WithLogger(ctx, common.Logger(ctx).With("stage", stage)))
}
}
func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor {
return func(ctx context.Context) error {
ctx = withStepLogger(ctx, stepModel.Number, stepModel.ID, rc.ExprEval.InterpolateName(ctx, stepModel.String()), stage.String())
+52 -108
View File
@@ -21,11 +21,11 @@ import (
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.com/gitea/runner/act/container"
"gitea.dev/actionslib/pkg/model"
log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@@ -44,7 +44,7 @@ func TestJobExecutor(t *testing.T) {
{workdir, "job-nil-step", "push", "invalid Step 0: missing run or uses key", platforms, secrets},
}
// These tests are sufficient to only check syntax.
ctx := common.WithDryrun(t.Context(), true)
ctx := common.WithDryrun(context.Background(), true)
for _, table := range tables {
t.Run(table.workflowPath, func(t *testing.T) {
t.Parallel()
@@ -136,8 +136,11 @@ func (apm *actionPreparerMock) actionDownloadInfo() (string, string, bool) {
func TestPrintPrepareActionsGolden(t *testing.T) {
buf := &bytes.Buffer{}
logger := logging.New(logging.NewLevelHandler(newJobLogHandler(buf, cyan), logging.LevelInfo))
ctx := common.WithLogger(t.Context(), logger.With("job", "j1"))
logger := log.New()
logger.SetOutput(buf)
logger.SetLevel(log.InfoLevel)
logger.SetFormatter(&jobLogFormatter{color: cyan})
ctx := common.WithLogger(context.Background(), logger.WithFields(log.Fields{"job": "j1"}))
preparers := []actionPreparer{
&actionPreparerMock{reference: "actions/checkout@v7", sha: "9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", ok: true},
@@ -159,8 +162,10 @@ func TestPrintPrepareActionsGolden(t *testing.T) {
func TestPrintPrepareActionsSkipsWithoutActions(t *testing.T) {
buf := &bytes.Buffer{}
logger := logging.New(logging.NewLevelHandler(newJobLogHandler(buf, cyan), logging.LevelInfo))
ctx := common.WithLogger(t.Context(), logger.With("job", "j1"))
logger := log.New()
logger.SetOutput(buf)
logger.SetFormatter(&jobLogFormatter{color: cyan})
ctx := common.WithLogger(context.Background(), logger.WithFields(log.Fields{"job": "j1"}))
require.NoError(t, printPrepareActions(&RunContext{}, nil)(ctx))
@@ -168,8 +173,8 @@ func TestPrintPrepareActionsSkipsWithoutActions(t *testing.T) {
}
func TestPrintPrepareActionsFailsJobOnDownloadError(t *testing.T) {
logger := logging.New(logtest.New())
ctx := common.WithJobErrorContainer(common.WithLogger(t.Context(), logger.With("job", "j1")))
logger, _ := logrustest.NewNullLogger()
ctx := common.WithJobErrorContainer(common.WithLogger(context.Background(), logger.WithField("job", "j1")))
downloadErr := errors.New("failed to fetch \"actions/checkout\"")
rc := &RunContext{}
@@ -187,27 +192,6 @@ func TestPrintPrepareActionsFailsJobOnDownloadError(t *testing.T) {
assert.Zero(t, remaining.prepared)
}
func TestNewStepsExecutorSkipsStepsAfterSetupFailure(t *testing.T) {
for name, tt := range map[string]struct {
preErr error
mainRan bool
}{
"setup failure": {preErr: setupError{assert.AnError}},
"pre step failure": {preErr: assert.AnError, mainRan: true}, // always() steps still run, as on GitHub
} {
t.Run(name, func(t *testing.T) {
ctx := common.WithJobErrorContainer(common.WithLogger(context.Background(), logging.New(logtest.New())))
mainRan := false
err := newStepsExecutor(&RunContext{},
[]common.Executor{func(context.Context) error { return tt.preErr }},
[]common.Executor{func(context.Context) error { mainRan = true; return nil }},
)(ctx)
require.ErrorIs(t, err, assert.AnError)
assert.Equal(t, tt.mainRan, mainRan)
})
}
}
func TestPrintCompleteJobName(t *testing.T) {
for name, tt := range map[string]struct {
rc *RunContext
@@ -219,8 +203,10 @@ func TestPrintCompleteJobName(t *testing.T) {
} {
t.Run(name, func(t *testing.T) {
buf := &bytes.Buffer{}
logger := logging.New(logging.NewLevelHandler(newJobLogHandler(buf, cyan), logging.LevelInfo))
ctx := common.WithLogger(t.Context(), logger.With("job", "j1"))
logger := log.New()
logger.SetOutput(buf)
logger.SetFormatter(&jobLogFormatter{color: cyan})
ctx := common.WithLogger(context.Background(), logger.WithFields(log.Fields{"job": "j1"}))
require.NoError(t, printCompleteJobName(tt.rc)(ctx))
@@ -239,7 +225,7 @@ type actionStepMock struct {
// every action is downloaded before any step runs, and the job name closes the section. A pre
// step that downloaded its own action would leave the log interleaved with the downloads.
func TestNewJobExecutorDownloadsAllActionsBeforeTheFirstStep(t *testing.T) {
ctx := common.WithJobErrorContainer(t.Context())
ctx := common.WithJobErrorContainer(context.Background())
jim := &jobInfoMock{}
sfm := &stepFactoryMock{}
rc := &RunContext{
@@ -270,9 +256,8 @@ func TestNewJobExecutorDownloadsAllActionsBeforeTheFirstStep(t *testing.T) {
apm := &actionPreparerMock{reference: "actions/checkout@v" + stepModel.ID, ok: true}
sfm.On("newStep", stepModel, rc).Return(&actionStepMock{stepMock: sm, actionPreparerMock: apm}, nil)
sm.On("pre").Return(func(ctx context.Context) error {
sm.On("pre").Return(func(context.Context) error {
executorOrder = append(executorOrder, "pre"+stepModel.ID)
common.Logger(ctx).Infof("Run Pre %s", stepModel.ID)
return nil
})
sm.On("main").Return(func(context.Context) error {
@@ -284,31 +269,24 @@ func TestNewJobExecutorDownloadsAllActionsBeforeTheFirstStep(t *testing.T) {
defer sm.AssertExpectations(t)
}
logger, hook := logtest.NewLogger()
err := newJobExecutor(jim, sfm, rc)(common.WithLogger(ctx, logger.With("job", "test")))
logger, hook := logrustest.NewNullLogger()
err := newJobExecutor(jim, sfm, rc)(common.WithLogger(ctx, logger.WithField("job", "test")))
require.NoError(t, err)
assert.Equal(t, []string{"pre1", "pre2", "step1", "step2"}, executorOrder)
// the setup is logged ahead of the pre steps and the cleanup is tagged, so the reporter can show both as steps of their own
setup := make([]string, 0)
for _, entry := range hook.AllRecords() {
for _, entry := range hook.AllEntries() {
if strings.HasPrefix(entry.Message, "Prepare all required actions") || strings.HasPrefix(entry.Message, "Download action") ||
strings.HasPrefix(entry.Message, "Complete job name") || strings.HasPrefix(entry.Message, "Run Pre") {
strings.HasPrefix(entry.Message, "Complete job name") {
setup = append(setup, entry.Message)
}
if _, ok := entry.Attr("jobResult"); ok || strings.HasPrefix(entry.Message, "Cleaning up container") {
stage, _ := entry.Attr("stage")
assert.Equal(t, StageComplete, stage, entry.Message)
}
}
assert.Equal(t, []string{
"Prepare all required actions",
"Download action repository 'actions/checkout@v1'",
"Download action repository 'actions/checkout@v2'",
"Complete job name: test",
"Run Pre 1",
"Run Pre 2",
}, setup)
}
@@ -322,8 +300,6 @@ func TestNewJobExecutor(t *testing.T) {
result string
hasError bool
output string
startError error
cancelOnStart bool
}{
{
name: "zeroSteps",
@@ -459,21 +435,6 @@ func TestNewJobExecutor(t *testing.T) {
result: "failure",
output: "${{ 'test' != test }}",
},
{
name: "start failure",
steps: []*model.Step{{ID: "1"}},
executedSteps: []string{"startContainer", "closeContainer"},
startError: errors.New("start failed"),
},
{
name: "cancelled at startup boundary",
steps: []*model.Step{{ID: "1"}},
preSteps: []bool{false},
postSteps: []bool{true},
executedSteps: []string{"startContainer", "step1", "post1", "interpolateOutputs", "stopContainer", "closeContainer"},
result: "cancelled",
cancelOnStart: true,
},
}
contains := func(needle string, haystack []string) bool {
@@ -484,8 +445,7 @@ func TestNewJobExecutor(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
fmt.Printf("::group::%s\n", tt.name) //nolint:forbidigo // pre-existing issue from nektos/act
ctx, cancel := context.WithCancel(common.WithJobErrorContainer(t.Context()))
defer cancel()
ctx := common.WithJobErrorContainer(context.Background())
jim := &jobInfoMock{}
sfm := &stepFactoryMock{}
rc := &RunContext{
@@ -501,7 +461,8 @@ func TestNewJobExecutor(t *testing.T) {
Config: &Config{},
}
if tt.output != "" {
require.NoError(t, rc.Run.Job().RawOutputs.Encode(map[string]string{"bad": tt.output}))
rc.Run.Job().Outputs = map[string]string{"bad": tt.output}
rc.outputTemplate = map[string]string{"bad": tt.output}
}
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
executorOrder := make([]string, 0)
@@ -509,12 +470,9 @@ func TestNewJobExecutor(t *testing.T) {
jim.On("steps").Return(tt.steps)
if len(tt.steps) > 0 {
jim.On("startContainer").Return(func(_ context.Context) error {
jim.On("startContainer").Return(func(ctx context.Context) error {
executorOrder = append(executorOrder, "startContainer")
if tt.cancelOnStart {
cancel()
}
return tt.startError
return nil
})
}
@@ -548,7 +506,7 @@ func TestNewJobExecutor(t *testing.T) {
defer sm.AssertExpectations(t)
}
if len(tt.steps) > 0 && tt.startError == nil {
if len(tt.steps) > 0 {
jim.On("matrix").Return(map[string]any{})
jim.On("interpolateOutputs").Return(func(ctx context.Context) error {
@@ -562,17 +520,12 @@ func TestNewJobExecutor(t *testing.T) {
if contains("stopContainer", tt.executedSteps) {
jim.On("stopContainer").Return(func(ctx context.Context) error {
executorOrder = append(executorOrder, "stopContainer")
require.NoError(t, ctx.Err())
_, bounded := ctx.Deadline()
require.True(t, bounded)
return nil
})
}
jim.On("result", tt.result)
}
if len(tt.steps) > 0 {
jim.On("closeContainer").Return(func(ctx context.Context) error {
executorOrder = append(executorOrder, "closeContainer")
return nil
@@ -581,14 +534,7 @@ func TestNewJobExecutor(t *testing.T) {
executor := newJobExecutor(jim, sfm, rc)
err := executor(ctx)
switch {
case tt.startError != nil:
require.ErrorIs(t, err, tt.startError)
case tt.cancelOnStart:
require.ErrorIs(t, err, context.Canceled)
default:
require.NoError(t, err)
}
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Empty(t, rc.Run.Job().Outputs["bad"])
assert.Equal(t, tt.executedSteps, executorOrder)
@@ -632,7 +578,7 @@ func (ctx *controllableDeadlineContext) expire() {
// still run against a fresh, non-expired context, and the job must still be
// reported as failed.
func TestNewJobExecutorRunsPostStepsAfterTimeout(t *testing.T) {
ctx := newControllableDeadlineContext(common.WithJobErrorContainer(t.Context()))
ctx := newControllableDeadlineContext(common.WithJobErrorContainer(context.Background()))
jim := &jobInfoMock{}
sfm := &stepFactoryMock{}
@@ -713,7 +659,7 @@ func TestSetJobResultMatrixContinueOnError(t *testing.T) {
return &model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: wf, JobID: "job1"}}}}}
}
ctx := t.Context()
ctx := context.Background()
// fail drives a single matrix combination through the failure path; each
// RunContext is its own jobInfo (rc implements jobInfo) and shares the job.
@@ -808,7 +754,7 @@ func TestTryUploadJobSummaryRetriesTransientFailure(t *testing.T) {
}))
defer server.Close()
ctx := t.Context()
ctx := context.Background()
cm := &containerMock{}
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "step-summary-0.md", body: "# summary"}))),
@@ -852,7 +798,7 @@ func TestTryUploadJobSummaryStopsAtPhaseTimeout(t *testing.T) {
defer server.Close()
defer close(release)
ctx := t.Context()
ctx := context.Background()
cm := &containerMock{}
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "step-summary-0.md", body: "# summary"}))),
@@ -880,7 +826,7 @@ func TestTryUploadJobSummaryStopsAtPhaseTimeout(t *testing.T) {
cm.AssertExpectations(t)
}
func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t *testing.T) {
func TestTryUploadJobSummaryUploadsEachStepIndependently(t *testing.T) {
runtimeToken := fakeRuntimeToken(34)
type upload struct {
@@ -888,7 +834,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t
body string
}
var got []upload
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
got = append(got, upload{r.URL.Path, string(body)})
@@ -896,7 +842,7 @@ func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t
}))
defer server.Close()
ctx := t.Context()
ctx := context.Background()
cm := &containerMock{}
// Three steps: 0 has content, 1 has empty content (skipped), 2 has content.
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
@@ -918,7 +864,6 @@ func TestTryUploadJobSummaryUploadsEachStepIndependentlyWithInstanceTransport(t
"ACTIONS_RUNTIME_TOKEN": runtimeToken,
"GITEA_RUN_ID": "12",
}, cm, 3)
rc.Config.GitHubInstance, rc.Config.InstanceTransport = server.URL, server.Client().Transport
tryUploadJobSummary(ctx, rc)
@@ -944,7 +889,7 @@ func TestTryUploadJobSummaryRequiresExactCapability(t *testing.T) {
"GITEA_RUN_ID": "12",
}, &containerMock{}, 1)
tryUploadJobSummary(t.Context(), rc)
tryUploadJobSummary(context.Background(), rc)
assert.Equal(t, 0, requests)
}
@@ -964,7 +909,7 @@ func TestTryUploadJobSummarySkipsWhenJobIDMissingFromToken(t *testing.T) {
"GITEA_RUN_ID": "12",
}, &containerMock{}, 1)
tryUploadJobSummary(t.Context(), rc)
tryUploadJobSummary(context.Background(), rc)
assert.Equal(t, 0, requests)
}
@@ -977,7 +922,7 @@ func TestExtractJobIDFromRuntimeToken(t *testing.T) {
}
func TestReadSingleFileFromContainerArchiveFindsMatchingRegularFile(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
cm := &containerMock{}
cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(
io.NopCloser(bytes.NewReader(tarArchive(t,
@@ -996,8 +941,8 @@ func TestReadSingleFileFromContainerArchiveFindsMatchingRegularFile(t *testing.T
}
func TestReadSingleFileFromContainerArchiveTruncatesWhenTooLarge(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := logrustest.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cm := &containerMock{}
content := strings.Repeat("a", 300)
cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(
@@ -1014,15 +959,15 @@ func TestReadSingleFileFromContainerArchiveTruncatesWhenTooLarge(t *testing.T) {
assert.LessOrEqual(t, len(body), maxBytes)
keep := maxBytes - len(jobSummaryTruncationMarker)
assert.Equal(t, []byte(content[:keep]+jobSummaryTruncationMarker), body)
if assert.Len(t, hook.AllRecords(), 1) {
assert.Contains(t, hook.AllRecords()[0].Message, "job summary truncated")
if assert.Len(t, hook.Entries, 1) {
assert.Contains(t, hook.Entries[0].Message, "job summary truncated")
}
cm.AssertExpectations(t)
}
func TestReadSingleFileFromContainerArchiveKeepsExactLimitWithoutWarning(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := logrustest.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cm := &containerMock{}
cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "SUMMARY.md", body: "abc"}))),
@@ -1034,7 +979,7 @@ func TestReadSingleFileFromContainerArchiveKeepsExactLimitWithoutWarning(t *test
// A summary that is exactly at the limit is kept whole and not flagged as truncated.
assert.True(t, ok)
assert.Equal(t, []byte("abc"), body)
assert.Empty(t, hook.AllRecords())
assert.Empty(t, hook.Entries)
cm.AssertExpectations(t)
}
@@ -1092,7 +1037,7 @@ func makeTestRC(t *testing.T, jobYAML string) *RunContext {
Name: "workflow1",
Jobs: map[string]*model.Job{"job1": job},
}, nil)
rc.ExprEval = rc.NewExpressionEvaluator(t.Context())
rc.ExprEval = rc.NewExpressionEvaluator(context.Background())
return rc
}
@@ -1104,13 +1049,12 @@ func TestApplyJobTimeout(t *testing.T) {
}{
{"empty", "runs-on: ubuntu-latest", false},
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", true},
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", false},
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rc := makeTestRC(t, tc.yaml)
ctx := t.Context()
ctx := context.Background()
newCtx, cancel := applyJobTimeout(ctx, rc, rc.Run.Job())
defer cancel()
_, hasDeadline := newCtx.Deadline()
@@ -1134,7 +1078,7 @@ func TestEvaluateJobContinueOnError(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rc := makeTestRC(t, tc.yaml)
got := evaluateJobContinueOnError(t.Context(), rc, rc.Run.Job())
got := evaluateJobContinueOnError(context.Background(), rc, rc.Run.Job())
assert.Equal(t, tc.want, got)
})
}
@@ -1199,7 +1143,7 @@ func TestTryUploadJobSummaryMasksSecrets(t *testing.T) {
rc.Config.ExtraMasks = []string{"pr0xypw"}
rc.Masks = []string{"runtime-added"}
tryUploadJobSummary(t.Context(), rc)
tryUploadJobSummary(context.Background(), rc)
assert.Equal(t, "deployed true with *** and *** via ***", got)
cm.AssertExpectations(t)
+7 -2
View File
@@ -7,6 +7,7 @@ import (
"cmp"
"context"
"fmt"
"maps"
"path"
"strings"
@@ -42,7 +43,7 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
}
cmd, shell := hookCommand(hookPath)
rawLogger := common.Logger(ctx).With(rawOutputField, true)
rawLogger := common.Logger(ctx).WithField(rawOutputField, true)
defer rawLogger.Infof("::endgroup::")
rawLogger.Infof("::group::Run '%s'", EscapeCommandData(hookPath))
rawLogger.Infof("A %s hook has been configured by the runner administrator", name)
@@ -50,7 +51,11 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
rawLogger.Infof("shell: %s", shell)
}
env := mergeMaps(rc.containerSpec.Env, rc.GetEnv())
env := map[string]string{}
if jobContainer := rc.Run.Job().Container(); jobContainer != nil {
maps.Copy(env, jobContainer.Env)
}
maps.Copy(env, rc.GetEnv())
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
rc.ApplyExtraPath(ctx, &env)
+3 -4
View File
@@ -12,10 +12,9 @@ import (
"testing"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.dev/actionslib/pkg/model"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -65,8 +64,8 @@ func newHookRunContext(jobContainer *hookContainer, config *Config) (*RunContext
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
JobContainer: jobContainer,
}
logger := logging.New(logtest.New())
ctx := common.WithJobErrorContainer(common.WithLogger(context.Background(), logger.With("test", true)))
logger, _ := test.NewNullLogger()
ctx := common.WithJobErrorContainer(common.WithLogger(context.Background(), logger.WithField("test", true)))
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
return rc, ctx
}
+99 -136
View File
@@ -19,8 +19,8 @@ import (
"sync"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
"golang.org/x/term"
)
@@ -35,16 +35,6 @@ const (
gray = 37
)
const (
// The "stage" of the log entries of a step, and of the job level entries that finalize the job after its steps.
StagePre = "Pre"
StageMain = "Main"
StagePost = "Post"
StageComplete = "Complete"
// StepStartedField marks the log entry of a step that is about to run, unlike one that is skipped.
StepStartedField = "stepStarted"
)
const (
rawOutputField = "raw_output"
scriptLineCyanField = "script_line_cyan"
@@ -83,9 +73,8 @@ func WithMasks(ctx context.Context, masks *[]string) context.Context {
return context.WithValue(ctx, masksContextKeyVal, masks)
}
// JobLoggerFactory supplies the handler a job's own log output goes to, in place of stdout.
type JobLoggerFactory interface {
WithJobLogger() logging.Handler
WithJobLogger() *logrus.Logger
}
type jobLoggerFactoryContextKey string
@@ -100,70 +89,84 @@ func WithJobLoggerFactory(ctx context.Context, factory JobLoggerFactory) context
func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, masks *[]string, matrix map[string]any) context.Context {
ctx = WithMasks(ctx, masks)
var output logging.Handler
var logger *logrus.Logger
if jobLoggerFactory, ok := ctx.Value(jobLoggerFactoryContextKeyVal).(JobLoggerFactory); ok && jobLoggerFactory != nil {
output = jobLoggerFactory.WithJobLogger()
} else if config.JSONLogger {
output = newJobLogJSONHandler(os.Stdout)
logger = jobLoggerFactory.WithJobLogger()
} else {
mux.Lock()
nextColor++
color := colors[nextColor%len(colors)]
mux.Unlock()
output = newJobLogHandler(os.Stdout, color)
var formatter logrus.Formatter
if config.JSONLogger {
formatter = &logrus.JSONFormatter{}
} else {
mux.Lock()
defer mux.Unlock()
nextColor++
formatter = &jobLogFormatter{color: colors[nextColor%len(colors)]}
}
logger = logrus.New()
logger.SetOutput(os.Stdout)
logger.SetLevel(logrus.GetLevel())
logger.SetFormatter(formatter)
}
var handlers []logging.Handler
if h := common.LoggerHandler(ctx); h != nil { // Adapt to Gitea
handlers = append(handlers, h)
{ // Adapt to Gitea
if hook := common.LoggerHook(ctx); hook != nil {
logger.AddHook(hook)
}
if config.JobLoggerLevel != nil {
logger.SetLevel(*config.JobLoggerLevel)
} else {
logger.SetLevel(logrus.TraceLevel)
}
}
handlers = append(handlers, &maskedHandler{
Handler: output,
masker: valueMasker(config.InsecureSecrets, config.maskers()),
logger.SetFormatter(&maskedFormatter{
Formatter: logger.Formatter,
masker: valueMasker(config.InsecureSecrets, config.maskers()),
})
level := logging.LevelTrace
if config.JobLoggerLevel != nil {
level = *config.JobLoggerLevel
}
rtn := logging.New(logging.NewLevelHandler(logging.NewMultiHandler(handlers...), level)).With(
"job", jobName,
"jobID", jobID,
"dryrun", common.Dryrun(ctx),
"matrix", matrix,
).WithContext(ctx)
rtn := logger.WithFields(logrus.Fields{
"job": jobName,
"jobID": jobID,
"dryrun": common.Dryrun(ctx),
"matrix": matrix,
}).WithContext(ctx)
return common.WithLogger(ctx, rtn)
}
func WithCompositeLogger(ctx context.Context, masks *[]string) context.Context {
ctx = WithMasks(ctx, masks)
return common.WithLogger(ctx, common.Logger(ctx).WithContext(ctx))
return common.WithLogger(ctx, common.Logger(ctx).WithFields(logrus.Fields{}).WithContext(ctx))
}
func WithCompositeStepLogger(ctx context.Context, stepID string) context.Context {
stepIDs := append(slices.Clone(common.StepIDs(ctx)), stepID)
ctx = common.WithStepIDs(ctx, stepIDs)
val := common.Logger(ctx)
stepIDs := make([]string, 0)
return common.WithLogger(ctx, common.Logger(ctx).With("stepID", stepIDs).WithContext(ctx))
if logger, ok := val.(*logrus.Entry); ok {
if oldStepIDs, ok := logger.Data["stepID"].([]string); ok {
stepIDs = append(stepIDs, oldStepIDs...)
}
}
stepIDs = append(stepIDs, stepID)
return common.WithLogger(ctx, common.Logger(ctx).WithFields(logrus.Fields{
"stepID": stepIDs,
}).WithContext(ctx))
}
func withStepLogger(ctx context.Context, stepNumber int, stepID, stepName, stageName string) context.Context {
stepIDs := []string{stepID}
ctx = common.WithStepIDs(ctx, stepIDs)
rtn := common.Logger(ctx).With(
"stepNumber", stepNumber,
"step", stepName,
"stepID", stepIDs,
"stage", stageName,
)
rtn := common.Logger(ctx).WithFields(logrus.Fields{
"stepNumber": stepNumber,
"step": stepName,
"stepID": []string{stepID},
"stage": stageName,
})
return common.WithLogger(ctx, rtn)
}
// entryProcessor rewrites a record before the output handler sees it.
type entryProcessor func(ctx context.Context, r logging.Record) logging.Record
type entryProcessor func(entry *logrus.Entry) *logrus.Entry
var secretValueEncoders = []func(string) string{
func(v string) string { return base64.StdEncoding.EncodeToString([]byte(v)) },
@@ -318,16 +321,16 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
replacer *strings.Replacer
)
return func(ctx context.Context, r logging.Record) logging.Record {
return func(entry *logrus.Entry) *logrus.Entry {
if insecureSecrets {
return r
return entry
}
masks := Masks(ctx)
masks := Masks(entry.Context)
if len(*masks) == 0 {
r.Message = defReplacer.Replace(r.Message)
return r
entry.Message = defReplacer.Replace(entry.Message)
return entry
}
mu.Lock()
@@ -346,129 +349,89 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
cmasker := replacer
mu.Unlock()
r.Message = cmasker.Replace(r.Message)
entry.Message = cmasker.Replace(entry.Message)
return r
return entry
}
}
type maskedHandler struct {
logging.Handler
type maskedFormatter struct {
logrus.Formatter
masker entryProcessor
}
func (m *maskedHandler) Handle(ctx context.Context, r logging.Record) error {
return m.Handler.Handle(ctx, m.masker(ctx, r))
func (f *maskedFormatter) Format(entry *logrus.Entry) ([]byte, error) {
return f.Formatter.Format(f.masker(entry))
}
func (m *maskedHandler) WithAttrs(attrs []logging.Attr) logging.Handler {
return &maskedHandler{Handler: m.Handler.WithAttrs(attrs), masker: m.masker}
}
// jobLogHandler writes a job's log to w, one record per line in the shape format gives it.
type jobLogHandler struct {
w io.Writer
mu *sync.Mutex // shared by every handler derived with WithAttrs, as they share w
attrs []logging.Attr
format func(logging.Record) ([]byte, error)
}
// newJobLogHandler writes readable text.
func newJobLogHandler(w io.Writer, color int) *jobLogHandler {
return &jobLogHandler{w: w, mu: &sync.Mutex{}, format: textFormatter{w: w, color: color}.format}
}
func (h *jobLogHandler) Enabled(context.Context, logging.Level) bool { return true }
func (h *jobLogHandler) WithAttrs(attrs []logging.Attr) logging.Handler {
c := *h
c.attrs = slices.Concat(h.attrs, attrs)
return &c
}
func (h *jobLogHandler) Handle(_ context.Context, r logging.Record) error {
r.Attrs = slices.Concat(h.attrs, r.Attrs)
line, err := h.format(r)
if err != nil {
return err
}
h.mu.Lock()
defer h.mu.Unlock()
_, err = h.w.Write(line)
return err
}
type textFormatter struct {
w io.Writer
type jobLogFormatter struct {
color int
}
func (f textFormatter) format(r logging.Record) ([]byte, error) {
func (f *jobLogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
b := &bytes.Buffer{}
// the web renderer decodes command data, so this local view has to as well
if _, _, _, ok := tryParseRawActionCommand(r.Message + "\n"); ok {
r.Message = UnescapeCommandData(r.Message)
if _, _, _, ok := tryParseRawActionCommand(entry.Message + "\n"); ok {
entry.Message = UnescapeCommandData(entry.Message)
}
r.Message = strings.TrimSuffix(r.Message, "\n")
if f.isColored() {
f.printColored(b, r)
if f.isColored(entry) {
f.printColored(b, entry)
} else {
f.print(b, r)
f.print(b, entry)
}
b.WriteByte('\n')
return b.Bytes(), nil
}
func attrIsTrue(r logging.Record, key string) bool {
v, _ := r.Attr(key)
return v == true
}
func (f *jobLogFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry) {
entry.Message = strings.TrimSuffix(entry.Message, "\n")
func (f textFormatter) printColored(b *bytes.Buffer, r logging.Record) {
job, _ := r.Attr("job")
job := entry.Data["job"]
debugFlag := ""
if r.Level == logging.LevelDebug {
if entry.Level == logrus.DebugLevel {
debugFlag = "[DEBUG] "
}
switch {
case attrIsTrue(r, rawOutputField):
if attrIsTrue(r, scriptLineCyanField) {
fmt.Fprintf(b, "\x1b[%dm|\x1b[0m \x1b[36;1m%s\x1b[0m", f.color, r.Message)
case entry.Data[rawOutputField] == true:
if entry.Data[scriptLineCyanField] == true {
fmt.Fprintf(b, "\x1b[%dm|\x1b[0m \x1b[36;1m%s\x1b[0m", f.color, entry.Message)
} else {
fmt.Fprintf(b, "\x1b[%dm|\x1b[0m %s", f.color, r.Message)
fmt.Fprintf(b, "\x1b[%dm|\x1b[0m %s", f.color, entry.Message)
}
case attrIsTrue(r, "dryrun"):
fmt.Fprintf(b, "\x1b[1m\x1b[%dm\x1b[7m*DRYRUN*\x1b[0m \x1b[%dm[%s] \x1b[0m%s%s", gray, f.color, job, debugFlag, r.Message)
case entry.Data["dryrun"] == true:
fmt.Fprintf(b, "\x1b[1m\x1b[%dm\x1b[7m*DRYRUN*\x1b[0m \x1b[%dm[%s] \x1b[0m%s%s", gray, f.color, job, debugFlag, entry.Message)
default:
fmt.Fprintf(b, "\x1b[%dm[%s] \x1b[0m%s%s", f.color, job, debugFlag, r.Message)
fmt.Fprintf(b, "\x1b[%dm[%s] \x1b[0m%s%s", f.color, job, debugFlag, entry.Message)
}
}
func (f textFormatter) print(b *bytes.Buffer, r logging.Record) {
job, _ := r.Attr("job")
func (f *jobLogFormatter) print(b *bytes.Buffer, entry *logrus.Entry) {
entry.Message = strings.TrimSuffix(entry.Message, "\n")
job := entry.Data["job"]
debugFlag := ""
if r.Level == logging.LevelDebug {
if entry.Level == logrus.DebugLevel {
debugFlag = "[DEBUG] "
}
switch {
case attrIsTrue(r, rawOutputField):
fmt.Fprintf(b, "[%s] | %s", job, r.Message)
case attrIsTrue(r, "dryrun"):
fmt.Fprintf(b, "*DRYRUN* [%s] %s%s", job, debugFlag, r.Message)
case entry.Data[rawOutputField] == true:
fmt.Fprintf(b, "[%s] | %s", job, entry.Message)
case entry.Data["dryrun"] == true:
fmt.Fprintf(b, "*DRYRUN* [%s] %s%s", job, debugFlag, entry.Message)
default:
fmt.Fprintf(b, "[%s] %s%s", job, debugFlag, r.Message)
fmt.Fprintf(b, "[%s] %s%s", job, debugFlag, entry.Message)
}
}
func (f textFormatter) isColored() bool {
isColored := checkIfTerminal(f.w)
func (f *jobLogFormatter) isColored(entry *logrus.Entry) bool {
isColored := checkIfTerminal(entry.Logger.Out)
if force, ok := os.LookupEnv("CLICOLOR_FORCE"); ok && force != "0" {
isColored = true
-52
View File
@@ -1,52 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package runner
import (
"encoding/json"
"fmt"
"io"
"sync"
"time"
"gitea.com/gitea/runner/act/common/logging"
)
// newJobLogJSONHandler writes one JSON object per record, in the shape of logrus's JSONFormatter.
func newJobLogJSONHandler(w io.Writer) *jobLogHandler {
return &jobLogHandler{w: w, mu: &sync.Mutex{}, format: formatJSON}
}
func formatJSON(r logging.Record) ([]byte, error) {
data := make(map[string]any, len(r.Attrs)+3)
for _, a := range r.Attrs {
switch v := a.Value.(type) {
case error:
data[a.Key] = v.Error() // an error marshals to {}
default:
data[a.Key] = v
}
}
for _, key := range []string{"time", "msg", "level"} {
if v, ok := data[key]; ok { // keep the fixed keys from being overwritten
data["fields."+key] = v
}
}
data["time"] = r.Time.Format(time.RFC3339)
data["msg"] = r.Message
data["level"] = jsonLevel(r.Level)
line, err := json.Marshal(data)
if err != nil {
return nil, fmt.Errorf("failed to marshal fields to JSON, %w", err)
}
return append(line, '\n'), nil
}
func jsonLevel(l logging.Level) string {
if l.String() == "warn" {
return "warning" // what logrus calls it
}
return l.String()
}
+15 -64
View File
@@ -4,19 +4,13 @@
package runner
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"io"
"net/url"
"strings"
"testing"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -55,7 +49,10 @@ func TestValueMasker(t *testing.T) {
ctx := WithMasks(t.Context(), &entry.masks)
masker := valueMasker(false, AppendSecretMaskers(nil, entry.secrets))
for line := range strings.SplitSeq(entry.lines, "\n") {
lentry := masker(ctx, logging.Record{Message: line})
lentry := masker(&logrus.Entry{
Context: ctx,
Message: line,
})
for _, line := range entry.disallowed {
assert.NotContains(t, lentry.Message, line)
}
@@ -82,7 +79,7 @@ func TestValueMaskerEncodedSecrets(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
masker := valueMasker(false, AppendSecretMaskers(nil, map[string]string{"TOKEN": tc.secret}))
entry := masker(t.Context(), logging.Record{Message: strings.Join(tc.encoded, " ")})
entry := masker(&logrus.Entry{Context: t.Context(), Message: strings.Join(tc.encoded, " ")})
assert.Contains(t, entry.Message, "***")
for _, disallowed := range tc.encoded {
@@ -108,7 +105,7 @@ func TestValueMaskerJSONEscapesBothWays(t *testing.T) {
{"literal (JS JSON.stringify / .NET)", jsonStringEscapeNoHTML(secret)},
} {
t.Run(tc.name, func(t *testing.T) {
entry := masker(t.Context(), logging.Record{Message: `{"t":"` + tc.form + `"}`})
entry := masker(&logrus.Entry{Context: t.Context(), Message: `{"t":"` + tc.form + `"}`})
assert.Contains(t, entry.Message, "***")
assert.NotContains(t, entry.Message, tc.form)
@@ -121,7 +118,7 @@ func TestValueMaskerJSONEscapesBothWays(t *testing.T) {
func TestValueMaskerHidesExtraMasks(t *testing.T) {
masker := valueMasker(false, (&Config{ExtraMasks: []string{"pr0xypw"}}).maskers())
entry := masker(t.Context(), logging.Record{Message: "proxy is http://user:pr0xypw@proxy:3128"})
entry := masker(&logrus.Entry{Context: t.Context(), Message: "proxy is http://user:pr0xypw@proxy:3128"})
assert.Equal(t, "proxy is http://user:***@proxy:3128", entry.Message)
}
@@ -140,7 +137,7 @@ func TestValueMaskerEncodedMasks(t *testing.T) {
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("s3cr3t")), "encoded: ***"},
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("first")), "encoded: ***"},
} {
entry := masker(WithMasks(t.Context(), &masks), logging.Record{Message: tc.line})
entry := masker(&logrus.Entry{Context: WithMasks(t.Context(), &masks), Message: tc.line})
assert.Equal(t, tc.want, entry.Message, tc.line)
}
}
@@ -157,7 +154,7 @@ func TestValueMaskerBase64Alignments(t *testing.T) {
for _, prefix := range []string{"x-access-token:", "user:", "ab:"} {
t.Run(prefix, func(t *testing.T) {
encoded := base64.StdEncoding.EncodeToString([]byte(prefix + secret))
entry := masker(t.Context(), logging.Record{Message: "Authorization: Basic " + encoded})
entry := masker(&logrus.Entry{Context: t.Context(), Message: "Authorization: Basic " + encoded})
assert.Contains(t, entry.Message, "***")
assert.NotEqual(t, "Authorization: Basic "+encoded, entry.Message)
@@ -175,7 +172,7 @@ func TestValueMaskerBase64Alignments(t *testing.T) {
func TestValueMaskerCachedReplacerSeesNewMasks(t *testing.T) {
masker := valueMasker(false, AppendSecretMaskers(nil, map[string]string{"TOKEN": "secret-token"}))
mask := func(masks *[]string, message string) string {
return masker(WithMasks(t.Context(), masks), logging.Record{Message: message}).Message
return masker(&logrus.Entry{Context: WithMasks(t.Context(), masks), Message: message}).Message
}
job := []string{"first mask"}
@@ -209,61 +206,15 @@ func TestAppendSecretMaskerSkipsUselessEncodings(t *testing.T) {
}
func TestJobLogFormatterDecodesCommandData(t *testing.T) {
logger := logrus.New()
logger.Out = io.Discard
format := func(message string) string {
line, err := textFormatter{w: io.Discard}.format(logging.Record{Message: message, Attrs: []logging.Attr{{Key: rawOutputField, Value: true}}})
out, err := (&jobLogFormatter{}).Format(&logrus.Entry{Logger: logger, Message: message, Data: logrus.Fields{rawOutputField: true}})
require.NoError(t, err)
return string(line)
return string(out)
}
assert.Contains(t, format("##[error]deploy 50%25 traffic"), "##[error]deploy 50% traffic")
// a plain line is not command data and keeps its literal escapes
assert.Contains(t, format("progress 50%25 done"), "progress 50%25 done")
}
type factoryFunc func() logging.Handler
func (f factoryFunc) WithJobLogger() logging.Handler { return f() }
func TestWithJobLoggerFansOutAndMasksOnlyOutput(t *testing.T) {
output, hook := logtest.NewWithLevel(logging.LevelTrace), logtest.NewWithLevel(logging.LevelTrace)
ctx := WithJobLoggerFactory(t.Context(), factoryFunc(func() logging.Handler { return output }))
ctx = common.WithLoggerHandler(ctx, hook)
level := logging.LevelInfo
cfg := &Config{Secrets: map[string]string{"TOKEN": "s3cr3t-value"}, JobLoggerLevel: &level}
ctx = WithJobLogger(ctx, "j1", "build", cfg, &[]string{}, nil)
ctx = withStepLogger(ctx, 2, "s2", "step two", "Main")
ctx = WithCompositeStepLogger(ctx, "inner")
common.Logger(ctx).Infof("token is %s", "s3cr3t-value")
common.Logger(ctx).Debugf("below the job logger level")
// the reporter masks for itself, so its record keeps the original
require.Len(t, hook.AllRecords(), 1)
assert.Equal(t, "token is s3cr3t-value", hook.LastRecord().Message)
require.Len(t, output.AllRecords(), 1)
assert.Equal(t, "token is ***", output.LastRecord().Message)
for _, want := range []struct {
key string
val any
}{{"job", "build"}, {"jobID", "j1"}, {"stepNumber", 2}, {"stage", "Main"}, {"stepID", []string{"s2", "inner"}}} {
got, ok := hook.LastRecord().Attr(want.key)
assert.True(t, ok, want.key)
assert.Equal(t, want.val, got, want.key)
}
}
func TestJobLogJSONHandler(t *testing.T) {
buf := &bytes.Buffer{}
logger := logging.New(newJobLogJSONHandler(buf)).With("job", "build", "msg", "clash").WithError(errors.New("boom"))
logger.Warnf("careful %d", 1)
var got map[string]any
require.NoError(t, json.Unmarshal(buf.Bytes(), &got))
assert.Equal(t, "careful 1", got["msg"])
assert.Equal(t, "warning", got["level"])
assert.Equal(t, "build", got["job"])
assert.Equal(t, "boom", got["error"])
assert.Equal(t, "clash", got["fields.msg"])
assert.NotEmpty(t, got["time"])
}
+3 -40
View File
@@ -4,7 +4,6 @@
package runner
import (
"strings"
"testing"
"gitea.dev/actionslib/pkg/model"
@@ -35,14 +34,9 @@ func TestMaxParallelStrategy(t *testing.T) {
expectedMaxParallel: 4,
},
{
name: "max-parallel-10-clamped-to-combinations",
name: "max-parallel-10",
maxParallelString: "10",
expectedMaxParallel: 5,
},
{
name: "max-parallel-invalid-falls-back-to-default",
maxParallelString: "tow",
expectedMaxParallel: 4,
expectedMaxParallel: 10,
},
}
@@ -67,29 +61,9 @@ func TestMaxParallelStrategy(t *testing.T) {
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.NotNil(t, matrixes)
assert.Len(t, matrixes, 5)
assert.Equal(t, tt.expectedMaxParallel, maxParallelFor(job.Strategy, len(matrixes)))
assert.Equal(t, tt.expectedMaxParallel, job.Strategy.MaxParallel)
})
}
t.Run("deferred strategy", func(t *testing.T) {
workflow, err := model.ReadWorkflow(strings.NewReader(`
jobs:
test:
if: false
strategy: ${{ fromJSON('{"max-parallel":2,"fail-fast":false,"matrix":{"os":["ubuntu","windows"]}}') }}
`))
require.NoError(t, err)
job := workflow.Jobs["test"]
rawStrategy := model.CloneYamlNode(job.RawStrategy)
require.NoError(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "test"}}}}})(t.Context()))
require.NotNil(t, job.Strategy)
assert.False(t, job.Strategy.GetFailFast())
assert.Equal(t, 2, maxParallelFor(job.Strategy, 5))
matrixes, err := job.GetMatrixes()
require.NoError(t, err)
assert.Equal(t, []map[string]any{{"os": "ubuntu"}, {"os": "windows"}}, matrixes)
assert.Equal(t, rawStrategy, job.RawStrategy)
})
}
func TestNewPlanExecutorInvalidMatrix(t *testing.T) {
@@ -107,15 +81,4 @@ func TestNewPlanExecutorInvalidMatrix(t *testing.T) {
runner := &runnerImpl{config: &Config{}}
require.ErrorContains(t, runner.NewPlanExecutor(plan)(t.Context()), "could not get job matrix:")
for _, strategy := range []string{
`${{ fromJSON('invalid') }}`,
`${{ '${{ inputs.unresolved }}' }}`,
} {
t.Run(strategy, func(t *testing.T) {
workflow, err := model.ReadWorkflow(strings.NewReader("jobs:\n test:\n strategy: " + strategy))
require.NoError(t, err)
require.Error(t, runner.NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "test"}}}}})(t.Context()))
})
}
}
+16 -24
View File
@@ -28,8 +28,8 @@ import (
// of upload-artifact.
//
// getCacheServiceURL() then resolves the cache service from ACTIONS_RESULTS_URL alone, where v1
// reads ACTIONS_CACHE_URL first. Both reads there are given the same preference, so a patched cache
// client reaches the cache server by its own address rather than through the results origin.
// reads ACTIONS_CACHE_URL first. Both reads there are given the same preference, which is what
// keeps the runner out of the artifact path: the results URL still points at Gitea.
//
// Either of these landing upstream makes this file deletable:
//
@@ -105,45 +105,37 @@ func actionScriptPaths(dir string, action *model.Action) []string {
return paths
}
// patchActions returns a restore: the tree is shared, so a job with patching off gets no edits.
func patchActions(ctx context.Context, scripts []string) func() {
restore := map[string][]byte{}
// patchActions edits the toolkit in an action's bundles. The caller holds the action directory's
// clone lock, which is what keeps another job's checkout from resetting them before the copy.
func patchActions(ctx context.Context, scripts []string) {
for _, script := range scripts {
original, err := patchBundle(script)
if original != nil {
restore[script] = original // also when the write failed part way through
}
if err != nil {
switch patched, err := patchBundle(script); {
case err != nil:
common.Logger(ctx).Warnf("actions toolkit: %s left unpatched: %v", script, err)
}
}
return func() {
for script, original := range restore {
if err := os.WriteFile(script, original, 0o644); err != nil { //nolint:gosec // as the checkout wrote it
common.Logger(ctx).Warnf("actions toolkit: %s left patched in the shared copy: %v", script, err)
}
case patched:
common.Logger(ctx).Debugf("actions toolkit: patched %s", script)
}
}
}
// patchBundle returns the bytes it replaced, or nil when it left the bundle alone.
func patchBundle(script string) ([]byte, error) {
func patchBundle(script string) (bool, error) {
info, err := os.Stat(script)
if err != nil {
return nil, err
return false, err
}
if info.Size() > maxBundleSize {
return nil, nil
return false, nil
}
data, err := os.ReadFile(script)
if err != nil {
return nil, err
return false, err
}
patched, ok := patchedBundle(data)
if !ok {
return nil, nil
return false, nil
}
return data, os.WriteFile(script, patched, info.Mode().Perm())
// No atomic write needed: every prepare checks the action out and hard resets it.
return true, os.WriteFile(script, patched, info.Mode().Perm())
}
// patchedBundle opens the GHES gate, and where the cache toolkit is present, points the cache
Loaded 100 of 215 files, more files were not shown because too many files have changed in this diff. Show more