mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 06:45:44 +00:00
Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5f255a3e92 | ||
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 | ||
|
|
f2fbd67962 | ||
|
|
d6f5b68a0d | ||
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e |
No files matched your search
@@ -21,7 +21,7 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
node-version: 26
|
||||
- run: make lint-pr-title
|
||||
env:
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
@@ -34,6 +34,13 @@ jobs:
|
||||
with:
|
||||
go-version-file: "go.mod"
|
||||
check-latest: true
|
||||
- name: Import GPG key
|
||||
id: import_gpg
|
||||
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||
with:
|
||||
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||
passphrase: ${{ secrets.PASSPHRASE }}
|
||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||
- name: goreleaser
|
||||
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||
with:
|
||||
@@ -47,6 +54,7 @@ jobs:
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
GORELEASER_FORCE_TOKEN: "gitea"
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||
|
||||
release-image:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||
- name: pre-pull test images
|
||||
env:
|
||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 # renovate: datasource=docker
|
||||
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||
run: |
|
||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||
@@ -57,6 +57,32 @@ jobs:
|
||||
make coverage-report
|
||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
windows:
|
||||
name: test on windows
|
||||
runs-on: ubuntu-latest-kvm
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- name: prepare anonymous docker config
|
||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||
- name: windows vm cache key
|
||||
id: windows-vm
|
||||
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
- name: test on windows
|
||||
id: test-windows
|
||||
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||
env:
|
||||
WINDOWS_TEST_TIMEOUT: 45
|
||||
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
|
||||
e2e:
|
||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/gitea-runner
|
||||
/gitea-runner.exe
|
||||
.env
|
||||
!/act/runner/testdata/secrets/.env
|
||||
.runner
|
||||
|
||||
@@ -39,6 +39,8 @@ linters:
|
||||
deny:
|
||||
- pkg: golang.org/x/exp
|
||||
desc: it's experimental and unreliable
|
||||
- pkg: unsafe
|
||||
desc: use typed APIs instead
|
||||
logging:
|
||||
files:
|
||||
- $all
|
||||
|
||||
+5
-2
@@ -34,8 +34,6 @@ builds:
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: s390x
|
||||
- goos: freebsd
|
||||
@@ -82,6 +80,7 @@ builds:
|
||||
publishers:
|
||||
- name: cloudflare-r2
|
||||
checksum: true
|
||||
signature: true
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
- glob: ./**.sha256
|
||||
@@ -102,6 +101,10 @@ checksum:
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
|
||||
signs:
|
||||
- artifacts: checksum
|
||||
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||
|
||||
snapshot:
|
||||
version_template: "{{ .Branch }}-devel"
|
||||
|
||||
|
||||
+18
-5
@@ -57,12 +57,25 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
||||
|
||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||
parallel.
|
||||
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||
|
||||
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped, tolerated and timed-out jobs.
|
||||
`label_routing` and `capacity` cover scheduling, `reconnect` drops the connection to Gitea through a proxy, and
|
||||
`host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||
Docker and Kubernetes.
|
||||
|
||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
||||
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||
|
||||
## Windows tests
|
||||
|
||||
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
|
||||
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
|
||||
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||
+2
-2
@@ -19,7 +19,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
### DIND VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind AS dind
|
||||
FROM docker:29.9.2-dind AS dind
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -41,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
### DIND-ROOTLESS VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
||||
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
DIST := dist
|
||||
EXECUTABLE := gitea-runner
|
||||
GO ?= go
|
||||
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:8856b200466e9a5af9cc0d4de4ca96f73547833bcdddf9a567189ff772c4d372 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||
|
||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||
|
||||
@@ -150,11 +151,15 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||
|
||||
.PHONY: test-windows
|
||||
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||
|
||||
E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||
|
||||
.PHONY: install
|
||||
install: $(GOFILES) ## install the runner binary via `go install`
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
# Gitea Runner
|
||||
|
||||
Gitea Runner runs the jobs of Gitea Actions workflows. Register it with a Gitea instance to run jobs in Docker containers, Kubernetes pods or directly on the host.
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
On the runner:
|
||||
- Linux, macOS or Windows, or FreeBSD.
|
||||
- `git` 2.34.1 or newer.
|
||||
- `docker`, or `podman` with `podman.socket` enabled, for `docker` labels and Docker actions. See Docker's [install instructions](https://docs.docker.com/engine/install/).
|
||||
|
||||
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
|
||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
In the job image, or on the host for host jobs:
|
||||
|
||||
Where steps run, in the job image or on the runner in host mode:
|
||||
|
||||
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
|
||||
- `bash` or `sh`, or `pwsh` or `powershell` on Windows.
|
||||
- `node` for JavaScript actions and `hashFiles()`.
|
||||
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
|
||||
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
|
||||
- `git-lfs` for `builtin:checkout` with `lfs: true`.
|
||||
- `ssh` for `builtin:checkout` with `ssh-key`.
|
||||
|
||||
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
|
||||
|
||||
### Download pre-built binary
|
||||
|
||||
Visit [here](https://dl.gitea.com/gitea-runner/) and download the right version for your platform.
|
||||
Download the binary for your platform from [dl.gitea.com](https://dl.gitea.com/gitea-runner/).
|
||||
|
||||
### Build from source
|
||||
|
||||
@@ -36,7 +38,7 @@ make docker
|
||||
|
||||
## Quickstart
|
||||
|
||||
Actions are disabled by default, so you need to add the following to the configuration file of your Gitea instance to enable it:
|
||||
Enable Actions in the configuration file of your Gitea instance:
|
||||
|
||||
```ini
|
||||
[actions]
|
||||
@@ -49,70 +51,72 @@ ENABLED=true
|
||||
./gitea-runner register
|
||||
```
|
||||
|
||||
And you will be asked to input:
|
||||
It asks for:
|
||||
|
||||
1. Gitea instance URL, like `http://192.168.8.8:3000/`. You should use your gitea instance ROOT_URL as the instance argument
|
||||
and you should not use `localhost` or `127.0.0.1` as instance IP;
|
||||
2. Runner token, you can get it from `http://192.168.8.8:3000/admin/actions/runners`;
|
||||
3. Runner name, you can just leave it blank;
|
||||
4. Runner labels, you can just leave it blank.
|
||||
1. The Gitea instance URL. Use its `ROOT_URL`, not `localhost` or `127.0.0.1`.
|
||||
2. A runner token, from `/admin/actions/runners` on your instance.
|
||||
3. A runner name, the hostname when left blank.
|
||||
4. The runner [labels](#labels), the defaults when left blank.
|
||||
|
||||
The process looks like:
|
||||
|
||||
```text
|
||||
INFO Registering runner, arch=amd64, os=darwin, version=0.1.5.
|
||||
WARN Runner in user-mode.
|
||||
INFO Enter the Gitea instance URL (for example, https://gitea.com/):
|
||||
http://192.168.8.8:3000/
|
||||
INFO Enter the runner token:
|
||||
fe884e8027dc292970d4e0303fe82b14xxxxxxxx
|
||||
INFO Enter the runner name (if set empty, use hostname: Test.local):
|
||||
|
||||
INFO Enter the runner labels, leave blank to use the default labels (comma-separated, for example, ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest):
|
||||
|
||||
INFO Registering runner, name=Test.local, instance=http://192.168.8.8:3000/, labels=[ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04 ubuntu-20.04:docker://docker.gitea.com/runner-images:ubuntu-20.04].
|
||||
DEBU Successfully pinged the Gitea instance server
|
||||
INFO Runner registered successfully.
|
||||
```
|
||||
|
||||
You can also register with command line arguments.
|
||||
To register without prompts:
|
||||
|
||||
```bash
|
||||
./gitea-runner register --instance http://192.168.8.8:3000 --token <my_runner_token> --no-interactive
|
||||
./gitea-runner register --instance https://gitea.example.com --token <token> --no-interactive
|
||||
```
|
||||
|
||||
If the registry succeed, it will run immediately. Next time, you could run the runner directly.
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
./gitea-runner daemon
|
||||
```
|
||||
|
||||
## Deployment
|
||||
|
||||
### Windows service
|
||||
|
||||
Register the runner first, keeping `gitea-runner.exe`, `config.yaml` and the `.runner` file in one directory. Then install and start the service from an elevated Command Prompt:
|
||||
|
||||
```bat
|
||||
sc.exe create gitea-runner binPath= "\"C:\gitea-runner\gitea-runner.exe\" daemon --config \"C:\gitea-runner\config.yaml\"" start= auto
|
||||
sc.exe start gitea-runner
|
||||
```
|
||||
|
||||
The service logs to `gitea-runner.log` next to the executable. Its account needs write access to that directory and access to the tools jobs use.
|
||||
|
||||
`sc.exe stop gitea-runner` lets running jobs finish within `runner.shutdown_timeout`. To allow the same on shutdown, raise the service's preshutdown timeout above it, here to six minutes. This takes effect after the next reboot:
|
||||
|
||||
```bat
|
||||
reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\gitea-runner /v PreshutdownTimeout /t REG_DWORD /d 360000 /f
|
||||
```
|
||||
|
||||
### Run with docker
|
||||
|
||||
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
|
||||
|
||||
```bash
|
||||
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:latest-dind
|
||||
```
|
||||
|
||||
The volumes keep the runner's registration and the job image cache across container recreation.
|
||||
The volumes keep the registration and the image cache across container recreation. The image reads more variables, such as `GITEA_RUNNER_LABELS` and `CONFIG_FILE`, see [scripts/run.sh](scripts/run.sh) and the [Docker example](examples/docker/README.md).
|
||||
|
||||
### Image flavours
|
||||
#### Image flavours
|
||||
|
||||
| Tag | Docker daemon for jobs |
|
||||
| --- | --- |
|
||||
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as UID 1000 for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Its image cache is in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's `/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
|
||||
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
|
||||
Versioned and `nightly` tags take the same suffixes, like `nightly-dind`. The images target Docker, not Podman.
|
||||
|
||||
### Configuration
|
||||
### Example deployments
|
||||
|
||||
The runner reads a YAML file. Without one, every option keeps its default.
|
||||
See the [examples](examples) directory for more deployments, among them systemd, Kubernetes and VMs.
|
||||
|
||||
## Configuration
|
||||
|
||||
The runner reads an optional YAML file:
|
||||
|
||||
```bash
|
||||
./gitea-runner config init # write config.yaml, with no option set
|
||||
@@ -120,37 +124,68 @@ The runner reads a YAML file. Without one, every option keeps its default.
|
||||
./gitea-runner -c config.yaml daemon # -c also works on register and cache-server
|
||||
```
|
||||
|
||||
`config generate` prints [config.example.yaml](internal/pkg/config/config.example.yaml). Every value in it is commented out, so copy the lines you want to change into your own file and uncomment them.
|
||||
`config generate` prints [config.example.yaml](internal/pkg/config/config.example.yaml), which documents every option. Copy the lines you need into your file and uncomment them.
|
||||
|
||||
#### Editing a config file
|
||||
### Editing a config file
|
||||
|
||||
`config` edits a file in place, which is handy in provisioning scripts:
|
||||
`config` edits a file in place and keeps its comments, which suits provisioning scripts:
|
||||
|
||||
```bash
|
||||
./gitea-runner config set runner.capacity 4
|
||||
./gitea-runner config set runner.timeout 90m # written as 1h30m0s
|
||||
./gitea-runner config set runner.envs.MY_VAR value
|
||||
./gitea-runner config add runner.labels 'ubuntu:docker://node:22'
|
||||
./gitea-runner config remove runner.labels 'ubuntu:docker://node:22'
|
||||
./gitea-runner config get runner.labels
|
||||
```
|
||||
|
||||
A key is its dotted YAML path. An unknown key, a value of the wrong type, or `add`/`remove` on anything but a list is refused before the file is touched. `set` replaces a whole list when you give it several values.
|
||||
Keys are dotted YAML paths. Unknown keys and invalid values are refused without touching the file. Without `-c`, it uses `config.yaml` from the working directory, then from the binary's directory.
|
||||
|
||||
An edit keeps the comments and the key order of the file. Indentation becomes two spaces, and a blank line between two values is dropped.
|
||||
### Environment variables
|
||||
|
||||
`config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path.
|
||||
Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset variable fails the load, and `$${NAME}` stays a literal `${NAME}`.
|
||||
|
||||
#### Tool cache
|
||||
The override variables of earlier releases, such as `GITEA_RUNNER_CAPACITY`, are gone. Set those options in the file instead.
|
||||
|
||||
Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`, which is `/opt/hostedtoolcache` inside a job. `runner.tool_cache_mode` selects what backs it:
|
||||
### Labels
|
||||
|
||||
| Mode | Tool cache | Trade-off |
|
||||
| --- | --- | --- |
|
||||
| `none` (default) | Per job, provided by the job image | A version the image lacks is downloaded in every job |
|
||||
| `shared` | One volume reused by every job | Two jobs writing the same tool version at once corrupt it, so use it only with `runner.capacity: 1` |
|
||||
Labels decide which jobs a runner accepts and how it runs them. A job's `runs-on` is matched against the label names, and the first match picks where it runs.
|
||||
|
||||
With `none`, tools must come from the job image. Install them into `/opt/hostedtoolcache/<tool>/<version>/<arch>`, with an empty `<arch>.complete` file next to the directory:
|
||||
A label is written as `<name>[:<schema>[:<args>]]`, with one of these schemas:
|
||||
|
||||
- `docker://<image>` runs the job in a container of `<image>`.
|
||||
- `kubernetes://<image>` runs it as a Kubernetes pod, see [docs/kubernetes.md](docs/kubernetes.md).
|
||||
- `host`, the default, runs its steps directly on the runner's machine.
|
||||
|
||||
With these labels, `runs-on: ubuntu-latest` runs in a container and `runs-on: macos` on the host:
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
```
|
||||
|
||||
[gitea/runner-images](https://gitea.com/gitea/runner-images) has suitable images. Jobs matching no label run in `runner.default_image`, or on the host when docker is unavailable.
|
||||
|
||||
Host jobs still need a Docker daemon for Docker actions and `docker://` steps. Windows hosts do not support those, as on GitHub. A step's background processes end with the job unless it opts out, see [host mode](docs/host-mode.md).
|
||||
|
||||
#### Registration vs config labels
|
||||
|
||||
Labels are set at registration, with `--labels` or the prompt. To change them later, use one of these, the leftmost set one wins:
|
||||
|
||||
```
|
||||
daemon --labels / GITEA_RUNNER_LABELS > runner.labels in the config file > labels in the .runner file
|
||||
```
|
||||
|
||||
Changed labels are saved to the `.runner` file and sent to Gitea on startup. When `runner.labels` is set, `register` uses it and ignores `--labels`.
|
||||
|
||||
### Tool cache
|
||||
|
||||
Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`. `runner.tool_cache_mode` selects what backs it:
|
||||
|
||||
| Mode | Tool cache |
|
||||
| --- | --- |
|
||||
| `none` (default) | Provided by the job image. Versions it lacks are downloaded in every job. |
|
||||
| `shared` | One volume for all jobs. Only safe with `runner.capacity: 1`. |
|
||||
|
||||
To ship tools in a job image, install them to `/opt/hostedtoolcache/<tool>/<version>/<arch>` with an empty `<arch>.complete` file next to it, as the `-full` [runner images](https://gitea.com/gitea/runner-images) do:
|
||||
|
||||
```dockerfile
|
||||
RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1\.26\.[0-9]*' | head -1); \
|
||||
@@ -161,83 +196,29 @@ RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1
|
||||
touch "${DIR}.complete"
|
||||
```
|
||||
|
||||
A workflow requesting a minor version, `go-version: "1.26"`, resolves to the newest matching version in the cache, so a patch update in the image still hits it.
|
||||
### Proxy
|
||||
|
||||
Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavour is the one that ships tools in this layout.
|
||||
Set the proxy in the runner's environment, for example with systemd `Environment=` or `docker run -e`:
|
||||
|
||||
`gitea-runner exec` reads no config file and takes `--tool-cache-mode` instead, defaulting to `none`.
|
||||
|
||||
#### Environment variables
|
||||
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
|
||||
### Labels
|
||||
|
||||
Labels decide **which jobs a runner accepts** and **how it runs them**. A job's `runs-on` is matched against the runner's label names; the first match wins and selects the execution environment for that job.
|
||||
|
||||
A label is written as:
|
||||
|
||||
```text
|
||||
<name>[:<schema>[:<args>]]
|
||||
```sh
|
||||
http_proxy=http://proxy.example:3128
|
||||
https_proxy=http://proxy.example:3128
|
||||
no_proxy=gitea.internal,.example.local
|
||||
```
|
||||
|
||||
| Part | Meaning |
|
||||
| --- | --- |
|
||||
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
|
||||
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
|
||||
The runner uses it and passes it on to jobs. Jobs always reach the cache server, `localhost`, their service containers and a `tcp://` Docker daemon directly. Add Gitea to `no_proxy` if it should be reached directly too.
|
||||
|
||||
Three schemas are supported:
|
||||
To change the proxy for jobs, use `runner.envs`, a step's `env:` or `$GITHUB_ENV`. Workflow and job level `env:` have no effect on it.
|
||||
|
||||
- **`docker://<image>`** — the job runs inside a container created from `<image>`:
|
||||
The Docker daemon pulls images with its own proxy settings. The `dind` images pass the variables above to it, for other daemons see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/).
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
A password in the proxy URL is hidden in logs, but every step can read it.
|
||||
|
||||
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
|
||||
## Jobs
|
||||
|
||||
```text
|
||||
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
### Service containers
|
||||
|
||||
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
|
||||
|
||||
```text
|
||||
macos:host
|
||||
```
|
||||
|
||||
So with the labels
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
```
|
||||
|
||||
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
|
||||
|
||||
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
|
||||
|
||||
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
|
||||
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
|
||||
|
||||
#### Registration vs config labels
|
||||
|
||||
If `runner.labels` is set in the YAML file, those labels are used during `register` and the `--labels` CLI flag is ignored.
|
||||
|
||||
The `daemon` command also accepts `--labels` (which defaults to the `GITEA_RUNNER_LABELS` environment variable), so the labels of an already registered runner can be changed without deleting its registration file. The most explicit source wins:
|
||||
|
||||
```
|
||||
--labels / GITEA_RUNNER_LABELS > runner.labels in the config file > labels in the .runner file
|
||||
```
|
||||
|
||||
Whenever the resulting labels differ from the ones in the registration file, they are written back to it and re-declared to the Gitea instance on startup.
|
||||
|
||||
> **Note:** A runner that only exposes `host` labels still needs access to a Docker daemon (e.g. a mounted `/var/run/docker.sock`) whenever a job uses a `docker://` action or a service container. `host` labels only change where the job's own steps run; container-based steps and actions are still executed with Docker.
|
||||
|
||||
#### Service containers
|
||||
|
||||
A job's `services` are started before its steps run. When a service's image or its `options` declare a healthcheck, the runner waits for it to report healthy, so a workflow does not have to poll for its own services:
|
||||
When a service declares a healthcheck, in its image or `options`, the job waits for it to become healthy:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
@@ -249,63 +230,32 @@ services:
|
||||
--health-retries 10
|
||||
```
|
||||
|
||||
A service that reports unhealthy fails the job right away, with its container log. One that never becomes healthy fails it after `container.service_ready_timeout` (default `5m`, negative disables the wait). A service that exits without declaring a healthcheck only gets its log and a warning.
|
||||
A service that turns unhealthy, or is not healthy within `container.service_ready_timeout`, fails the job. Steps reach a service by its id and port, like `psql -h postgres -p 5432`.
|
||||
|
||||
A job in a container reaches a service by its id on the job network, on the port the service listens on, for example `psql -h postgres -p 5432`. The started containers also fill the `job` context: `job.container.{id,network}` and `job.services.<id>.{id,network,ports}`, where `ports` maps a container port to the host port Docker published it on, for the services that publish one.
|
||||
Unlike GitHub, jobs running on the host start no services. Give them a `container:` to use services.
|
||||
|
||||
Unlike GitHub, a job whose steps run on the host (a `host` label without `container:`) starts no service containers, so `job.services` and `job.container` stay empty. Give such a job a `container:` when it needs services.
|
||||
### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
|
||||
|
||||
#### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
|
||||
|
||||
Containers a job starts through the Docker socket can bind-mount the workspace by the job's own path, as on a host, for example `docker run -v "$PWD:/src"`. Without the Docker proxy, for example with a remote `DOCKER_HOST`, use `GITEA_DOCKER_WORKSPACE`, the path the daemon sees, as the prefix of workspace binds, with `.` as the fallback for local use:
|
||||
Containers a job starts can bind-mount the workspace by its path, like `docker run -v "$PWD:/src"`. Where the Docker daemon sees another path, for example a remote `DOCKER_HOST`, prefix the bind with `GITEA_DOCKER_WORKSPACE`:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
|
||||
In most Linux container jobs, the containers, networks and volumes a job creates are removed when it ends, `docker compose` services included. Set `container.sweep: false` if jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
#### Proxy
|
||||
### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
|
||||
`uses: builtin:checkout` checks out a repository without downloading `actions/checkout` or needing Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md).
|
||||
|
||||
```sh
|
||||
http_proxy=http://proxy.example:3128
|
||||
https_proxy=http://proxy.example:3128
|
||||
no_proxy=gitea.internal,.example.local
|
||||
```
|
||||
### Caching (`actions/cache`)
|
||||
|
||||
The runner uses them for its own requests and gives them to every job, in lower and upper case.
|
||||
Each runner runs its own cache server. As on GitHub, a job restores caches of its own ref, its pull request's base branch and the default branch.
|
||||
|
||||
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
|
||||
#### Sharing a cache between runners
|
||||
|
||||
- the built-in cache server, whose address is assigned at startup
|
||||
- `localhost`, `127.0.0.1` and `::1`
|
||||
- the job's service containers
|
||||
- the Docker daemon, when it is reached over `tcp://`
|
||||
|
||||
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
|
||||
|
||||
For one step, set it in the step's `env:`, for the rest of a job, write it to `$GITHUB_ENV`. Setting it at workflow or job level or in `container.env` has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
|
||||
|
||||
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
|
||||
|
||||
Dockerfile actions are built with these variables as build arguments, so their `RUN` steps can reach the network.
|
||||
|
||||
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
|
||||
|
||||
#### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
Use `uses: builtin:checkout` to check out a repository without downloading `actions/checkout` or needing Node in the job image, see **[docs/builtin-actions.md](docs/builtin-actions.md)**.
|
||||
|
||||
#### Caching (`actions/cache`)
|
||||
|
||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||
|
||||
**Sharing a cache between runners**
|
||||
|
||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||
Run one `gitea-runner cache-server` and point every runner at it with the same secret, for example from `openssl rand -hex 32`:
|
||||
|
||||
```yaml
|
||||
# cache-server.yaml
|
||||
@@ -326,13 +276,11 @@ cache:
|
||||
external_secret: "<secret>"
|
||||
```
|
||||
|
||||
`--dir`, `--host` and `--port` override the matching `cache` keys. Eviction settings take effect on the cache server, not on the runners.
|
||||
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower requests.
|
||||
|
||||
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower cache requests.
|
||||
#### Storing the cache in an S3-compatible bucket
|
||||
|
||||
**Storing the cache in an S3-compatible bucket**
|
||||
|
||||
The cache can also be stored in an S3-compatible bucket such as AWS S3, MinIO or RustFS, with no cache server. Runners with the same settings use the same cache:
|
||||
Runners can share a cache in an S3-compatible bucket, such as AWS S3, MinIO or RustFS, without a cache server:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
@@ -346,66 +294,48 @@ cache:
|
||||
secret_key_file: "/run/secrets/cache-s3-secret-key"
|
||||
```
|
||||
|
||||
A cache saved on one runner can be restored on the others within 30 seconds. The eviction settings below do not apply to the bucket, so add lifecycle rules to it that expire old caches and abort incomplete multipart uploads.
|
||||
Eviction does not apply to the bucket. Add lifecycle rules that expire old caches and abort incomplete multipart uploads.
|
||||
|
||||
**Eviction**
|
||||
#### Eviction
|
||||
|
||||
Entries not read or written within `retention` (default `168h`) are removed. A repository over `repo_size_limit` (default `10GB`) loses its least recently used entries, and `size_limit` (off by default) caps the whole cache the same way. Entries in use are never removed. The cache also keeps 1024 MiB free on its volume, or `health_check.min_free_disk_space_mb` when health checks are enabled. See [config.example.yaml](internal/pkg/config/config.example.yaml) for all options.
|
||||
Entries unused for `cache.retention` (default `168h`) are removed. A repository over `cache.repo_size_limit` (default `10GB`) loses its least recently used entries. See [config.example.yaml](internal/pkg/config/config.example.yaml) for more limits.
|
||||
|
||||
**Cache service v2**
|
||||
#### Cache service v2
|
||||
|
||||
`actions/cache` v3.4.0, v4.2.0 and later use the cache service v2 API, which the runner serves by default. These actions fall back to v1 on hosts they do not recognize as GitHub, so the runner removes that check from them while a job runs. This also lets the stock `actions/upload-artifact` v4.4.0 and `actions/download-artifact` v4.1.8 and later work without the `gitea-upload-artifact` fork. With `runner.patch_actions: false`, the cache stays on v1 and the artifact actions fail.
|
||||
The runner serves the cache service v2 API used by current `actions/cache` releases. It patches them to use it on Gitea, which also makes the stock `actions/upload-artifact` and `actions/download-artifact` v4 work. Artifact calls then go through the cache server, so jobs must be able to reach it.
|
||||
|
||||
With v2, artifact calls also go through the cache server, so jobs must be able to reach it. `cache.v2: false` sends them to Gitea directly, unless the Gitea URL has a path or `runner.insecure` is set with HTTPS.
|
||||
### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
|
||||
|
||||
#### Official Docker image
|
||||
Hook scripts run where the job's steps run, before the first step and after the last. They match GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`.
|
||||
|
||||
Besides `GITEA_INSTANCE_URL` and `GITEA_RUNNER_REGISTRATION_TOKEN`, the image entrypoint supports optional variables such as `CONFIG_FILE` (passed through as `-c`), `GITEA_RUNNER_LABELS`, `GITEA_RUNNER_EPHEMERAL`, `GITEA_RUNNER_ONCE`, `GITEA_RUNNER_NAME`, `GITEA_MAX_REG_ATTEMPTS`, `RUNNER_STATE_FILE`, and `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`. See [scripts/run.sh](scripts/run.sh) for exact behavior.
|
||||
Use them for per-job setup no workflow should carry, like registry logins or masking runner-wide secrets. They block the job, and a non-zero exit fails it. See [docs/job-hooks.md](docs/job-hooks.md).
|
||||
|
||||
For a fuller container-oriented walkthrough, see [examples/docker](examples/docker/README.md).
|
||||
### Secret masking
|
||||
|
||||
While the runner is idle it cleans up after earlier jobs:
|
||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||
Secrets and `::add-mask::` values are hidden in job logs, [local job logs](#local-job-logs-logjobdir), job summaries and container names. As on GitHub, a job output containing one is dropped with a warning.
|
||||
|
||||
#### Post-task script (`runner.post_task_script`)
|
||||
## Operation
|
||||
|
||||
Optional host script that runs **after** each task's built-in cleanup (post-steps, container teardown, bind-workdir removal). Use it for extra machine housekeeping — Docker pruning, disk cleanup, and similar.
|
||||
### Idle cleanup
|
||||
|
||||
**While the script runs, the runner stops task heartbeats and stays offline from Gitea's perspective until the script exits (or hits `runner.post_task_script_timeout`, default `5m`).** A script that blocks without exiting keeps the runner from taking new work for up to that timeout. Script output goes to the runner log, not the job log; a non-zero exit is warned but does not change the job result.
|
||||
While idle, the runner removes leftovers of earlier jobs every `runner.idle_cleanup_interval` (default `10m`):
|
||||
|
||||
On Windows, use `.exe`, `.bat`, or `.cmd` paths; **PowerShell (`.ps1`) is not supported yet** as the configured path — wrap commands in a `.cmd` file instead.
|
||||
- Workspaces older than `runner.workdir_cleanup_age` (default `24h`), when `container.bind_workdir` is on. This assumes no other runner shares `container.workdir_parent`.
|
||||
- Unused Docker networks and volumes of jobs the runner did not tear down.
|
||||
- Kubernetes job pods older than `runner.workdir_cleanup_age`, with their Secrets.
|
||||
|
||||
See **[docs/post-task-script.md](docs/post-task-script.md)** for lifecycle details, environment variables, timeout interaction, and platform notes.
|
||||
Set either option to `0` to turn cleanup off.
|
||||
|
||||
#### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
|
||||
### Post-task script (`runner.post_task_script`)
|
||||
|
||||
Optional scripts that run **inside the job environment** (the job container, or the host in host mode), before the job's first step and after its last one. They are the equivalent of GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`, which are read when the settings are unset.
|
||||
A host script that runs after each task's cleanup, for housekeeping like Docker pruning. The runner takes no new jobs until it exits or hits `runner.post_task_script_timeout` (default `5m`). Its output goes to the runner log.
|
||||
|
||||
Because they run where the steps run and see the job's environment, they are the place for per-job setup no workflow should have to carry: registry logins, mirror configuration, or masking runner-wide secrets with `::add-mask::`. Their output is part of the job log and is scanned for workflow commands, and they can export to the job through `$GITHUB_ENV` and `$GITHUB_PATH`.
|
||||
On Windows, use an `.exe`, `.bat` or `.cmd` file, PowerShell scripts are not supported. See [docs/post-task-script.md](docs/post-task-script.md).
|
||||
|
||||
Both hooks are synchronous and block the job while they run. Either one exiting non-zero fails the job, and there is no per-hook timeout.
|
||||
### Local job logs (`log.job.dir`)
|
||||
|
||||
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
|
||||
Set `log.job.dir` to keep a copy of every task's log, masked as in Gitea, with the job's result on the last line. Copies are kept for `log.job.retention` (default `168h`), so keep it above `runner.timeout`. Only the runner's user can read them.
|
||||
|
||||
#### OpenTelemetry (`OTEL_*`)
|
||||
### OpenTelemetry (`OTEL_*`)
|
||||
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
|
||||
|
||||
#### Local job logs (`log.job.dir`)
|
||||
|
||||
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
|
||||
|
||||
#### Secret masking
|
||||
|
||||
A job's secrets and its `::add-mask::` values are hidden from what the runner writes and uploads: the job log, the local copy above, job summaries, and the names of the containers it creates. A job output carrying one is skipped with a warning rather than sent masked, as GitHub does, so a downstream `needs.<job>.outputs.<name>` reading it is empty.
|
||||
|
||||
`log.job.retention` (default `168h`) is how long a log is kept, expired ones being deleted as new tasks start, and `log.job.max_size` (default `1GB`) caps one log. Keep `retention` above `runner.timeout` so a long job cannot outlive its own log, and prefer local disk, the file is written while the job runs. Only the runner's own user can read it.
|
||||
|
||||
### Example Deployments
|
||||
|
||||
Check out the [examples](examples) directory for sample deployment types.
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send job traces and runner metrics to an OpenTelemetry collector. See [docs/telemetry.md](docs/telemetry.md).
|
||||
@@ -6,7 +6,5 @@
|
||||
//
|
||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||
//
|
||||
// TODO: Authorization
|
||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||
package artifactcache
|
||||
@@ -59,6 +59,10 @@ type credKey struct{}
|
||||
type JobCredential struct {
|
||||
Repo string `json:"repo"`
|
||||
|
||||
Ref string `json:"ref"`
|
||||
BaseRef string `json:"base_ref"`
|
||||
DefaultRef string `json:"default_ref"`
|
||||
|
||||
// Results is the instance whose artifact service this server forwards for the job, and
|
||||
// InsecureTLS how the runner reaches it; see results.go.
|
||||
Results string `json:"-"`
|
||||
@@ -68,6 +72,10 @@ type JobCredential struct {
|
||||
PublicURL string `json:"public_url"`
|
||||
}
|
||||
|
||||
func (cred JobCredential) owns(cache *Cache) bool {
|
||||
return cache.Repo == cred.Repo && cache.Ref == cred.Ref
|
||||
}
|
||||
|
||||
// credEntry holds a registered job's credential along with an active
|
||||
// registration count. RegisterJob is reference-counted so that if two tasks
|
||||
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
||||
@@ -96,7 +104,8 @@ type Handler struct {
|
||||
stopSync context.CancelFunc
|
||||
syncing sync.WaitGroup
|
||||
|
||||
outboundIP string
|
||||
outboundIP string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||
// runner can use these endpoints to pre-register per-job
|
||||
@@ -116,7 +125,7 @@ type Handler struct {
|
||||
|
||||
policy Policy
|
||||
|
||||
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
|
||||
// freeDisk is a field so tests can drive makeRoom without a full volume.
|
||||
freeDisk func(string) (uint64, error)
|
||||
}
|
||||
|
||||
@@ -135,19 +144,21 @@ type Options struct {
|
||||
// in-process RegisterJob method directly.
|
||||
InternalSecret string
|
||||
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
InstanceTransport http.RoundTripper // reaches the instance artifact requests are forwarded to, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
||||
func StartHandler(opts Options) (*Handler, error) {
|
||||
dir, logger := opts.Dir, opts.Logger
|
||||
h := &Handler{
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
instanceTransport: opts.InstanceTransport,
|
||||
}
|
||||
|
||||
if logger == nil {
|
||||
@@ -653,7 +664,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, keys, version)
|
||||
cache, err := h.lookupCache(db, cred, keys, version)
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -669,26 +680,27 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
|
||||
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
|
||||
// entry is dropped on the way out.
|
||||
func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
||||
cache, err := findCache(db, repo, keys, version)
|
||||
if err != nil || cache == nil {
|
||||
return nil, err
|
||||
// lookupCache returns the entry to restore or (nil, nil), dropping matches whose blob was pruned.
|
||||
func (h *Handler) lookupCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
for {
|
||||
cache, err := findCache(db, cred, keys, version)
|
||||
if err != nil || cache == nil {
|
||||
return nil, err
|
||||
}
|
||||
ok, err := h.storage.Exist(cache.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||
// this call and the GET that follows it.
|
||||
h.touch(db, cache)
|
||||
return cache, nil
|
||||
}
|
||||
if !h.deleteCache(db, cache) {
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
}
|
||||
ok, err := h.storage.Exist(cache.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
h.deleteCache(db, cache)
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||
// this call and the GET that follows it.
|
||||
h.touch(db, cache)
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// POST /_apis/artifactcache/caches
|
||||
@@ -700,7 +712,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
cache := &Cache{Repo: cred.Repo, Ref: cred.Ref, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
if cache.Size == 0 {
|
||||
cache.Size = -1
|
||||
}
|
||||
@@ -749,7 +761,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -764,6 +776,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, 400, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -808,7 +821,7 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -866,11 +879,8 @@ func (h *Handler) commitCache(cache *Cache) error {
|
||||
if h.s3 != nil {
|
||||
return nil // lifecycle rules on the bucket bound its size
|
||||
}
|
||||
// A commit is the only thing that grows the store, so the only thing that can push the
|
||||
// volume under the floor.
|
||||
h.evictRepo(db, cache.Repo)
|
||||
h.evictTotal(db)
|
||||
h.evictForFreeSpace(db)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1071,25 +1081,35 @@ func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time
|
||||
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
||||
}
|
||||
|
||||
// if not found, return (nil, nil) instead of an error.
|
||||
func findCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
||||
for _, prefix := range keys {
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
exact, err := findExactCache(db, repo, prefix, version, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// findCache tries every key in one ref before the next ref, and returns (nil, nil) on a miss.
|
||||
func findCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
refs := []string{cred.Ref}
|
||||
for _, ref := range []string{cred.BaseRef, cred.DefaultRef} {
|
||||
if ref != "" && !slices.Contains(refs, ref) {
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
if exact != nil {
|
||||
return exact, nil
|
||||
}
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Repo == repo && strings.HasPrefix(cache.Key, prefix) && cache.Version == version && cache.Complete
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
if cache := latest(caches, (*Cache).createdAtNano); cache != nil {
|
||||
return cache, nil
|
||||
}
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Repo == cred.Repo && cache.Version == version && cache.Complete && slices.Contains(refs, cache.Ref) &&
|
||||
slices.ContainsFunc(keys, func(prefix string) bool { return strings.HasPrefix(cache.Key, prefix) })
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
for _, ref := range refs {
|
||||
for _, prefix := range keys {
|
||||
var exact, partial []*Cache
|
||||
for _, cache := range caches {
|
||||
if cache.Ref == ref && cache.Key == prefix {
|
||||
exact = append(exact, cache)
|
||||
} else if cache.Ref == ref && strings.HasPrefix(cache.Key, prefix) {
|
||||
partial = append(partial, cache)
|
||||
}
|
||||
}
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
if cache := cmp.Or(latest(exact, (*Cache).createdAtNano), latest(partial, (*Cache).createdAtNano)); cache != nil {
|
||||
return cache, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
||||
@@ -1111,9 +1131,9 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
||||
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
||||
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
||||
// touches UsedAt on every part.
|
||||
func findExactCache(db *cacheDB, repo, key, version string, complete bool) (*Cache, error) {
|
||||
func findExactCache(db *cacheDB, cred JobCredential, key, version string, complete bool) (*Cache, error) {
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Key == key && cache.Repo == repo && cache.Version == version && cache.Complete == complete
|
||||
return cred.owns(cache) && cache.Key == key && cache.Version == version && cache.Complete == complete
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
@@ -1154,9 +1174,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
||||
defer db.Close()
|
||||
|
||||
cache, err := db.get(id)
|
||||
if errors.Is(err, errCacheNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1241,23 +1258,29 @@ func (h *Handler) gcCache() {
|
||||
h.evictExpired(db)
|
||||
h.evictSuperseded(db)
|
||||
h.evictOversized(db)
|
||||
h.evictForFreeSpace(db)
|
||||
_ = db.Close()
|
||||
h.makeRoom(0)
|
||||
}
|
||||
|
||||
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
|
||||
// accounted for.
|
||||
func (h *Handler) evictForFreeSpace(db *cacheDB) {
|
||||
// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
|
||||
func (h *Handler) makeRoom(need int64) {
|
||||
free, err := h.freeDisk(h.dir)
|
||||
if err != nil {
|
||||
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
||||
return
|
||||
}
|
||||
if free >= uint64(h.policy.MinFreeDisk) {
|
||||
shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
|
||||
if h.s3 != nil || shortfall <= 0 {
|
||||
return
|
||||
}
|
||||
db, err := h.openDB()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
caches := h.completedByUse(db)
|
||||
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
|
||||
total := totalSize(caches)
|
||||
if total <= shortfall {
|
||||
// Say so, or shedding everything and still being short reads as the backstop working.
|
||||
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
||||
@@ -1280,13 +1303,12 @@ func (h *Handler) evictExpired(db *cacheDB) {
|
||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
||||
}
|
||||
|
||||
// evictSuperseded removes entries a newer one with the same key and version replaced. The
|
||||
// grouping includes Repo so two repos sharing a (key, version) do not evict each other.
|
||||
// evictSuperseded removes entries a newer one with the same repo, ref, key and version replaced.
|
||||
func (h *Handler) evictSuperseded(db *cacheDB) {
|
||||
type entry struct{ repo, key, version string }
|
||||
type entry struct{ repo, ref, key, version string }
|
||||
groups := make(map[entry][]*Cache)
|
||||
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
||||
group := entry{cache.Repo, cache.Key, cache.Version}
|
||||
group := entry{cache.Repo, cache.Ref, cache.Key, cache.Version}
|
||||
groups[group] = append(groups[group], cache)
|
||||
}
|
||||
for _, caches := range groups {
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
@@ -673,7 +674,7 @@ func TestHandler_evictPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
func TestHandler_makeRoom(t *testing.T) {
|
||||
t.Parallel()
|
||||
free := func(n int64) func(string) (uint64, error) {
|
||||
return func(string) (uint64, error) { return uint64(n), nil }
|
||||
@@ -682,14 +683,16 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
freeDisk func(string) (uint64, error)
|
||||
upload int64
|
||||
kept []string
|
||||
}{
|
||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
|
||||
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
|
||||
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
|
||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), 0, []string{"oldest", "middle", "newest"}},
|
||||
{"room for an upload sheds the oldest entry", free(defaultMinFreeDisk), 4 * miB, []string{"middle", "newest"}},
|
||||
{"a shortfall the cache cannot cover sheds all of it", free(0), 0, nil},
|
||||
{
|
||||
"an unreadable volume is treated as unavailable, not as full",
|
||||
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
||||
0,
|
||||
[]string{"oldest", "middle", "newest"},
|
||||
},
|
||||
} {
|
||||
@@ -702,11 +705,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
}
|
||||
handler := newTestHandler(t, Policy{}, entries...)
|
||||
handler.freeDisk = tc.freeDisk
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
handler.evictForFreeSpace(db)
|
||||
require.NoError(t, db.Close())
|
||||
handler.makeRoom(tc.upload)
|
||||
|
||||
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
||||
})
|
||||
@@ -759,11 +758,55 @@ func TestHandler_PrefixUsesNewestReservationWithinSecond(t *testing.T) {
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
cache, err := findCache(db, testRepo, []string{"deps-"}, "v")
|
||||
cache, err := findCache(db, JobCredential{Repo: testRepo}, []string{"deps-"}, "v")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, newer, cache)
|
||||
}
|
||||
|
||||
func TestHandler_RestoreRefs(t *testing.T) {
|
||||
t.Parallel()
|
||||
const main, release, pull = "refs/heads/main", "refs/heads/release", "refs/pull/1/head"
|
||||
now := time.Now().Unix()
|
||||
entry := func(ref, key string, age int64) *Cache {
|
||||
return &Cache{Repo: testRepo, Ref: ref, Key: key, Version: "v", Complete: true, CreatedAt: now - 3600 + age, UsedAt: now}
|
||||
}
|
||||
exactOnMain := entry(main, "deps-exact", -1)
|
||||
onRelease := entry(release, "deps-release", 0)
|
||||
onMain := entry(main, "deps-main", 1)
|
||||
onPull := entry(pull, "deps-pull", 2)
|
||||
unscoped := entry("", "deps-unscoped", 3)
|
||||
handler := newTestHandler(t, Policy{}, exactOnMain, onRelease, onMain, onPull, unscoped, entry("refs/pull/2/head", "deps-lost", 4))
|
||||
for _, cache := range []*Cache{exactOnMain, onRelease, onMain, onPull, unscoped} {
|
||||
writeBlob(t, handler, cache.ID)
|
||||
}
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cred JobCredential
|
||||
keys []string
|
||||
want *Cache
|
||||
}{
|
||||
{"the default branch never restores what a pull request wrote", JobCredential{Ref: main, DefaultRef: main}, []string{"deps-"}, onMain},
|
||||
{"a pull request restores its own entry first", JobCredential{Ref: pull, BaseRef: release, DefaultRef: main}, []string{"deps-"}, onPull},
|
||||
{"a pull request falls back past a lost blob to its base branch before the default branch", JobCredential{Ref: "refs/pull/2/head", BaseRef: release, DefaultRef: main}, []string{"deps-"}, onRelease},
|
||||
{"a branch does not restore a sibling branch", JobCredential{Ref: "refs/heads/other", DefaultRef: main}, []string{"deps-"}, onMain},
|
||||
{"a prefix in the own ref wins over an exact key in the default branch", JobCredential{Ref: release, DefaultRef: main}, []string{"deps-exact", "deps-"}, onRelease},
|
||||
{"a job with a ref never restores entries saved without one", JobCredential{Ref: "refs/heads/other"}, []string{"deps-"}, nil},
|
||||
{"a job without a ref only restores entries saved without one", JobCredential{}, []string{"deps-"}, unscoped},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.cred.Repo = testRepo
|
||||
cache, err := handler.lookupCache(db, tc.cred, tc.keys, "v")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_evictOnCommit(t *testing.T) {
|
||||
t.Parallel()
|
||||
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||
@@ -844,6 +887,13 @@ func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
|
||||
return kept
|
||||
}
|
||||
|
||||
func TestStorageDropsFailedWrites(t *testing.T) {
|
||||
storage, err := NewStorage(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
require.Error(t, storage.Write(1, 0, io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(io.ErrUnexpectedEOF))))
|
||||
assert.NoFileExists(t, storage.tempName(1, 0))
|
||||
}
|
||||
|
||||
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
||||
t.Helper()
|
||||
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
||||
@@ -938,6 +988,7 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||
defer handler.Close()
|
||||
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
||||
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
||||
handler.RegisterJob("token-a-pull", JobCredential{Repo: "owner/repoA", Ref: "refs/pull/1/head"})
|
||||
|
||||
base := handler.ExternalURL() + apiPath
|
||||
key := "shared-key"
|
||||
@@ -972,13 +1023,20 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Range", "bytes 0-5/*")
|
||||
resp, err = clientB.Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
for _, client := range []*http.Client{clientB, {Transport: &bearerTransport{token: "token-a-pull"}}} {
|
||||
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Range", "bytes 0-5/*")
|
||||
resp, err = client.Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
resp, err = client.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_ArtifactSignature(t *testing.T) {
|
||||
@@ -1083,49 +1141,20 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
||||
assert.Equal(t, content, body)
|
||||
}
|
||||
|
||||
func TestHandler_GC_PerRepoDedup(t *testing.T) {
|
||||
func TestHandler_GC_DedupPerRepoAndRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||
handler, err := StartHandler(Options{Dir: dir})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"})
|
||||
handler.RegisterJob("tok-b", JobCredential{Repo: "owner/repoB"})
|
||||
|
||||
key := "shared-dedup-key"
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
now := time.Now().Unix()
|
||||
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
||||
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
|
||||
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
|
||||
require.NoError(t, handler.insertCache(db, a))
|
||||
require.NoError(t, handler.insertCache(db, b))
|
||||
require.NoError(t, handler.storage.Write(a.ID, 0, strings.NewReader("a")))
|
||||
_, err = handler.storage.Commit(a.ID, 1)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, handler.storage.Write(b.ID, 0, strings.NewReader("b")))
|
||||
_, err = handler.storage.Commit(b.ID, 1)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
entries := []*Cache{
|
||||
{Repo: "owner/repoA", Key: "shared", Version: "v", Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1},
|
||||
{Repo: "owner/repoB", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||
{Repo: "owner/repoA", Ref: "refs/pull/1/head", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||
}
|
||||
handler := newTestHandler(t, Policy{}, entries...)
|
||||
handler.gcAt = time.Time{}
|
||||
handler.gcCache()
|
||||
|
||||
db, err = handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
after, err := db.find(func(cache *Cache) bool { return cache.Key == key && cache.Version == version })
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := make(map[string]bool)
|
||||
for _, c := range after {
|
||||
repos[c.Repo] = true
|
||||
}
|
||||
assert.True(t, repos["owner/repoA"], "repoA's cache must survive dedup against repoB")
|
||||
assert.True(t, repos["owner/repoB"], "repoB's cache must survive dedup against repoA")
|
||||
assert.Len(t, keptKeys(t, handler, entries), len(entries))
|
||||
}
|
||||
|
||||
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
||||
@@ -1190,10 +1219,10 @@ func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
||||
}
|
||||
|
||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","ref":"refs/pull/1/head","base_ref":"refs/heads/release","default_ref":"refs/heads/main","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||
cred, ok := handler.lookupCredential("via-internal-api")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, JobCredential{Repo: testRepo, PublicURL: "http://runner.example"}, cred)
|
||||
assert.Equal(t, JobCredential{Repo: testRepo, Ref: "refs/pull/1/head", BaseRef: "refs/heads/release", DefaultRef: "refs/heads/main", PublicURL: "http://runner.example"}, cred)
|
||||
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
|
||||
@@ -73,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||
// existing and silently never saved.
|
||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
||||
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if existing != nil {
|
||||
@@ -84,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||
owner := hashedToken(bearerToken(r))
|
||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
||||
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||
@@ -103,6 +103,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
cache := &Cache{
|
||||
Repo: cred.Repo,
|
||||
Ref: cred.Ref,
|
||||
Key: req.Key,
|
||||
Version: req.Version,
|
||||
Owner: owner,
|
||||
@@ -137,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
||||
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||
cache = nil // not the reservation this job made, so not this job's to commit
|
||||
}
|
||||
if err == nil && cache == nil {
|
||||
// A retry whose first response was lost finds the entry already committed.
|
||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
||||
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||
}
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
@@ -186,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
||||
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
@@ -217,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
|
||||
query := r.URL.Query()
|
||||
switch strings.ToLower(query.Get("comp")) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -84,6 +85,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
|
||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||
|
||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
@@ -231,6 +234,24 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||
clients := map[string]*http.Client{"x": testClient}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
require.Equal(t, true, created["ok"])
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||
}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||
}
|
||||
for content, client := range clients {
|
||||
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,7 @@ type Request struct {
|
||||
type Cache struct {
|
||||
ID uint64 `json:"id"`
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref,omitzero"`
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
Size int64 `json:"cacheSize"`
|
||||
|
||||
@@ -67,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||
},
|
||||
}
|
||||
if cred.InsecureTLS {
|
||||
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||
proxy.Transport = insecureTransport
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
|
||||
@@ -133,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||
t.Parallel()
|
||||
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||
defer gitea.Close()
|
||||
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -51,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(file, reader)
|
||||
if err = errors.Join(err, file.Close()); err != nil {
|
||||
_ = os.Remove(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -102,50 +104,46 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
return 0, err
|
||||
}
|
||||
written, err := assemble(name, tempNames)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||
}
|
||||
// If size is less than 0, it means the size is unknown.
|
||||
// We can't check the size of the file, just skip the check.
|
||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
||||
if size >= 0 && written != size {
|
||||
if err != nil {
|
||||
_ = os.Remove(name)
|
||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
||||
return 0, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
||||
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||
func assemble(name string, tempNames []string) (int64, error) {
|
||||
if len(tempNames) == 1 {
|
||||
info, err := os.Stat(tempNames[0])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), os.Rename(tempNames[0], name)
|
||||
if len(tempNames) == 0 {
|
||||
return 0, errors.New("nothing was uploaded")
|
||||
}
|
||||
|
||||
file, err := os.Create(name)
|
||||
if err := os.Rename(tempNames[0], name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var written int64
|
||||
for _, v := range tempNames {
|
||||
f, err := os.Open(v)
|
||||
for _, v := range tempNames[1:] {
|
||||
part, err := os.Open(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := io.Copy(file, f)
|
||||
_ = f.Close()
|
||||
_, err = io.Copy(file, part)
|
||||
_ = part.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written += n
|
||||
_ = os.Remove(v)
|
||||
}
|
||||
return written, nil
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), file.Close()
|
||||
}
|
||||
|
||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||
|
||||
@@ -281,7 +281,7 @@ func TestSafeResolve(t *testing.T) {
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
||||
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
@@ -158,6 +160,28 @@ func (e Executor) IfBool(conditional bool) Executor {
|
||||
})
|
||||
}
|
||||
|
||||
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||
|
||||
// Retry runs the executor up to attempts times until it succeeds.
|
||||
func (e Executor) Retry(attempts int) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := e(ctx)
|
||||
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
backoff := RetryBackoff()
|
||||
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finally adds an executor to run after other executor
|
||||
func (e Executor) Finally(finally Executor) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -134,6 +135,31 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||
}
|
||||
|
||||
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||
backoff := RetryBackoff
|
||||
RetryBackoff = func() time.Duration { return 0 }
|
||||
t.Cleanup(func() { RetryBackoff = backoff })
|
||||
transient := errors.New("transient")
|
||||
for _, tc := range []struct {
|
||||
failures, calls int
|
||||
err error
|
||||
}{
|
||||
{failures: 2, calls: 3},
|
||||
{failures: 5, calls: 3, err: transient},
|
||||
} {
|
||||
var calls int
|
||||
err := Executor(func(context.Context) error {
|
||||
calls++
|
||||
if calls <= tc.failures {
|
||||
return transient
|
||||
}
|
||||
return nil
|
||||
}).Retry(3)(t.Context())
|
||||
require.Equal(t, tc.err, err)
|
||||
require.Equal(t, tc.calls, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||
finalErr := errors.New("cleanup failed")
|
||||
for _, mainErr := range []error{context.Canceled, nil} {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
@@ -29,6 +30,10 @@ import (
|
||||
|
||||
var actionFetches sync.Map
|
||||
|
||||
var symlink = (*os.Root).Symlink
|
||||
|
||||
var symlinkFallback = runtime.GOOS == "windows"
|
||||
|
||||
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||
|
||||
type treeFile struct {
|
||||
@@ -229,6 +234,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
|
||||
sha := ""
|
||||
sizes := map[string]int64{}
|
||||
links := map[string]string{}
|
||||
var deferred []string
|
||||
reader := tar.NewReader(archive)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
@@ -257,9 +264,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
case tar.TypeReg:
|
||||
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||
case tar.TypeSymlink:
|
||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
}
|
||||
links[filepath.Clean(name)] = header.Linkname
|
||||
deferred = append(deferred, filepath.Clean(name))
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -268,12 +274,166 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
if !isFullObjectID(sha) {
|
||||
return "", errors.New("the archive names no commit")
|
||||
}
|
||||
linksByParent := map[string][]string{}
|
||||
if symlinkFallback {
|
||||
for name := range links {
|
||||
linksByParent[filepath.Dir(name)] = append(linksByParent[filepath.Dir(name)], filepath.Base(name))
|
||||
}
|
||||
}
|
||||
for requireTarget := symlinkFallback; len(deferred) > 0; {
|
||||
unresolved := deferred[:0]
|
||||
for _, name := range deferred {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if requireTarget {
|
||||
if _, err := root.Stat(filepath.Dir(name) + string(filepath.Separator) + links[name]); err != nil {
|
||||
unresolved = append(unresolved, name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := symlink(root, links[name], name); err == nil || !symlinkFallback {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := copyTreeLink(ctx, root, name, links, linksByParent, sizes); err != nil {
|
||||
return "", fmt.Errorf("materialize symlink %q: %w", filepath.ToSlash(name), err)
|
||||
}
|
||||
}
|
||||
if len(unresolved) == len(deferred) {
|
||||
requireTarget = false
|
||||
}
|
||||
deferred = unresolved
|
||||
}
|
||||
if err := root.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||
}
|
||||
|
||||
func resolveTreeLink(name string, links map[string]string) (string, bool) {
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(name, string(filepath.Separator))
|
||||
var resolved []string
|
||||
for count := 0; len(parts) > 0; {
|
||||
part := parts[0]
|
||||
parts = parts[1:]
|
||||
switch part {
|
||||
case "", ".":
|
||||
continue
|
||||
case "..":
|
||||
if len(resolved) == 0 {
|
||||
return "", false
|
||||
}
|
||||
resolved = resolved[:len(resolved)-1]
|
||||
continue
|
||||
}
|
||||
target, ok := links[filepath.Join(append(resolved, part)...)]
|
||||
if !ok {
|
||||
resolved = append(resolved, part)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if target == "" || os.IsPathSeparator(target[0]) || filepath.VolumeName(target) != "" || count > 255 {
|
||||
return "", false
|
||||
}
|
||||
parts = append(strings.Split(filepath.FromSlash(target), string(filepath.Separator)), parts...)
|
||||
}
|
||||
return filepath.Join(append([]string{"."}, resolved...)...), true
|
||||
}
|
||||
|
||||
var (
|
||||
errTreeLinkTarget = errors.New("unmaterializable symlink target")
|
||||
errTreeLinkCycle = fmt.Errorf("cyclic %w", errTreeLinkTarget)
|
||||
)
|
||||
|
||||
func copyTreeLink(ctx context.Context, root *os.Root, name string, links map[string]string, linksByParent map[string][]string, sizes map[string]int64) error {
|
||||
writeLinkText := func(destination, target string) error { // as git writes links it cannot create
|
||||
if err := root.RemoveAll(destination); err != nil {
|
||||
return err
|
||||
}
|
||||
for written := range sizes {
|
||||
if written == destination || strings.HasPrefix(written, destination+string(filepath.Separator)) {
|
||||
delete(sizes, written)
|
||||
}
|
||||
}
|
||||
size, err := writeTreeFile(root, destination, 0o644, strings.NewReader(target))
|
||||
sizes[destination] = size
|
||||
return err
|
||||
}
|
||||
active := map[string]bool{}
|
||||
var copyTarget func(string, string) error
|
||||
copyTarget = func(source, destination string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source, ok := resolveTreeLink(source, links)
|
||||
if !ok {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
file, err := root.Open(source)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode().IsRegular() {
|
||||
sizes[destination], err = writeTreeFile(root, destination, int64(info.Mode().Perm()), file)
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
if active[source] {
|
||||
return errTreeLinkCycle
|
||||
}
|
||||
active[source] = true
|
||||
defer delete(active, source)
|
||||
entries, err := file.ReadDir(-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children := map[string]bool{}
|
||||
for _, entry := range entries {
|
||||
children[entry.Name()] = true
|
||||
}
|
||||
for _, child := range linksByParent[source] {
|
||||
children[child] = true
|
||||
}
|
||||
if err := root.MkdirAll(destination, info.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
for child := range children {
|
||||
childSource, childDestination := filepath.Join(source, child), filepath.Join(destination, child)
|
||||
err := copyTarget(childSource, childDestination)
|
||||
if target, isLink := links[childSource]; isLink && errors.Is(err, errTreeLinkTarget) && !errors.Is(err, errTreeLinkCycle) {
|
||||
err = writeLinkText(childDestination, target)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := copyTarget(name, name); !errors.Is(err, errTreeLinkTarget) {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeLinkText(name, links[name])
|
||||
}
|
||||
|
||||
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||
|
||||
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
@@ -40,6 +41,7 @@ func TestFetchAction(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||
require.NoError(t, gitCmd("-C", remote, "update-index", "--chmod=+x", "run.sh"))
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||
expectedSha := gitRevParse(t, remote, "main")
|
||||
@@ -192,7 +194,9 @@ func TestFetchAction(t *testing.T) {
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.EqualValues(t, 0o755, info.Mode().Perm())
|
||||
}
|
||||
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", link)
|
||||
@@ -391,6 +395,79 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTreeCopiesDeferredLinks(t *testing.T) {
|
||||
originalSymlink, originalFallback := symlink, symlinkFallback
|
||||
symlink = func(_root *os.Root, _target, _name string) error { return errors.New("symlink privilege unavailable") }
|
||||
symlinkFallback = true
|
||||
t.Cleanup(func() { symlink, symlinkFallback = originalSymlink, originalFallback })
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
links [][2]string
|
||||
files []string
|
||||
text bool
|
||||
}{
|
||||
{name: "file chain", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "dist/index.js"}}, files: []string{"index.js", "entry.js"}},
|
||||
{name: "directory chain", links: [][2]string{{"lib", "bundle"}, {"bundle", "dist"}, {"dist/entry.js", "index.js"}}, files: []string{"lib/index.js", "lib/entry.js", "lib/nested/index.js", "bundle/index.js", "bundle/entry.js", "bundle/nested/index.js"}},
|
||||
{name: "directory in target path", links: [][2]string{{"index.js", "bundle/nested/index.js"}, {"bundle", "dist"}}, files: []string{"index.js", "bundle/nested/index.js"}},
|
||||
{name: "parent after directory link", links: [][2]string{{"index.js", "bundle/../index.js"}, {"bundle", "dist/nested"}}, files: []string{"index.js", "bundle/index.js"}},
|
||||
{name: "outside", links: [][2]string{{"index.js", "../outside.js"}}, text: true},
|
||||
{name: "absolute", links: [][2]string{{"index.js", t.TempDir()}}, text: true},
|
||||
{name: "dangling", links: [][2]string{{"index.js", "missing.js"}}, text: true},
|
||||
{name: "dangling child of directory link", links: [][2]string{{"lib", "dist"}, {"dist/optional", "missing.js"}}, files: []string{"lib/index.js", "lib/nested/index.js"}},
|
||||
{name: "cycle", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "index.js"}}, text: true},
|
||||
{name: "directory cycle", links: [][2]string{{"dist/loop", "."}}, text: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
sha := strings.Repeat("a", 40)
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist/nested", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
for _, link := range test.links {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: link[0], Typeflag: tar.TypeSymlink, Linkname: link[1]}))
|
||||
}
|
||||
content := "console.log('action');\n"
|
||||
for _, name := range []string{"dist/index.js", "dist/nested/index.js"} {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(content))}))
|
||||
_, err := writer.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
if test.text {
|
||||
for _, link := range test.links {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, link[1], string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
}
|
||||
}
|
||||
for _, name := range test.files {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), nil, 0o644))
|
||||
assert.False(t, treeIntact(dir, sha), name)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), []byte(content), 0o755))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||
@@ -410,3 +487,76 @@ func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
}
|
||||
|
||||
func TestExtractTreeCreatesLinksInPlatformOrder(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires symlink privileges")
|
||||
}
|
||||
original, originalFallback := symlink, symlinkFallback
|
||||
var created []string
|
||||
symlink = func(root *os.Root, target, name string) error {
|
||||
created = append(created, name)
|
||||
return root.Symlink(target, name)
|
||||
}
|
||||
t.Cleanup(func() { symlink, symlinkFallback = original, originalFallback })
|
||||
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||
for _, entry := range []*tar.Header{
|
||||
{Name: "a", Typeflag: tar.TypeSymlink, Linkname: "b"},
|
||||
{Name: "b", Typeflag: tar.TypeSymlink, Linkname: "dist"},
|
||||
{Name: "dangling", Typeflag: tar.TypeSymlink, Linkname: "missing"},
|
||||
{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||
} {
|
||||
require.NoError(t, writer.WriteHeader(entry))
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
fallback bool
|
||||
want []string
|
||||
}{
|
||||
{name: "native", want: []string{"a", "b", "dangling"}},
|
||||
{name: "fallback", fallback: true, want: []string{"b", "a", "dangling"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
symlinkFallback = test.fallback
|
||||
created = nil
|
||||
_, err := extractTree(t.Context(), bytes.NewReader(archive.Bytes()), t.TempDir(), false)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, created)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyTreeLinkCancellationBetweenEntries(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
defer root.Close()
|
||||
require.NoError(t, root.Mkdir("source", 0o755))
|
||||
for _, name := range []string{"first", "second"} {
|
||||
_, err := writeTreeFile(root, filepath.Join("source", name), 0o644, strings.NewReader("content"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
sizes := map[string]int64{}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
observed := &treeCopyContext{Context: ctx, check: func() {
|
||||
if len(sizes) == 1 {
|
||||
cancel()
|
||||
}
|
||||
}}
|
||||
require.ErrorIs(t, copyTreeLink(observed, root, "copy", map[string]string{"copy": "source"}, nil, sizes), context.Canceled)
|
||||
assert.Len(t, sizes, 1)
|
||||
}
|
||||
|
||||
type treeCopyContext struct {
|
||||
context.Context
|
||||
check func()
|
||||
}
|
||||
|
||||
func (ctx *treeCopyContext) Err() error {
|
||||
ctx.check()
|
||||
return ctx.Context.Err()
|
||||
}
|
||||
+37
-8
@@ -5,6 +5,7 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
"gitea.com/gitea/runner/internal/pkg/process"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -183,6 +185,7 @@ type NewGitCloneExecutorInput struct {
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
@@ -387,6 +390,21 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
return config
|
||||
}
|
||||
|
||||
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||
var config []string
|
||||
if certFile != "" {
|
||||
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||
}
|
||||
if caFile != "" {
|
||||
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||
}
|
||||
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||
config = append(config, prefix+"sslBackend=openssl")
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
@@ -444,9 +462,7 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
@@ -461,15 +477,28 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
var treeKill *process.TreeKill
|
||||
if runtime.GOOS == "windows" {
|
||||
treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
|
||||
} else {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stdout bytes.Buffer
|
||||
var stderr strings.Builder
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
err := cmd.Start()
|
||||
if err == nil {
|
||||
if treeKill != nil {
|
||||
if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
|
||||
defer killer.Close()
|
||||
}
|
||||
}
|
||||
err = cmd.Wait()
|
||||
}
|
||||
return output, nil
|
||||
if err != nil {
|
||||
return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
@@ -54,6 +54,22 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestSSLConfig(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
prefix, certFile, caFile, executionOS string
|
||||
expected []string
|
||||
}{
|
||||
{"http.https://example.com/.", "cert.pem", "ca.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslCAInfo=ca.pem", "http.https://example.com/.sslBackend=openssl"}},
|
||||
{"http.", "", "ca.pem", "windows", []string{"http.sslCAInfo=ca.pem", "http.sslBackend=openssl"}},
|
||||
{"http.", "cert.pem", "ca.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslCAInfo=ca.pem"}},
|
||||
{"http.", "", "", "Windows", nil},
|
||||
} {
|
||||
t.Run(testcase.prefix+testcase.certFile+testcase.caFile+testcase.executionOS, func(t *testing.T) {
|
||||
assert.Equal(t, testcase.expected, SSLConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.caFile, testcase.executionOS))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindGitSlug(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
remoteURL string
|
||||
@@ -185,7 +201,8 @@ func TestGitFindRef(t *testing.T) {
|
||||
func TestGitCloneExecutor(t *testing.T) {
|
||||
remoteDir, workDir := newGitRemote(t)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".gitattributes"), []byte("*.sh text\n"), 0o644))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
@@ -200,7 +217,7 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
@@ -676,8 +693,8 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
|
||||
select {
|
||||
case <-reached:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
case err := <-done:
|
||||
t.Fatalf("the executor finished before reaching the remote: %v", err)
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import "strings"
|
||||
|
||||
// LookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||
func LookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||
if value, ok := values[name]; ok || !caseInsensitive {
|
||||
return value, ok
|
||||
}
|
||||
for key, value := range values {
|
||||
if strings.EqualFold(key, name) {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
var zero V
|
||||
return zero, false
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
"github.com/moby/moby/api/types/container"
|
||||
)
|
||||
|
||||
var hostOS = runtime.GOOS // replaced in tests
|
||||
|
||||
// ExitCodeError reports a non-zero process exit code from a container command.
|
||||
type ExitCodeError int
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -48,6 +48,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
|
||||
options := client.ImageBuildOptions{
|
||||
Tags: []string{input.ImageTag},
|
||||
Labels: map[string]string{"com.gitea.runner.image": input.ImageTag}, // own image ID per tag, so removing one never deletes another job's identical build
|
||||
Remove: true,
|
||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||
Dockerfile: input.Dockerfile,
|
||||
@@ -125,6 +126,10 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
||||
}
|
||||
|
||||
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||
contextDir, err := filepath.EvalSymlinks(contextDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
seen := map[string]bool{}
|
||||
hardlinks := map[uint64]string{}
|
||||
@@ -132,8 +137,7 @@ func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, incl
|
||||
for _, include := range includes {
|
||||
include = filepath.FromSlash(include)
|
||||
var parents []parentMatch
|
||||
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
||||
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
||||
err := filepath.WalkDir(filepath.Join(contextDir, include), func(file string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
@@ -210,7 +214,7 @@ func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string,
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
}
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, filepath.ToSlash(link))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -18,10 +18,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||
func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||
var labels string
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||
labels = request.URL.Query().Get("labels")
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||
@@ -35,6 +37,7 @@ func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||
})(t.Context()), "build rejected")
|
||||
require.JSONEq(t, `{"com.gitea.runner.image":"action:test"}`, labels)
|
||||
}
|
||||
|
||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||
@@ -50,7 +53,7 @@ func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile
|
||||
|
||||
contexts := []string{dir}
|
||||
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||
contexts = append(contexts, link+string(filepath.Separator))
|
||||
contexts = append(contexts, link, link+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, contextDir := range contexts {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||
// * appended with license information
|
||||
@@ -351,7 +351,7 @@ type containerConfig struct {
|
||||
// parse parses the args for the specified command and generates a Config,
|
||||
// a HostConfig and returns them with the specified command.
|
||||
// If the specified args are not valid, it will return an error.
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo,unparam // verbatim copy from docker/cli
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo // verbatim copy from docker/cli
|
||||
var (
|
||||
attachStdin = copts.attach.Get("stdin")
|
||||
attachStdout = copts.attach.Get("stdout")
|
||||
@@ -724,6 +724,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isUserDefinedNetwork(mode container.NetworkMode) bool { // Docker's helper uses the host OS instead of the daemon's.
|
||||
return !mode.IsDefault() && mode != network.NetworkBridge && mode != network.NetworkHost && !mode.IsNone() && !mode.IsContainer()
|
||||
}
|
||||
|
||||
// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
|
||||
// them with the old --network-alias and --links. If returns an error if conflicting options
|
||||
// are found.
|
||||
@@ -752,7 +756,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
||||
}
|
||||
|
||||
for i, n := range copts.netMode.Value() {
|
||||
if container.NetworkMode(n.Target).IsUserDefined() {
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) {
|
||||
hasUserDefined = true
|
||||
} else {
|
||||
hasNonUserDefined = true
|
||||
@@ -814,7 +818,7 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
||||
}
|
||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||
// for the default bridge it defines a legacy-link.
|
||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) && copts.links.Len() > 0 {
|
||||
n.Links = slices.Clone(copts.links.GetSlice())
|
||||
}
|
||||
if copts.ipv4Address != nil {
|
||||
@@ -840,7 +844,7 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
||||
if strings.TrimSpace(ep.Target) == "" {
|
||||
return nil, errors.New("no name set for network")
|
||||
}
|
||||
if !container.NetworkMode(ep.Target).IsUserDefined() {
|
||||
if !isUserDefinedNetwork(container.NetworkMode(ep.Target)) {
|
||||
if len(ep.Aliases) > 0 {
|
||||
return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2025 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -40,7 +40,9 @@ const (
|
||||
jobLabel = "com.gitea.runner.job."
|
||||
maxCreateBody = 8 << 20
|
||||
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
dockerProxyProbeWorkerTimeout = time.Minute
|
||||
dockerProxyProbeTarget = "/gitea-runner-probe"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -48,7 +50,7 @@ var (
|
||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||
)
|
||||
|
||||
var dockerProxyDirectories, dockerProxyProbes sync.Map
|
||||
var dockerProxyProbes sync.Map
|
||||
|
||||
type dockerProxyDirectory struct {
|
||||
dir, daemonDir string
|
||||
@@ -68,7 +70,7 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
cli, err := GetDockerClient(probeCtx)
|
||||
cli, err := client.New(client.FromEnv) // negotiates on first request, so a cached probe costs no round trip
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -83,65 +85,69 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
}
|
||||
workdir, _ := os.Getwd()
|
||||
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||
cached, _ := dockerProxyDirectories.Load(key)
|
||||
directory, seen := cached.(dockerProxyDirectory)
|
||||
seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
|
||||
if !seen {
|
||||
directory, seen, err = probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||
}
|
||||
probe, err := probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
if !seen {
|
||||
if !probe.seen {
|
||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
proxy, err := StartDockerProxy(daemonSocket, directory.dir, job)
|
||||
proxy, err := StartDockerProxy(daemonSocket, probe.directory.dir, job)
|
||||
if err != nil {
|
||||
dockerProxyDirectories.Delete(key)
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
dockerProxyDirectories.Store(key, directory)
|
||||
proxy.Socket = directory.daemonDir + strings.TrimPrefix(proxy.Socket, directory.dir)
|
||||
proxy.Socket = probe.directory.daemonDir + strings.TrimPrefix(proxy.Socket, probe.directory.dir)
|
||||
return proxy
|
||||
}
|
||||
|
||||
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (dockerProxyDirectory, bool, error) {
|
||||
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (*dockerProxyProbe, error) {
|
||||
for {
|
||||
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
|
||||
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
|
||||
if loaded {
|
||||
probe, _ = actual.(*dockerProxyProbe)
|
||||
if !os.SameFile(probe.directory.socket, socket) {
|
||||
if !os.SameFile(probe.directory.socket, socket) { // daemon replacement invalidates cached paths
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return dockerProxyDirectory{}, false, ctx.Err()
|
||||
case <-probe.done:
|
||||
}
|
||||
if ctx.Err() == nil && (errors.Is(probe.err, context.Canceled) || errors.Is(probe.err, context.DeadlineExceeded)) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||
probe.seen = probe.directory.daemonDir != ""
|
||||
if !probe.seen {
|
||||
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
|
||||
probe.directory.daemonDir = probe.directory.dir
|
||||
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||
go func() { // outlives callers' deadlines so a slow daemon still gets its result cached
|
||||
defer cli.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeWorkerTimeout)
|
||||
defer cancel()
|
||||
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||
probe.seen = probe.directory.daemonDir != ""
|
||||
if !probe.seen {
|
||||
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
|
||||
probe.directory.daemonDir = probe.directory.dir
|
||||
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
close(probe.done)
|
||||
if probe.seen {
|
||||
dockerProxyProbes.LoadOrStore(key, probe)
|
||||
} else {
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
}
|
||||
close(probe.done)
|
||||
removeStaleProbeContainers(cli)
|
||||
}()
|
||||
}
|
||||
return probe.directory, probe.seen, probe.err
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return &dockerProxyProbe{}, ctx.Err()
|
||||
case <-probe.done:
|
||||
}
|
||||
if ctx.Err() == nil && errors.Is(probe.err, context.DeadlineExceeded) {
|
||||
continue
|
||||
}
|
||||
return probe, probe.err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,7 +206,7 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: dockerProxyProbeTarget, ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
}},
|
||||
})
|
||||
if !cerrdefs.IsNotFound(err) {
|
||||
@@ -213,7 +219,7 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, dockerProxyProbeTarget)
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||
@@ -222,6 +228,36 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
return seen, err
|
||||
}
|
||||
|
||||
// removeStaleProbeContainers removes probe containers left by abandoned creates once any probe in flight when listed has ended.
|
||||
func removeStaleProbeContainers(cli client.APIClient) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
listed, err := cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
Filters: make(client.Filters).Add("status", string(container.StateCreated)).Add("volume", dockerProxyProbeTarget),
|
||||
})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("listing stale docker proxy probe containers failed: %v", err)
|
||||
return
|
||||
}
|
||||
listed.Items = slices.DeleteFunc(listed.Items, func(stale container.Summary) bool {
|
||||
return stale.State != container.StateCreated || !slices.ContainsFunc(stale.Mounts, func(mountPoint container.MountPoint) bool {
|
||||
return mountPoint.Type == mount.TypeBind && !mountPoint.RW && mountPoint.Destination == dockerProxyProbeTarget
|
||||
})
|
||||
})
|
||||
if len(listed.Items) == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(dockerProxyProbeWorkerTimeout + dockerProxyProbeTimeout)
|
||||
for _, stale := range listed.Items {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
if err := (&containerReference{cli: cli, id: stale.ID}).remove()(ctx); err != nil {
|
||||
common.Logger(ctx).Warnf("removing stale docker proxy probe container failed: %v", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
info, err := os.Stat(daemonSocket)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -94,6 +94,8 @@ func TestDockerProxy(t *testing.T) {
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||
http.NotFound(w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||
@@ -235,7 +237,8 @@ func TestDockerProxy(t *testing.T) {
|
||||
t.Skip("automatic proxy discovery requires Linux")
|
||||
}
|
||||
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||
t.Setenv("TMPDIR", shortTempDir(t))
|
||||
tempDir := shortTempDir(t)
|
||||
t.Setenv("TMPDIR", tempDir)
|
||||
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||
for _, job := range []string{"first", "second"} {
|
||||
proxy := NewDockerProxy(t.Context(), job)
|
||||
@@ -243,12 +246,10 @@ func TestDockerProxy(t *testing.T) {
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
}
|
||||
assert.EqualValues(t, 2, probes.Load())
|
||||
dockerProxyDirectories.Range(func(key, value any) bool {
|
||||
directory, _ := value.(dockerProxyDirectory)
|
||||
directory.socket, _ = os.Stat(t.TempDir())
|
||||
dockerProxyDirectories.Store(key, directory)
|
||||
return true
|
||||
})
|
||||
workdir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
|
||||
cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
|
||||
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
@@ -476,10 +477,27 @@ func BenchmarkDockerProxy(b *testing.B) {
|
||||
|
||||
type probeClient struct {
|
||||
mobyclient.APIClient
|
||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
create func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
containers []container.Summary
|
||||
}
|
||||
|
||||
func (c *probeClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
return mobyclient.SystemInfoResult{}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
|
||||
return mobyclient.ContainerListResult{Items: c.containers}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||
return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||
@@ -490,8 +508,8 @@ func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (m
|
||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(opts)
|
||||
func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(ctx, opts)
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
@@ -518,7 +536,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
var calls, removals atomic.Int32
|
||||
visible := false
|
||||
cli := &probeClient{
|
||||
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
|
||||
@@ -538,23 +556,18 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
results := make(chan result, 8)
|
||||
for range cap(results) {
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
results <- result{seen, err}
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
results <- result{probe.seen, err}
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.False(t, seen)
|
||||
replacement, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
replaced := make(chan result, 1)
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||
replaced <- result{seen, err}
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||
replaced <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
@@ -574,11 +587,11 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
assert.Zero(t, removals.Load())
|
||||
}
|
||||
createErr, visible = nil, true
|
||||
directory, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, seen)
|
||||
assert.Equal(t, dir, directory.dir)
|
||||
assert.Equal(t, dir, directory.daemonDir)
|
||||
assert.True(t, probe.seen)
|
||||
assert.Equal(t, dir, probe.directory.dir)
|
||||
assert.Equal(t, dir, probe.directory.daemonDir)
|
||||
assert.EqualValues(t, 3, calls.Load())
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
@@ -586,7 +599,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
})
|
||||
})
|
||||
}
|
||||
t.Run("cancelled probe does not cancel another caller", func(t *testing.T) {
|
||||
t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
@@ -595,49 +608,105 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
calls := 0
|
||||
var calls, removals atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
<-release
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removals.Add(1)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
leader <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
follower <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
leaderResult := <-leader
|
||||
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||
assert.False(t, leaderResult.seen)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
late <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
close(release)
|
||||
for _, results := range []chan result{follower, late} {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
assert.True(t, result.seen)
|
||||
}
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
assert.EqualValues(t, 1, removals.Load())
|
||||
_, cached := dockerProxyProbes.Load(key)
|
||||
assert.True(t, cached)
|
||||
})
|
||||
})
|
||||
t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
var calls atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
<-ctx.Done()
|
||||
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||
} else {
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
leader, follower := make(chan result, 1), make(chan result, 1)
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
leader <- result{seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
follower <- result{seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
assert.Equal(t, 1, calls)
|
||||
cancel()
|
||||
close(release)
|
||||
leaderResult, followerResult := <-leader, <-follower
|
||||
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||
assert.False(t, leaderResult.seen)
|
||||
require.NoError(t, followerResult.err)
|
||||
assert.True(t, followerResult.seen)
|
||||
assert.Equal(t, 2, calls)
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, probe.seen)
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveStaleProbeContainers(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
|
||||
writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
|
||||
var removed []string
|
||||
cli := &probeClient{
|
||||
containers: []container.Summary{
|
||||
{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
|
||||
{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
|
||||
{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
|
||||
},
|
||||
remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removed = append(removed, id)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
start := time.Now()
|
||||
removeStaleProbeContainers(cli)
|
||||
assert.Equal(t, []string{"abandoned"}, removed)
|
||||
assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDaemonSeesDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
markers := make(map[string]bool)
|
||||
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
daemonDir string
|
||||
@@ -656,7 +725,7 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
removed := false
|
||||
var attemptedImages []string
|
||||
cli := &probeClient{
|
||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||
attemptedImages = append(attemptedImages, opts.Config.Image)
|
||||
if opts.Config.Image == "removed-image" {
|
||||
@@ -692,6 +761,7 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
|
||||
assert.Equal(t, testCase.seen, seen)
|
||||
assert.Equal(t, testCase.removed, removed)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
@@ -700,6 +770,9 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerContainerWorkdir(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("a runner container is Linux")
|
||||
}
|
||||
workdir := t.TempDir()
|
||||
t.Chdir(workdir)
|
||||
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
// NewDockerPullExecutor function to create a run executor for the container
|
||||
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
logger.Debugf("docker pull %v", input.Image)
|
||||
|
||||
@@ -83,7 +83,7 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||
}
|
||||
}).Retry(3)
|
||||
}
|
||||
|
||||
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
|
||||
|
||||
+39
-16
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -366,6 +366,7 @@ type containerReference struct {
|
||||
id string
|
||||
input *NewContainerInput
|
||||
pullPolicy string
|
||||
daemonOS string
|
||||
UID int
|
||||
GID int
|
||||
// attachDone is closed by the attach() streaming goroutine once it has
|
||||
@@ -375,13 +376,18 @@ type containerReference struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
}
|
||||
|
||||
func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
func GetDockerClient(ctx context.Context) (client.APIClient, error) {
|
||||
cli, _, err := newDockerClient(ctx)
|
||||
return cli, err
|
||||
}
|
||||
|
||||
func newDockerClient(ctx context.Context) (cli client.APIClient, daemonOS string, err error) {
|
||||
dockerHost := os.Getenv("DOCKER_HOST")
|
||||
|
||||
if strings.HasPrefix(dockerHost, "ssh://") {
|
||||
var args []string
|
||||
if args, err = sshArgs(dockerHost); err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
cli, err = client.New(
|
||||
client.WithHost("http://docker.example.com"), // the dialer below ignores the address
|
||||
@@ -391,16 +397,17 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
cli, err = client.New(client.FromEnv)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
return nil, "", fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
}
|
||||
// Best-effort API version negotiation, matching the old client.NegotiateAPIVersion
|
||||
// behaviour. Ping failures here are non-fatal: the connection is exercised again on
|
||||
// the first real call, and the client falls back to the daemon's default API version.
|
||||
if _, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true}); err != nil {
|
||||
ping, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("docker daemon ping during version negotiation failed, continuing: %v", err)
|
||||
}
|
||||
|
||||
return cli, nil
|
||||
return cli, ping.OSType, nil
|
||||
}
|
||||
|
||||
func GetHostInfo(ctx context.Context) (system.Info, error) {
|
||||
@@ -440,16 +447,32 @@ func RunnerArch(ctx context.Context) string {
|
||||
return info.Architecture
|
||||
}
|
||||
|
||||
func (cr *containerReference) targetOS(ctx context.Context) string {
|
||||
if cr.daemonOS == "" {
|
||||
cr.daemonOS = "linux"
|
||||
if cr.cli != nil {
|
||||
if result, err := cr.cli.Info(ctx, client.InfoOptions{}); err == nil && result.Info.OSType != "" {
|
||||
cr.daemonOS = result.Info.OSType
|
||||
}
|
||||
}
|
||||
}
|
||||
return cr.daemonOS
|
||||
}
|
||||
|
||||
func (cr *containerReference) markExecutable(ctx context.Context) bool {
|
||||
return hostOS == "windows" && cr.targetOS(ctx) == "linux"
|
||||
}
|
||||
|
||||
func (cr *containerReference) connect() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.cli != nil {
|
||||
return nil
|
||||
}
|
||||
cli, err := GetDockerClient(ctx)
|
||||
cli, daemonOS, err := newDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cr.cli = cli
|
||||
cr.cli, cr.daemonOS = cli, daemonOS
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -459,6 +482,7 @@ func (cr *containerReference) Close() common.Executor {
|
||||
if cr.cli != nil {
|
||||
err := cr.cli.Close()
|
||||
cr.cli = nil
|
||||
cr.daemonOS = ""
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close client: %w", err)
|
||||
}
|
||||
@@ -603,7 +627,8 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
copts.privileged = false
|
||||
}
|
||||
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
serverOS := cr.targetOS(ctx)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
@@ -618,7 +643,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
|
||||
// For Gitea, forcing --privileged off is not enough, other options reach the host too
|
||||
if !hostConfig.Privileged {
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions, serverOS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -731,9 +756,7 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
|
||||
var networkingConfig *network.NetworkingConfig
|
||||
n := hostConfig.NetworkMode
|
||||
// IsUserDefined and IsHost are broken on windows
|
||||
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
||||
if isUserDefinedNetwork(hostConfig.NetworkMode) && len(input.NetworkAliases) > 0 {
|
||||
networkingConfig = &network.NetworkingConfig{
|
||||
EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
input.NetworkMode: {Aliases: input.NetworkAliases},
|
||||
@@ -919,7 +942,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID, cr.markExecutable(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1090,12 +1113,12 @@ func resetOption[T any](logger *logging.Logger, option string, field *T, trusted
|
||||
|
||||
// parseOptionsHostConfig parses one options string on its own, to see what it alone asks for.
|
||||
// Even "" goes through the parser, or its empty slices and maps would differ from a real parse.
|
||||
func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
func parseOptionsHostConfig(options, serverOS string) (*container.HostConfig, error) {
|
||||
flags, copts, _, err := parseContainerOptions(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
|
||||
@@ -27,10 +27,12 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/api/types/system"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
@@ -65,6 +67,14 @@ func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
|
||||
type mockDockerClient struct {
|
||||
mobyclient.APIClient
|
||||
mock.Mock
|
||||
info system.Info
|
||||
infoErr error
|
||||
infoCalls int
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
m.infoCalls++
|
||||
return mobyclient.SystemInfoResult{Info: m.info}, m.infoErr
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ServerVersion(ctx context.Context, opts mobyclient.ServerVersionOptions) (mobyclient.ServerVersionResult, error) {
|
||||
@@ -689,11 +699,11 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||
hostFile := filepath.Join(t.TempDir(), "host.env")
|
||||
hostFile := filepath.Join(t.TempDir(), "host file.env")
|
||||
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
||||
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
||||
|
||||
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
||||
for _, option := range []string{shellquote.Join("--env-file", hostFile), shellquote.Join("--label-file", hostFile)} {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
||||
|
||||
@@ -933,6 +943,7 @@ func TestContainerInfoFromInspect(t *testing.T) {
|
||||
"/etc/hosts": "",
|
||||
"/etc/hostname": "",
|
||||
}, info.Mounts)
|
||||
assert.Equal(t, "/var/lib/docker/volumes/job/_data/subdir", info.DaemonPath("/workspace/owner/repo/subdir"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -980,7 +991,7 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||
},
|
||||
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if len(entrypoint) > 0 {
|
||||
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||
} else {
|
||||
@@ -998,6 +1009,39 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCreateNetworkAliases(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
networkMode string
|
||||
wantAliases bool
|
||||
}{
|
||||
{"job-network", true},
|
||||
{"host", false},
|
||||
{"bridge", false},
|
||||
{"default", false},
|
||||
{"none", false},
|
||||
{"container:x", false},
|
||||
} {
|
||||
t.Run(test.networkMode, func(t *testing.T) {
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{NetworkMode: test.networkMode, NetworkAliases: []string{"service"}},
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, container.NetworkMode(test.networkMode), opts.HostConfig.NetworkMode)
|
||||
if test.wantAliases {
|
||||
assert.Equal(t, &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
"job-network": {Aliases: []string{"service"}},
|
||||
}}, opts.NetworkingConfig)
|
||||
} else {
|
||||
assert.Nil(t, opts.NetworkingConfig)
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "created"}, nil
|
||||
}},
|
||||
}
|
||||
require.NoError(t, cr.create(nil, nil)(t.Context()))
|
||||
assert.Equal(t, "created", cr.id)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
@@ -1019,3 +1063,92 @@ func TestSupportsContainerImagePlatformDaemonError(t *testing.T) {
|
||||
require.ErrorContains(t, err, "cannot connect to the Docker daemon")
|
||||
assert.False(t, supported)
|
||||
}
|
||||
|
||||
func TestContainerOptionsUseDaemonOS(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
serverOS string
|
||||
infoErr error
|
||||
device string
|
||||
expected container.DeviceMapping
|
||||
}{
|
||||
{name: "Linux daemon", serverOS: "linux", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unknown daemon", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unavailable daemon info", infoErr: errors.New("unavailable"), device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "Windows daemon", serverOS: "windows", device: "class/01234567-89ab-cdef-0123-456789abcdef", expected: container.DeviceMapping{PathOnHost: "class/01234567-89ab-cdef-0123-456789abcdef"}},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}, infoErr: testcase.infoErr}
|
||||
cr := &containerReference{cli: cli, input: &NewContainerInput{RunnerOptions: "--device=" + testcase.device, WorkflowOptions: "--device=" + testcase.device, NetworkMode: "bridge"}}
|
||||
for range 2 {
|
||||
hostConfig := &container.HostConfig{}
|
||||
require.NoError(t, cr.mergeContainerConfigs(t.Context(), &container.Config{}, hostConfig))
|
||||
assert.Equal(t, []container.DeviceMapping{testcase.expected}, hostConfig.Devices)
|
||||
}
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTakesDaemonOSFromPing(t *testing.T) {
|
||||
var infos atomic.Int32
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
writer.Header().Set("OSType", "windows")
|
||||
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||
infos.Add(1)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
cr := &containerReference{input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.connect()(t.Context()))
|
||||
assert.Equal(t, "windows", cr.targetOS(t.Context()))
|
||||
assert.Zero(t, infos.Load())
|
||||
}
|
||||
|
||||
func TestContainerCopyModes(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
hostOS, serverOS string
|
||||
expected int64
|
||||
}{
|
||||
{"windows", "linux", 0o755},
|
||||
{"windows", "windows", 0o666},
|
||||
{"linux", "linux", 0o666},
|
||||
} {
|
||||
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
||||
original := hostOS
|
||||
hostOS = testcase.hostOS
|
||||
t.Cleanup(func() { hostOS = original })
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}}
|
||||
var modes []int64
|
||||
cli.On("CopyToContainer", mock.Anything, "job", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
reader := tar.NewReader(args.Get(2).(mobyclient.CopyToContainerOptions).Content)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
modes = append(modes, header.Mode)
|
||||
assert.Equal(t, "run.sh", filepath.Base(header.Name))
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, file.Body, string(content))
|
||||
_, err = reader.Next()
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
}).Return(mobyclient.CopyToContainerResult{}, nil).Twice()
|
||||
cr := &containerReference{cli: cli, id: "job", input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.copyContent("/workspace", file)(t.Context()))
|
||||
require.NoError(t, cr.copyDir("/workspace", source+string(filepath.Separator), false, true)(t.Context()))
|
||||
assert.Equal(t, []int64{0o600, testcase.expected}, modes)
|
||||
if testcase.hostOS == "windows" {
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
} else {
|
||||
assert.Zero(t, cli.infoCalls)
|
||||
}
|
||||
cli.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -4,7 +4,18 @@
|
||||
|
||||
package container
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func JoinPath(elem ...string) string { // path.Join would collapse the leading // of a UNC act path on a Windows host
|
||||
if len(elem) > 0 && strings.HasPrefix(elem[0], "//") {
|
||||
return "/" + path.Join(elem...)
|
||||
}
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
type ExecutionsEnvironment interface {
|
||||
Container
|
||||
|
||||
@@ -40,7 +40,8 @@ type HostEnvironment struct {
|
||||
// when Workdir points at a caller-owned checkout (e.g. `act` local mode).
|
||||
CleanWorkdir bool
|
||||
ActPath string
|
||||
CleanUp func()
|
||||
Root string
|
||||
TrackingID string // RUNNER_TRACKING_ID of the job, a step with another value keeps its leftovers
|
||||
StdOut io.Writer
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
|
||||
@@ -56,7 +57,7 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
||||
e.procGroupOnce.Do(func() {
|
||||
group, err := process.NewGroup()
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("could not create the job's process group; processes a step leaves behind can only be reclaimed by the workspace scan: %v", err)
|
||||
common.Logger(ctx).Warnf("could not create the job's process group, processes a step leaves behind may outlive the job: %v", err)
|
||||
return
|
||||
}
|
||||
e.procGroup.Store(group)
|
||||
@@ -261,6 +262,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||
defer common.FlushWriter(e.StdOut)
|
||||
process.UTF8Console()
|
||||
|
||||
f, err := lookupPathHost(command[0], env, e.StdOut)
|
||||
if err != nil {
|
||||
@@ -270,7 +272,12 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
cmd.Path = f
|
||||
cmd.Args = command
|
||||
cmd.Stdout = e.StdOut
|
||||
cmd.Env = getEnvListFromMap(env)
|
||||
trackingID := e.TrackingID
|
||||
if value, ok := common.LookupFold(env, "RUNNER_TRACKING_ID", e.IsEnvironmentCaseInsensitive()); ok {
|
||||
trackingID = value
|
||||
}
|
||||
tracked := trackingID == e.TrackingID
|
||||
cmd.Env = append(getEnvListFromMap(env), "RUNNER_TRACKING_ID="+trackingID)
|
||||
cmd.Stderr = e.StdOut
|
||||
cmd.Dir = wd
|
||||
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
||||
@@ -278,6 +285,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
// Kills the step's whole tree on cancellation and bounds the post-exit I/O
|
||||
// wait, so an orphan holding cmd's stdout pipe cannot hang cmd.Wait().
|
||||
treeKill := process.NewTreeKill(cmd)
|
||||
treeKill.KeepLeftovers = !tracked
|
||||
|
||||
var ppty *os.File
|
||||
var tty *os.File
|
||||
@@ -311,10 +319,10 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Assign before the step's Killer so the step's job nests inside the group's;
|
||||
// cancellation still scopes to this step's tree.
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group; a process it leaves behind may outlive the job: %v", err)
|
||||
if tracked {
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group, a process it leaves behind may outlive the job: %v", err)
|
||||
}
|
||||
}
|
||||
if k, kerr := treeKill.Capture(cmd.Process); kerr != nil {
|
||||
common.Logger(ctx).Warnf("process tree kill setup failed, falling back to single-process kill: %v", kerr)
|
||||
@@ -365,9 +373,7 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string)
|
||||
// removeAll is a var so tests can substitute a blocking stub.
|
||||
var removeAll = os.RemoveAll
|
||||
|
||||
// removeAllWithContext returns once the delete finishes or ctx is cancelled. On
|
||||
// cancellation the goroutine leaks: a delete inside a syscall cannot be
|
||||
// interrupted (see runWithTimeout).
|
||||
// Returns on cancel while an uninterruptible delete keeps running, freeing the runner's capacity slot.
|
||||
func removeAllWithContext(ctx context.Context, path string) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- removeAll(path) }()
|
||||
@@ -409,129 +415,22 @@ func removePathWithRetry(ctx context.Context, path string) error {
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// buildWindowsWorkspaceKillScript builds a PowerShell command that taskkills
|
||||
// every process tree whose ExecutablePath or CommandLine references one of the
|
||||
// given workspace dirs, releasing file handles for cleanup. Win32_Process
|
||||
// exposes both fields (Get-Process doesn't, wmic is deprecated); matching is on
|
||||
// the dir+separator prefix via ordinal String methods, so a name-prefix sibling
|
||||
// (job1 vs job10) is spared and path metacharacters stay literal.
|
||||
func buildWindowsWorkspaceKillScript(dirs []string) string {
|
||||
quoted := make([]string, len(dirs))
|
||||
for i, d := range dirs {
|
||||
// Single-quoted PowerShell literal; escape ' by doubling it.
|
||||
quoted[i] = "'" + strings.ReplaceAll(d, "'", "''") + "'"
|
||||
}
|
||||
|
||||
return `$paths = @(` + strings.Join(quoted, ",") + `)
|
||||
$selfPid = $PID
|
||||
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object {
|
||||
if ($_.ProcessId -eq $selfPid) { return $false }
|
||||
foreach ($p in $paths) {
|
||||
$prefix = $p + '\'
|
||||
if ($_.ExecutablePath -and $_.ExecutablePath.StartsWith($prefix, [System.StringComparison]::OrdinalIgnoreCase)) { return $true }
|
||||
if ($_.CommandLine -and $_.CommandLine.IndexOf($prefix, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) { return $true }
|
||||
}
|
||||
return $false
|
||||
} | ForEach-Object {
|
||||
& taskkill.exe /PID $_.ProcessId /T /F 2>$null | Out-Null
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) terminateRunningProcesses(ctx context.Context) {
|
||||
if runtime.GOOS != "windows" {
|
||||
return
|
||||
}
|
||||
|
||||
// Detached: exec.CommandContext won't start on a cancelled ctx, and a
|
||||
// server cancel has already cancelled the parent ctx.
|
||||
killCtx, killCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer killCancel()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// Dirs we own; a process referencing one is a leftover. ToolCache is shared
|
||||
// across jobs, and Workdir may be a caller-owned checkout.
|
||||
owned := []string{e.Path, e.TmpDir}
|
||||
if e.CleanWorkdir {
|
||||
owned = append(owned, e.Workdir)
|
||||
}
|
||||
dirs := make([]string, 0, len(owned))
|
||||
for _, d := range owned {
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := filepath.Abs(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dirs = append(dirs, abs)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
script := buildWindowsWorkspaceKillScript(dirs)
|
||||
|
||||
cmd := exec.CommandContext(killCtx, "powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
logger.Debugf("workspace process-tree kill via PowerShell failed: %v output=%s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Win32_Process exposes no working directory, so the scan above misses a
|
||||
// process that merely runs in a workspace dir while pinning a handle on it.
|
||||
if killed, err := process.KillProcessesWithCWDUnder(killCtx, dirs); err != nil {
|
||||
logger.Debugf("workspace process kill by working directory reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("terminated %d leftover process(es) by workspace working directory", killed)
|
||||
}
|
||||
}
|
||||
|
||||
// hostCleanupTimeout bounds each teardown phase so one stalled delete cannot
|
||||
// wedge the runner slot. A var so tests can shrink it.
|
||||
var hostCleanupTimeout = 30 * time.Second
|
||||
|
||||
// runWithTimeout returns context.DeadlineExceeded once timeout elapses, leaking
|
||||
// the goroutine: a delete blocked in a syscall (AV filter driver, dead network
|
||||
// mount) cannot be interrupted, and leaking scratch state beats losing the
|
||||
// runner's capacity slot forever. The idle stale-dir sweep reclaims it later.
|
||||
func runWithTimeout(fn func(), timeout time.Duration) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
fn()
|
||||
}()
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-timer.C:
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// End lingering processes before removing the workspace; on Windows their
|
||||
// file locks block cleanup. Closing the group is deterministic, the scan a net.
|
||||
// End lingering processes before removing the workspace, on Windows their file locks block cleanup.
|
||||
if err := e.procGroup.Load().Close(); err != nil {
|
||||
logger.Debugf("closing the job's process group failed: %v", err)
|
||||
}
|
||||
e.terminateRunningProcesses(ctx)
|
||||
|
||||
// Removes per-job misc state only, never the toolcache root. Bounded because
|
||||
// CleanUp is a caller-supplied, typically unbounded os.RemoveAll.
|
||||
if e.CleanUp != nil {
|
||||
logger.Debugf("running host environment cleanup callback")
|
||||
if err := runWithTimeout(e.CleanUp, hostCleanupTimeout); err != nil {
|
||||
logger.Warnf("host environment cleanup did not finish within %s; continuing job completion, scratch state may be leaked and is reclaimed by the idle stale-dir sweep", hostCleanupTimeout)
|
||||
} else {
|
||||
logger.Debugf("host environment cleanup callback finished")
|
||||
}
|
||||
if killed, err := process.KillTrackedProcesses(e.TrackingID); err != nil {
|
||||
logger.Debugf("killing the job's leftover processes reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("killed %d leftover process(es) of the job", killed)
|
||||
}
|
||||
|
||||
// Detach: a cancelled ctx would skip removePathWithRetry's retries,
|
||||
@@ -540,8 +439,8 @@ func (e *HostEnvironment) Remove() common.Executor {
|
||||
defer rmCancel()
|
||||
|
||||
var errs []error
|
||||
if err := removePathWithRetry(rmCtx, e.Path); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", e.Path, err)
|
||||
if err := removePathWithRetry(rmCtx, cmp.Or(e.Root, e.Path)); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", cmp.Or(e.Root, e.Path), err)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if e.CleanWorkdir {
|
||||
@@ -613,6 +512,7 @@ func goOsToActionOs(os string) string {
|
||||
"linux": "Linux",
|
||||
"darwin": "macOS",
|
||||
"windows": "Windows",
|
||||
"freebsd": "FreeBSD",
|
||||
}[os], os)
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
var _ ExecutionsEnvironment = &HostEnvironment{}
|
||||
|
||||
func TestActionPlatformNames(t *testing.T) {
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows")})
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows", "FreeBSD"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows"), goOsToActionOs("freebsd")})
|
||||
assert.Equal(t, []string{"X86", "X64", "ARM", "ARM64"}, []string{goArchToActionArch("386"), goArchToActionArch("amd64"), goArchToActionArch("arm"), goArchToActionArch("arm64")})
|
||||
}
|
||||
|
||||
@@ -161,10 +161,12 @@ func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
||||
Path: filepath.Join(miscRoot, "hostexecutor"),
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: owned,
|
||||
CleanUp: func() { _ = os.RemoveAll(miscRoot) },
|
||||
Root: miscRoot,
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(miscRoot, "tmp"), 0o700))
|
||||
require.NoError(t, e.Remove()(t.Context()))
|
||||
assert.NoDirExists(t, miscRoot)
|
||||
_, err := os.Stat(workdir)
|
||||
if owned {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
@@ -201,87 +203,38 @@ func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
origTimeout := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
block := func() {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
}
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
if phase == "CleanUp callback" {
|
||||
e.CleanUp = block
|
||||
} else {
|
||||
origRemoveAll := removeAll
|
||||
removeAll = func(string) error {
|
||||
block()
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeAll = origRemoveAll })
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
})
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("Remove() hung on a stuck %s", phase)
|
||||
}
|
||||
})
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
origTimeout, origRemoveAll := hostCleanupTimeout, removeAll
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
hostCleanupTimeout, removeAll = origTimeout, origRemoveAll
|
||||
})
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck path removal")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestJoinPath(t *testing.T) {
|
||||
assert.Equal(t, "//server/share/act/workflow/envs.txt", JoinPath("//server/share/act", "workflow/envs.txt"))
|
||||
assert.Equal(t, "/var/run/act/workflow/envs.txt", JoinPath("/var/run/act/", "workflow", "envs.txt"))
|
||||
for _, root := range []string{"//server/share/act", "/var/run/act", "C:/act"} {
|
||||
assert.Equal(t, root+"/actions/key/main.go.out", JoinPath(root+"/actions/key", "main.go.out"))
|
||||
assert.Equal(t, root+"/workflow", JoinPath(root+"/workflow/outputcmd.txt", ".."))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
t.Run("single dir", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
||||
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
||||
assert.Contains(t, s, "$selfPid = $PID")
|
||||
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
||||
assert.Contains(t, s, `$prefix = $p + '\'`)
|
||||
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
||||
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
||||
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
||||
})
|
||||
|
||||
t.Run("multiple dirs comma-separated", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{
|
||||
`C:\work\path`,
|
||||
`C:\work\workdir`,
|
||||
`C:\Users\runner\AppData\Local\Temp\job-42`,
|
||||
})
|
||||
assert.Contains(t, s, `'C:\work\path'`)
|
||||
assert.Contains(t, s, `'C:\work\workdir'`)
|
||||
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
||||
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
||||
})
|
||||
|
||||
t.Run("path with single quote is escaped", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
||||
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
||||
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
||||
})
|
||||
|
||||
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
||||
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
||||
assert.NotContains(t, s, "-like")
|
||||
assert.Contains(t, s, "StartsWith")
|
||||
assert.Contains(t, s, "IndexOf")
|
||||
})
|
||||
|
||||
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript(nil)
|
||||
assert.Contains(t, s, "$paths = @()")
|
||||
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func TestHostEnvironmentTrackingCleanup(t *testing.T) {
|
||||
for _, tracking := range []string{"default", "", "shared-helper"} {
|
||||
t.Run("tracking="+tracking, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer listener.Close()
|
||||
stop := context.AfterFunc(ctx, func() { _ = listener.Close() })
|
||||
defer stop()
|
||||
workspace := t.TempDir()
|
||||
environment := &HostEnvironment{Path: t.TempDir(), TmpDir: workspace, Workdir: workspace, TrackingID: "gitea_test", StdOut: io.Discard}
|
||||
env := map[string]string{}
|
||||
for _, entry := range os.Environ() {
|
||||
if name, value, ok := strings.Cut(entry, "="); ok && !strings.EqualFold(name, "RUNNER_TRACKING_ID") {
|
||||
env[name] = value
|
||||
}
|
||||
}
|
||||
env["GITEA_TRACKING_HELPER"] = "step"
|
||||
env["GITEA_TRACKING_ADDRESS"] = listener.Addr().String()
|
||||
if tracking == "default" {
|
||||
env["RUNNER_TRACKING_ID"] = environment.TrackingID
|
||||
env["runner_tracking_id"] = "shared-helper"
|
||||
} else {
|
||||
env["runner_tracking_id"] = tracking
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
result <- environment.Exec([]string{os.Args[0], "-test.run=^TestHostTrackingHelper$"}, env, "", workspace)(ctx)
|
||||
}()
|
||||
var helpers []windows.Handle
|
||||
for range 2 {
|
||||
connection, err := listener.Accept()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = connection.Close() })
|
||||
require.NoError(t, connection.SetReadDeadline(time.Now().Add(20*time.Second)))
|
||||
var pid int
|
||||
_, err = fmt.Fscanln(connection, &pid)
|
||||
require.NoError(t, err)
|
||||
handle, err := windows.OpenProcess(windows.SYNCHRONIZE|windows.PROCESS_TERMINATE, false, uint32(pid))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = windows.TerminateProcess(handle, 1)
|
||||
_, _ = windows.WaitForSingleObject(handle, 5000)
|
||||
_ = windows.CloseHandle(handle)
|
||||
})
|
||||
helpers = append(helpers, handle)
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
require.NoError(t, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
require.NoError(t, environment.Remove()(ctx))
|
||||
for _, handle := range helpers {
|
||||
if tracking == "default" {
|
||||
result, err := windows.WaitForSingleObject(handle, 5000)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, windows.WAIT_OBJECT_0, result)
|
||||
} else {
|
||||
result, err := windows.WaitForSingleObject(handle, 0)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, windows.WAIT_TIMEOUT, result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostTrackingHelper(t *testing.T) {
|
||||
mode := os.Getenv("GITEA_TRACKING_HELPER")
|
||||
if mode == "" {
|
||||
return
|
||||
}
|
||||
if mode != "descendant" {
|
||||
child := exec.Command(os.Args[0], "-test.run=^TestHostTrackingHelper$")
|
||||
child.Env = append(os.Environ(), "GITEA_TRACKING_HELPER="+map[string]string{"step": "helper", "helper": "descendant"}[mode])
|
||||
require.NoError(t, child.Start())
|
||||
if mode == "step" {
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
connection, err := net.Dial("tcp4", os.Getenv("GITEA_TRACKING_ADDRESS"))
|
||||
require.NoError(t, err)
|
||||
_, err = fmt.Fprintln(connection, os.Getpid())
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(io.Discard, connection)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, connection.Close())
|
||||
os.Exit(0)
|
||||
}
|
||||
+69
-10
@@ -10,10 +10,12 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -24,6 +26,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/spf13/pflag"
|
||||
@@ -37,6 +40,7 @@ type KubernetesOptions struct {
|
||||
MaxLifetime time.Duration
|
||||
ForcePull bool
|
||||
RunnerUUID string
|
||||
Transfer *KubernetesTransfer
|
||||
}
|
||||
|
||||
// KubernetesPod runs a job container and its service containers as one pod.
|
||||
@@ -47,6 +51,10 @@ type KubernetesPod struct {
|
||||
job *kubernetesContainer
|
||||
services []*kubernetesContainer
|
||||
arch, path string
|
||||
ips []net.IP
|
||||
transfer *transferServer
|
||||
transferImage string
|
||||
transferHelper string // empty when copies go through the API server
|
||||
}
|
||||
|
||||
type kubernetesContainer struct {
|
||||
@@ -76,6 +84,7 @@ type podStatus struct {
|
||||
Spec struct{ Containers []struct{} }
|
||||
Status struct {
|
||||
Phase, Reason, Message string
|
||||
PodIPs []struct{ IP string }
|
||||
Conditions []struct{ Type, Status, Message string }
|
||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||
}
|
||||
@@ -131,14 +140,24 @@ func (p *KubernetesPod) create(ctx context.Context) error {
|
||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||
}
|
||||
metadata := map[string]any{"name": name, "labels": labels}
|
||||
var runnerImage string
|
||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||
var runner struct{ Metadata struct{ Name, UID string } }
|
||||
var runner struct {
|
||||
Metadata struct{ Name, UID string }
|
||||
Spec struct{ Containers []struct{ Image string } }
|
||||
}
|
||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||
if len(runner.Spec.Containers) == 1 {
|
||||
runnerImage = runner.Spec.Containers[0].Image
|
||||
}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||
}
|
||||
}
|
||||
if err := p.setupTransfer(runnerImage); err != nil {
|
||||
return err
|
||||
}
|
||||
secretData := map[string]string{}
|
||||
manifest, err := p.manifest(name, secretData, metadata)
|
||||
if err != nil {
|
||||
@@ -207,6 +226,18 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
||||
}
|
||||
}
|
||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||
if p.transferHelper != "" {
|
||||
transfer := map[string]any{
|
||||
"name": "gitea-runner-transfer", "image": p.transferImage, "terminationMessagePolicy": "FallbackToLogsOnError",
|
||||
"command": []string{"gitea-runner", KubernetesTransferCommand, "install", p.transferHelper},
|
||||
"volumeMounts": []any{map[string]any{"name": "act", "mountPath": p.job.GetActPath()}},
|
||||
"securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}},
|
||||
}
|
||||
if p.options.ForcePull {
|
||||
transfer["imagePullPolicy"] = "Always"
|
||||
}
|
||||
spec["initContainers"] = []any{transfer}
|
||||
}
|
||||
if len(hostnames) > 0 {
|
||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||
}
|
||||
@@ -343,7 +374,13 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if pod.Status.Phase == "Failed" {
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
||||
reason := cmp.Or(pod.Status.Message, pod.Status.Reason)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
if terminated := status.State.Terminated; reason == "" && terminated != nil && terminated.ExitCode != 0 {
|
||||
reason = fmt.Sprintf("container %s exited with %d: %s", status.Name, terminated.ExitCode, strings.TrimSpace(terminated.Message))
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, reason)
|
||||
}
|
||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
@@ -356,6 +393,9 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
if started {
|
||||
for _, ip := range pod.Status.PodIPs {
|
||||
p.ips = append(p.ips, net.ParseIP(ip.IP))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
@@ -457,22 +497,35 @@ func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, w
|
||||
}
|
||||
}
|
||||
|
||||
// compressCopyMin is the size from which Copy compresses: below it gzip saves next to nothing on the wire and costs a process in the pod.
|
||||
// compressCopyMin is the size from which Copy compresses and uses the transfer helper: below it that saves next to nothing on the wire and costs a process in the pod.
|
||||
const compressCopyMin = 64 << 10
|
||||
|
||||
// extract streams the tar write produces into destPath over exec, which goes through the API server.
|
||||
// With compress it is gzipped on the way, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||
// extract streams the tar write produces into destPath, gzipped with compress, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, compress bool, write func(io.Writer) error) error {
|
||||
if compress && c.pod.transferHelper != "" {
|
||||
request, done := c.pod.transfer.offer(c.pod.ips, write)
|
||||
stdin, _ := json.Marshal(request)
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{c.pod.transferHelper, KubernetesTransferCommand, "fetch", destPath}, bytes.NewReader(stdin), io.Discard, &stderr)
|
||||
taken, writeErr := done()
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
if taken || !c.pod.options.Transfer.Fallback {
|
||||
return fmt.Errorf("copy to %s over the pod network: %w: %s", destPath, errors.Join(writeErr, err), strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
c.pod.transferHelper = "" // a pod that cannot reach the runner once will not later
|
||||
logging.Warnf("Copying into pod %s over the pod network failed, copying through the API server instead: %v: %s", c.pod.name, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
compress = compress && c.gzip
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
go func() {
|
||||
if !compress {
|
||||
if compress {
|
||||
writer.CloseWithError(writeGzip(writer, write))
|
||||
} else {
|
||||
writer.CloseWithError(write(writer))
|
||||
return
|
||||
}
|
||||
compressed, _ := gzip.NewWriterLevel(writer, gzip.BestSpeed) // fast enough to not slow copies down, nearly as small as the default level
|
||||
writer.CloseWithError(errors.Join(write(compressed), compressed.Close()))
|
||||
}()
|
||||
script := `mkdir -p "$1" && tar -xf - -C "$1"`
|
||||
if compress {
|
||||
@@ -485,6 +538,12 @@ func (c *kubernetesContainer) extract(ctx context.Context, destPath string, comp
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeGzip gzips what write produces into w, at a level fast enough to not slow copies down and nearly as small as the default.
|
||||
func writeGzip(w io.Writer, write func(io.Writer) error) error {
|
||||
compressed, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||
return errors.Join(write(compressed), compressed.Close())
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
size := 0
|
||||
@@ -498,7 +557,7 @@ func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0, hostOS == "windows")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,6 +403,13 @@ func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||
failed, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Failed","initContainerStatuses":[
|
||||
{"name":"gitea-runner-transfer","state":{"terminated":{"exitCode":1,"message":"exec format error\n"}}}]}}`)
|
||||
pod, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: failed})
|
||||
require.NoError(t, err)
|
||||
job = pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "container gitea-runner-transfer exited with 1: exec format error")
|
||||
|
||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
// KubernetesTransfer has job pods fetch large copies from the runner over the pod network instead of through the API server.
|
||||
type KubernetesTransfer struct {
|
||||
Host string // empty uses the outbound IP
|
||||
Port uint16 // 0 picks a free port
|
||||
Image string // has gitea-runner on its PATH, for an init container to copy into each job pod
|
||||
Fallback bool // copy through the API server when a job pod cannot reach the runner, else fail
|
||||
}
|
||||
|
||||
// KubernetesTransferCommand is the hidden runner subcommand job pods run the transfer helper with.
|
||||
const KubernetesTransferCommand = "kubernetes-transfer"
|
||||
|
||||
// transferRequest goes to the helper on the exec stream's stdin, keeping the token out of the command line and the audit log.
|
||||
type transferRequest struct {
|
||||
URL string `json:"url"`
|
||||
Pin string `json:"pin"` // hex SHA-256 of the runner's certificate
|
||||
}
|
||||
|
||||
type transferServer struct {
|
||||
base string
|
||||
pin string
|
||||
mu sync.Mutex
|
||||
pending map[string]transferOffer
|
||||
}
|
||||
|
||||
type transferOffer struct {
|
||||
clients []net.IP
|
||||
write func(io.Writer) error
|
||||
result chan error
|
||||
}
|
||||
|
||||
var (
|
||||
transferServersMu sync.Mutex
|
||||
transferServers = map[string]*transferServer{}
|
||||
)
|
||||
|
||||
// startTransferServer returns the runner's server for these settings, started on first use.
|
||||
func startTransferServer(host string, port uint16) (*transferServer, error) {
|
||||
transferServersMu.Lock()
|
||||
defer transferServersMu.Unlock()
|
||||
configured := net.JoinHostPort(host, strconv.Itoa(int(port)))
|
||||
if server, ok := transferServers[configured]; ok {
|
||||
return server, nil
|
||||
}
|
||||
if host == "" {
|
||||
ip := common.GetOutboundIP()
|
||||
if ip == nil {
|
||||
return nil, errors.New("no outbound IP for job pods to reach the runner at, set kubernetes.transfer.host")
|
||||
}
|
||||
host = ip.String()
|
||||
}
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(1)} // trusted by its pin alone, so no CA, names or validity
|
||||
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port))))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen for job pod transfers: %w", err)
|
||||
}
|
||||
pin := sha256.Sum256(certificate)
|
||||
server := &transferServer{base: "https://" + listener.Addr().String() + "/", pin: hex.EncodeToString(pin[:]), pending: map[string]transferOffer{}}
|
||||
httpServer := &http.Server{
|
||||
Handler: server,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{certificate}, PrivateKey: privateKey}}, MinVersion: tls.VersionTLS13},
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
IdleTimeout: 30 * time.Second,
|
||||
}
|
||||
go func() { _ = httpServer.ServeTLS(listener, "", "") }()
|
||||
transferServers[configured] = server
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// offer serves write once to one of clients until done, which reports whether a client took it and what write failed with.
|
||||
func (s *transferServer) offer(clients []net.IP, write func(io.Writer) error) (request transferRequest, done func() (taken bool, err error)) {
|
||||
token := rand.Text()
|
||||
result := make(chan error, 1)
|
||||
s.mu.Lock()
|
||||
s.pending[token] = transferOffer{clients: clients, write: write, result: result}
|
||||
s.mu.Unlock()
|
||||
return transferRequest{URL: s.base + token, Pin: s.pin}, func() (bool, error) {
|
||||
s.mu.Lock()
|
||||
_, pending := s.pending[token]
|
||||
delete(s.pending, token)
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case err := <-result:
|
||||
return true, err
|
||||
default:
|
||||
return !pending, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *transferServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
token := strings.TrimPrefix(r.URL.Path, "/")
|
||||
s.mu.Lock()
|
||||
offer, ok := s.pending[token]
|
||||
ok = ok && r.Method == http.MethodGet && slices.ContainsFunc(offer.clients, net.ParseIP(host).Equal)
|
||||
if ok {
|
||||
delete(s.pending, token)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
err := writeGzip(w, offer.write)
|
||||
offer.result <- err
|
||||
if err != nil {
|
||||
panic(http.ErrAbortHandler) // a cut-off body fails the fetch, a short tar might not
|
||||
}
|
||||
}
|
||||
|
||||
// FetchKubernetesTransfer reads a transferRequest from request and extracts the tar it points to into dest.
|
||||
func FetchKubernetesTransfer(ctx context.Context, request io.Reader, dest string) error {
|
||||
var transfer transferRequest
|
||||
if err := json.NewDecoder(request).Decode(&transfer); err != nil {
|
||||
return fmt.Errorf("read transfer request: %w", err)
|
||||
}
|
||||
client := &http.Client{Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: time.Minute,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: true, // VerifyConnection checks the pin instead
|
||||
VerifyConnection: func(state tls.ConnectionState) error {
|
||||
if sum := sha256.Sum256(state.PeerCertificates[0].Raw); hex.EncodeToString(sum[:]) != transfer.Pin {
|
||||
return errors.New("runner certificate does not match the pin")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}}
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, transfer.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Do(httpRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("fetch: %s", response.Status)
|
||||
}
|
||||
if err := os.MkdirAll(dest, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", dest)
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = response.Body, os.Stderr, os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(io.Discard, response.Body) // tar stops at the first zero blocks, a cut-off body after them still fails
|
||||
return err
|
||||
}
|
||||
|
||||
// InstallKubernetesTransferHelper copies the running binary to target.
|
||||
func InstallKubernetesTransferHelper(target string) error {
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := os.Open(executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer source.Close()
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(target) // a file left in a reused act volume would keep its mode
|
||||
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, source)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
// setupTransfer turns on transfers for the pod, with the runner pod's image unless one is set.
|
||||
func (p *KubernetesPod) setupTransfer(runnerImage string) error {
|
||||
settings := p.options.Transfer
|
||||
if settings == nil {
|
||||
return nil
|
||||
}
|
||||
image := cmp.Or(settings.Image, runnerImage)
|
||||
var err error
|
||||
var server *transferServer
|
||||
if image == "" {
|
||||
err = errors.New("set kubernetes.transfer.image, the runner is no single-container pod in kubernetes.namespace to take it from")
|
||||
} else {
|
||||
server, err = startTransferServer(settings.Host, settings.Port)
|
||||
}
|
||||
if err != nil {
|
||||
if !settings.Fallback {
|
||||
return fmt.Errorf("kubernetes.transfer: %w", err)
|
||||
}
|
||||
logging.Warnf("Job pod copies go through the API server: %v", err)
|
||||
return nil
|
||||
}
|
||||
p.transfer, p.transferImage = server, image
|
||||
p.transferHelper = path.Join(p.job.GetActPath(), "bin", "gitea-runner")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const transferHelperEnv = "GITEA_RUNNER_TEST_TRANSFER_HELPER"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv(transferHelperEnv) == "1" && len(os.Args) == 4 && os.Args[1] == KubernetesTransferCommand && os.Args[2] == "fetch" {
|
||||
if err := FetchKubernetesTransfer(context.Background(), os.Stdin, os.Args[3]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fetch(t *testing.T, request transferRequest) error {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(request)
|
||||
require.NoError(t, err)
|
||||
return FetchKubernetesTransfer(t.Context(), bytes.NewReader(body), t.TempDir())
|
||||
}
|
||||
|
||||
func TestTransferServer(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("extracts with tar")
|
||||
}
|
||||
server, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
again, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, server, again)
|
||||
loopback := []net.IP{net.ParseIP("127.0.0.1")}
|
||||
nothing := func(io.Writer) error { return nil }
|
||||
|
||||
t.Run("serves an offer gzipped once to an IP of its job pod", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("fd00::1"), loopback[0]}, nothing)
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableCompression: true}}
|
||||
response, err := client.Get(request.URL)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, response.Body.Close())
|
||||
assert.Equal(t, "gzip", response.Header.Get("Content-Encoding"))
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.True(t, taken)
|
||||
})
|
||||
|
||||
t.Run("another pod can neither take nor block an offer", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("10.1.2.3")}, nothing)
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
|
||||
t.Run("a failed write fails the fetch even past the end blocks tar stops at, and done reports it", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, func(w io.Writer) error {
|
||||
_, _ = w.Write(make([]byte, 1024))
|
||||
_, _ = io.CopyN(w, rand.Reader, 1<<20)
|
||||
return errors.New("source gone")
|
||||
})
|
||||
require.Error(t, fetch(t, request))
|
||||
taken, err := done()
|
||||
assert.True(t, taken)
|
||||
require.EqualError(t, err, "source gone")
|
||||
})
|
||||
|
||||
t.Run("the helper refuses a runner whose certificate does not match its pin", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, nothing)
|
||||
request.Pin = strings.Repeat("0", 64)
|
||||
require.ErrorContains(t, fetch(t, request), "does not match the pin")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKubernetesPodTransferImage(t *testing.T) {
|
||||
newPod := func(image string, fallback bool) *KubernetesPod {
|
||||
pod := &KubernetesPod{options: KubernetesOptions{Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: image, Fallback: fallback}}}
|
||||
pod.JobContainer(&NewContainerInput{})
|
||||
return pod
|
||||
}
|
||||
|
||||
pod := newPod("", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "gitea/runner:latest", pod.transferImage)
|
||||
|
||||
pod = newPod("registry.example/runner:1", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "registry.example/runner:1", pod.transferImage)
|
||||
|
||||
pod = newPod("", true)
|
||||
require.NoError(t, pod.setupTransfer(""))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
|
||||
pod = newPod("", false)
|
||||
require.ErrorContains(t, pod.setupTransfer(""), "kubernetes.transfer.image")
|
||||
}
|
||||
|
||||
func TestKubernetesPodCopiesLargeFilesOverPodNetwork(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","podIPs":[{"ip":"127.0.0.1"}],"containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}}]}}`, transferHelperEnv+"=1")
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, ForcePull: true, Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: "gitea/runner:latest", Fallback: true}})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Name: "job", Image: "node", WorkingDir: workdir, Stdout: io.Discard, Stderr: io.Discard})
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
|
||||
var manifest struct {
|
||||
Spec struct {
|
||||
InitContainers []struct {
|
||||
Image, ImagePullPolicy string
|
||||
Command []string
|
||||
VolumeMounts []struct{ Name, MountPath string }
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||
require.Len(t, manifest.Spec.InitContainers, 1)
|
||||
initContainer := manifest.Spec.InitContainers[0]
|
||||
assert.Equal(t, "gitea/runner:latest", initContainer.Image)
|
||||
assert.Equal(t, "Always", initContainer.ImagePullPolicy)
|
||||
assert.Equal(t, []string{"gitea-runner", KubernetesTransferCommand, "install", "/var/run/act/bin/gitea-runner"}, initContainer.Command)
|
||||
assert.Equal(t, []struct{ Name, MountPath string }{{"act", "/var/run/act"}}, initContainer.VolumeMounts)
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
|
||||
helper := filepath.Join(t.TempDir(), "bin", "gitea-runner")
|
||||
require.NoError(t, InstallKubernetesTransferHelper(helper))
|
||||
pod.transferHelper = helper
|
||||
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||
large := &FileEntry{Name: "large", Mode: 0o644, Body: strings.Repeat("a", compressCopyMin)}
|
||||
copyLarge := func(dir string) error { return job.Copy(filepath.Join(workdir, dir)+"/", large)(ctx) }
|
||||
require.NoError(t, copyLarge("transferred"))
|
||||
content, err := os.ReadFile(filepath.Join(workdir, "transferred", "large"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large.Body, string(content))
|
||||
assert.Contains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
assert.NotContains(t, lastExec(), "127.0.0.1")
|
||||
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "event.json", Mode: 0o644, Body: `{}`})(ctx))
|
||||
assert.NotContains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "file"), nil, 0o644))
|
||||
require.ErrorContains(t, copyLarge("file"), "over the pod network")
|
||||
assert.Equal(t, helper, pod.transferHelper)
|
||||
|
||||
unreachable := filepath.Join(workdir, "unreachable")
|
||||
require.NoError(t, os.WriteFile(unreachable, []byte("#!/bin/sh\necho 'connection refused' >&2\nexit 1\n"), 0o755))
|
||||
pod.transferHelper = unreachable
|
||||
pod.options.Transfer.Fallback = false
|
||||
require.ErrorContains(t, copyLarge("direct"), "connection refused")
|
||||
assert.NoFileExists(t, filepath.Join(workdir, "direct", "large"))
|
||||
|
||||
pod.options.Transfer.Fallback = true
|
||||
require.NoError(t, copyLarge("auto"))
|
||||
assert.FileExists(t, filepath.Join(workdir, "auto", "large"))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
}
|
||||
@@ -21,7 +21,12 @@ import (
|
||||
|
||||
// parseEnvFile reads a GITHUB_ENV style file, splitting lines like GitHub's runner on Windows or elsewhere.
|
||||
func parseEnvFile(e Container, srcPath string, env *map[string]string, windows bool) common.Executor {
|
||||
localEnv := *env
|
||||
return ParseEnvFile(e, srcPath, func(name, value string) {
|
||||
(*env)[name] = value
|
||||
}, windows)
|
||||
}
|
||||
|
||||
func ParseEnvFile(e Container, srcPath string, setEnv func(name, value string), windows bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
||||
if err != nil {
|
||||
@@ -64,7 +69,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
|
||||
multiLineEnv := strings.Index(line, "<<")
|
||||
switch {
|
||||
case singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv):
|
||||
localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:]
|
||||
setEnv(line[:singleLineEnv], line[singleLineEnv+1:])
|
||||
case multiLineEnv != -1:
|
||||
var multiLineEnvContent []string
|
||||
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
||||
@@ -86,7 +91,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
|
||||
if !delimiterFound {
|
||||
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
||||
}
|
||||
localEnv[line[:multiLineEnv]] = trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows)
|
||||
setEnv(line[:multiLineEnv], trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows))
|
||||
default:
|
||||
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ func TestParseEnvFile(t *testing.T) {
|
||||
want map[string]string
|
||||
}{
|
||||
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
{"repeated names", "FOO=first\nfoo=middle\nFOO=last\n", map[string]string{"FOO": "last", "foo": "middle"}},
|
||||
{"single line CRLF and final bare CR", "FOO=bar\r\nBAZ=qux\r", map[string]string{"FOO": "bar\r", "BAZ": "qux\r"}},
|
||||
{"single line without final newline", "FOO=bar", map[string]string{"FOO": "bar"}},
|
||||
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
@@ -57,6 +58,17 @@ func TestParseEnvFile(t *testing.T) {
|
||||
assert.Equal(t, test.want, env)
|
||||
})
|
||||
}
|
||||
t.Run("ordered entries preserve repeated names", func(t *testing.T) {
|
||||
hostEnv, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=first\r\nfoo<<EOF\r\nmiddle\r\nEOF\r\nFOO=last\r\n"), 0o600))
|
||||
var entries [][2]string
|
||||
require.NoError(t, ParseEnvFile(hostEnv, envPath, func(name, value string) {
|
||||
entries = append(entries, [2]string{name, value})
|
||||
}, true)(t.Context()))
|
||||
assert.Equal(t, [][2]string{
|
||||
{"FOO", "first"}, {"foo", "middle"}, {"FOO", "last"},
|
||||
}, entries)
|
||||
})
|
||||
t.Run("Windows line endings", func(t *testing.T) {
|
||||
hostEnv, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||
@@ -86,13 +98,13 @@ func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("APPLIED=before-error\n"+content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "delimiter")
|
||||
assert.Empty(t, env)
|
||||
assert.Equal(t, map[string]string{"APPLIED": "before-error"}, env)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*Fil
|
||||
}
|
||||
|
||||
// writeDirTar prefixes entries with dstPath for extraction at root.
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int, executable bool) error {
|
||||
logger := common.Logger(ctx)
|
||||
tw := tar.NewWriter(w)
|
||||
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
||||
@@ -47,7 +47,7 @@ func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useG
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
UseGitIgnore: useGitIgnore,
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:], Executable: executable},
|
||||
}).CollectFiles(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -27,22 +27,25 @@ type Handler interface {
|
||||
}
|
||||
|
||||
type TarCollector struct {
|
||||
TarWriter *tar.Writer
|
||||
UID int
|
||||
GID int
|
||||
DstDir string
|
||||
TarWriter *tar.Writer
|
||||
UID int
|
||||
GID int
|
||||
DstDir string
|
||||
Executable bool // Windows has no exec bit, so mark everything executable like docker build
|
||||
}
|
||||
|
||||
func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error {
|
||||
// create a new dir/file header
|
||||
header, err := tar.FileInfoHeader(fi, linkName)
|
||||
header, err := tar.FileInfoHeader(fi, filepath.ToSlash(linkName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update the name to correctly reflect the desired destination when untaring
|
||||
header.Name = path.Join(tc.DstDir, fpath)
|
||||
header.Mode = int64(fi.Mode())
|
||||
if tc.Executable {
|
||||
header.Mode = header.Mode&0o755 | 0o111
|
||||
}
|
||||
header.Uid = tc.UID
|
||||
header.Gid = tc.GID
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
}
|
||||
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Chmod(filepath.Join(repoDir, "renamed"), 0o777|os.ModeSticky))
|
||||
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
||||
expected["test.env"] = ".env"
|
||||
}
|
||||
@@ -69,6 +70,9 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
for header, err := tr.Next(); err != io.EOF; header, err = tr.Next() {
|
||||
require.NoError(t, err)
|
||||
files[header.Name] = header.Linkname
|
||||
if header.Name == "renamed" && runtime.GOOS != "windows" {
|
||||
assert.Equal(t, int64(0o1777), header.Mode)
|
||||
}
|
||||
}
|
||||
assert.Contains(t, files, ".git/HEAD")
|
||||
maps.DeleteFunc(files, func(name, _ string) bool { return strings.Contains("/"+name, "/.git/") })
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func lookPathWindows(file string, env map[string]string, lookup func(string, map[string]string) (string, error)) (string, error) {
|
||||
found, err := lookup(file, env)
|
||||
isBash := strings.EqualFold(file, "bash") || strings.EqualFold(file, "bash.exe")
|
||||
if !isBash || err == nil && !strings.EqualFold(filepath.Base(filepath.Dir(found)), "System32") { // System32's bash is the WSL launcher, Git's bash replaces it
|
||||
return found, err
|
||||
}
|
||||
if git, gitErr := lookup("git.exe", env); gitErr == nil {
|
||||
for _, relative := range []string{"../bin/bash.exe", "../usr/bin/bash.exe", "bash.exe"} {
|
||||
if bash, bashErr := lookup(filepath.Join(filepath.Dir(git), relative), env); bashErr == nil {
|
||||
return bash, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLookPathWindowsFallsBackToGitBash(t *testing.T) {
|
||||
root := "C:" + string(filepath.Separator)
|
||||
system32 := filepath.Join(root, "Windows", "System32")
|
||||
gitCmd, gitBin := filepath.Join(root, "Git", "cmd"), filepath.Join(root, "Git", "bin")
|
||||
msys := filepath.Join(root, "msys64", "usr", "bin")
|
||||
for _, test := range []struct {
|
||||
name, command, want string
|
||||
path []string
|
||||
}{
|
||||
{"wsl launcher first", "bash", filepath.Join(gitBin, "bash.exe"), []string{system32, gitCmd}},
|
||||
{"no bash on path", "BASH.EXE", filepath.Join(gitBin, "bash.exe"), []string{gitCmd}},
|
||||
{"own bash first", "bash", filepath.Join(msys, "bash.exe"), []string{msys, system32, gitCmd}},
|
||||
{"no git", "bash", filepath.Join(system32, "bash.exe"), []string{system32}},
|
||||
{"other command", "sh", filepath.Join(system32, "sh.exe"), []string{system32, gitCmd}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
files := map[string]bool{
|
||||
filepath.Join(system32, "bash.exe"): true, filepath.Join(system32, "sh.exe"): true,
|
||||
filepath.Join(gitCmd, "git.exe"): true, filepath.Join(gitBin, "bash.exe"): true, filepath.Join(msys, "bash.exe"): true,
|
||||
}
|
||||
lookup := func(file string, env map[string]string) (string, error) {
|
||||
candidates := []string{file}
|
||||
if !strings.ContainsAny(file, `:\/`) {
|
||||
candidates = nil
|
||||
for directory := range strings.SplitSeq(env["PATH"], ";") {
|
||||
candidates = append(candidates, filepath.Join(directory, strings.TrimSuffix(strings.ToLower(file), ".exe")+".exe"))
|
||||
}
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if files[filepath.Clean(candidate)] {
|
||||
return filepath.Clean(candidate), nil
|
||||
}
|
||||
}
|
||||
return "", fs.ErrNotExist
|
||||
}
|
||||
got, err := lookPathWindows(test.command, map[string]string{"PATH": strings.Join(test.path, ";")}, lookup)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,10 @@ func findExecutable(file string, exts []string) (string, error) {
|
||||
// a suitable candidate.
|
||||
// The result may be an absolute path or a path relative to the current directory.
|
||||
func LookPath2(file string, env map[string]string) (string, error) {
|
||||
return lookPathWindows(file, env, lookPath)
|
||||
}
|
||||
|
||||
func lookPath(file string, env map[string]string) (string, error) {
|
||||
var exts []string
|
||||
x := getenv(env, `PATHEXT`)
|
||||
if x != "" {
|
||||
@@ -76,9 +80,6 @@ func LookPath2(file string, env map[string]string) (string, error) {
|
||||
return "", &Error{file, err}
|
||||
}
|
||||
}
|
||||
if f, err := findExecutable(filepath.Join(".", file), exts); err == nil {
|
||||
return f, nil
|
||||
}
|
||||
path := getenv(env, "path")
|
||||
for _, dir := range filepath.SplitList(path) {
|
||||
if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLookPath2IgnoresTheCurrentDirectory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "tool.exe"), nil, 0o755))
|
||||
t.Chdir(dir)
|
||||
_, err := LookPath2("tool", map[string]string{"PATH": t.TempDir()})
|
||||
require.EqualError(t, err, ErrNotFound.Error())
|
||||
}
|
||||
+38
-51
@@ -180,9 +180,9 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
|
||||
action := step.getActionModel()
|
||||
|
||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *step.getEnv())
|
||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *step.getEnv())
|
||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
warnUnexpectedInputs(ctx, action, stepModel.With)
|
||||
@@ -198,7 +198,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil {
|
||||
return err
|
||||
}
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Main))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Main))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
@@ -226,7 +226,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
execFileName := action.Runs.Main + ".out"
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Main}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return fmt.Errorf("the runs.using key must be one of: %v, got %s", []string{
|
||||
@@ -283,7 +283,7 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
||||
}
|
||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||
if !ok {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
image = dockerActionImageTag(getStepGithubContext(ctx, step).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||
job.dockerActionImages[step.getStepModel()] = image
|
||||
}
|
||||
@@ -295,6 +295,9 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
||||
func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, basedir string, localAction bool, stage stepStage) error {
|
||||
logger := common.Logger(ctx)
|
||||
rc := step.getRunContext()
|
||||
if rc.isWindowsHost() {
|
||||
return errWindowsHostContainer
|
||||
}
|
||||
action := step.getActionModel()
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
@@ -336,34 +339,30 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
|
||||
if !correctArchExists || rc.Config.ForceRebuild {
|
||||
logger.Debugf("image '%s' for architecture '%s' will be built from context '%s", image, rc.Config.ContainerArchitecture, contextDir)
|
||||
var buildContext io.ReadCloser
|
||||
if localAction {
|
||||
buildContext, err = rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
||||
if err != nil {
|
||||
return err
|
||||
prepImage = common.Executor(func(ctx context.Context) error {
|
||||
input := container.NewDockerBuildExecutorInput{
|
||||
ContextDir: contextDir,
|
||||
Dockerfile: fileName,
|
||||
ImageTag: image,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
BuildArgs: rc.proxyBuildArgs(),
|
||||
}
|
||||
defer buildContext.Close()
|
||||
}
|
||||
prepImage = ContainerNewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
|
||||
ContextDir: contextDir,
|
||||
Dockerfile: fileName,
|
||||
ImageTag: image,
|
||||
BuildContext: buildContext,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
BuildArgs: rc.proxyBuildArgs(),
|
||||
})
|
||||
if buildContext == nil {
|
||||
// Held across the whole build: the daemon drains contextDir lazily.
|
||||
inner := prepImage
|
||||
prepImage = func(ctx context.Context) error {
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir)
|
||||
if localAction {
|
||||
buildContext, err := rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer buildContext.Close()
|
||||
input.BuildContext = buildContext
|
||||
} else {
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir) // held across the whole build, the daemon drains contextDir lazily
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
return inner(ctx)
|
||||
}
|
||||
}
|
||||
return ContainerNewDockerBuildExecutor(input)(ctx)
|
||||
}).Retry(3)
|
||||
} else {
|
||||
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
||||
}
|
||||
@@ -437,7 +436,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
for key := range step.getStepModel().With {
|
||||
providedInputs[strings.ToLower(key)] = true
|
||||
}
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
eval := rc.NewStepExpressionEvaluator(ctx, step)
|
||||
for k, input := range action.Inputs {
|
||||
if providedInputs[strings.ToLower(k)] {
|
||||
continue
|
||||
@@ -456,6 +455,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
}
|
||||
}
|
||||
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
||||
aliasGiteaEnv(*step.getEnv())
|
||||
return ee, nil
|
||||
}
|
||||
|
||||
@@ -509,8 +509,9 @@ func inputEnvKey(inputID string) string {
|
||||
return "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
}
|
||||
|
||||
func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *model.Action, rc *RunContext) error {
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
func populateEnvsFromInput(ctx context.Context, step step, action *model.Action) error {
|
||||
env := step.getEnv()
|
||||
eval := step.getRunContext().NewStepExpressionEvaluator(ctx, step)
|
||||
for inputID, input := range action.Inputs {
|
||||
envKey := inputEnvKey(inputID)
|
||||
if _, ok := (*env)[envKey]; !ok {
|
||||
@@ -523,22 +524,8 @@ func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||
func lookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||
if value, ok := values[name]; ok || !caseInsensitive {
|
||||
return value, ok
|
||||
}
|
||||
for key, value := range values {
|
||||
if strings.EqualFold(key, name) {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
var zero V
|
||||
return zero, false
|
||||
}
|
||||
|
||||
func stepInput(step *model.Step, name string) string {
|
||||
value, _ := lookupFold(step.With, name, true)
|
||||
value, _ := common.LookupFold(step.With, name, true)
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -546,7 +533,7 @@ func warnUnexpectedInputs(ctx context.Context, action *model.Action, inputs map[
|
||||
docker := action.Runs.Using.IsDocker()
|
||||
var unexpectedInputs []string
|
||||
for input := range inputs {
|
||||
_, declared := lookupFold(action.Inputs, input, true)
|
||||
_, declared := common.LookupFold(action.Inputs, input, true)
|
||||
containerInput := docker && (strings.EqualFold(input, "entryPoint") || strings.EqualFold(input, "args"))
|
||||
if !declared && !containerInput {
|
||||
unexpectedInputs = append(unexpectedInputs, input)
|
||||
@@ -652,7 +639,7 @@ func runPreStep(step actionStep) common.Executor {
|
||||
x := action.Runs.Using
|
||||
if !x.IsComposite() {
|
||||
// defaults in pre steps were missing, however provided inputs are available
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -663,7 +650,7 @@ func runPreStep(step actionStep) common.Executor {
|
||||
return err
|
||||
}
|
||||
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Pre))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Pre))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
@@ -695,7 +682,7 @@ func runPreStep(step actionStep) common.Executor {
|
||||
execFileName := action.Runs.Pre + ".out"
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Pre}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return nil
|
||||
@@ -763,7 +750,7 @@ func runPostStep(step actionStep) common.Executor {
|
||||
|
||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Post))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Post))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
@@ -792,7 +779,7 @@ func runPostStep(step actionStep) common.Executor {
|
||||
execFileName := action.Runs.Post + ".out"
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Post}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
|
||||
default:
|
||||
|
||||
@@ -57,9 +57,6 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
|
||||
env[envKey] = value
|
||||
}
|
||||
}
|
||||
gh := step.getGithubContext(ctx)
|
||||
env["GITHUB_ACTION_REPOSITORY"] = gh.ActionRepository
|
||||
env["GITHUB_ACTION_REF"] = gh.ActionRef
|
||||
|
||||
return env, nil
|
||||
}
|
||||
|
||||
@@ -19,6 +19,35 @@ import (
|
||||
)
|
||||
|
||||
func TestCompositeActionParity(t *testing.T) {
|
||||
t.Run("input defaults use the step identity and docker env keeps the GITEA aliases", func(t *testing.T) {
|
||||
for _, uses := range []string{"./action", "own/action@v2"} {
|
||||
t.Run(uses, func(t *testing.T) {
|
||||
rc := &RunContext{
|
||||
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
}
|
||||
var currentStep step = &stepActionLocal{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}}
|
||||
want := "|"
|
||||
if uses == "own/action@v2" {
|
||||
currentStep = &stepActionRemote{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}, remoteAction: mustNewRemoteAction(t, uses)}
|
||||
want = "own/action|v2"
|
||||
}
|
||||
action := &model.Action{
|
||||
Inputs: map[string]model.Input{"identity": {Default: "${{ github.action_repository }}|${{ github.action_ref }}"}},
|
||||
Runs: model.ActionRuns{Env: map[string]string{"GITHUB_WORKFLOW_SHA": "runs-sha"}},
|
||||
}
|
||||
require.NoError(t, setupEnv(t.Context(), currentStep))
|
||||
require.NoError(t, populateEnvsFromInput(t.Context(), currentStep, action))
|
||||
assert.Equal(t, want, (*currentStep.getEnv())["INPUT_IDENTITY"])
|
||||
_, err := evalDockerEnv(t.Context(), currentStep, action)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, (*currentStep.getEnv())["identity"])
|
||||
assert.Equal(t, "runs-sha", (*currentStep.getEnv())["GITEA_WORKFLOW_SHA"])
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inherits contexts without leaking inputs", func(t *testing.T) {
|
||||
hook := logtest.NewWithLevel(logging.LevelWarn)
|
||||
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||
@@ -54,7 +83,7 @@ func TestCompositeActionParity(t *testing.T) {
|
||||
assert.NotContains(t, composite.Env, "INPUT_EXTRA")
|
||||
|
||||
nestedEnv := composite.GetEnv()
|
||||
require.NoError(t, populateEnvsFromInput(ctx, &nestedEnv, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}, composite))
|
||||
require.NoError(t, populateEnvsFromInput(ctx, &stepActionLocal{RunContext: composite, env: nestedEnv}, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}))
|
||||
assert.Equal(t, "inner-default", nestedEnv["INPUT_SHARED"])
|
||||
})
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ runs:
|
||||
"cwd": {
|
||||
Description: "(Actual working directory)",
|
||||
Required: false,
|
||||
Default: "actionDir/actionPath",
|
||||
Default: filepath.Join("actionDir", "actionPath"),
|
||||
},
|
||||
"command": {
|
||||
Description: "(Actual program)",
|
||||
@@ -144,7 +144,7 @@ runs:
|
||||
}
|
||||
|
||||
writeFile := func(filename string, _data []byte, perm fs.FileMode) error {
|
||||
assert.Equal(t, "actionDir/actionPath/trampoline.js", filename)
|
||||
assert.Equal(t, filepath.Join("actionDir", "actionPath", "trampoline.js"), filename)
|
||||
assert.Equal(t, fs.FileMode(0o644), perm)
|
||||
return nil
|
||||
}
|
||||
@@ -248,6 +248,39 @@ func TestActionRunner(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
||||
for _, stage := range []string{"main", "pre", "post"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
cm := &containerMock{}
|
||||
containerActionDir := "/var/run/act/actions/" + model.UsesHash("/org/repo@ref")
|
||||
cm.On("CopyDir", containerActionDir+"/", mock.Anything, false, true).Return(noopExecutor).Maybe()
|
||||
cm.On("Exec", []string{"go", "build", "-o", stage + ".go.out", stage + ".go"}, mock.Anything, "", containerActionDir).Return(noopExecutor).Once()
|
||||
cm.On("Exec", []string{containerActionDir + "/" + stage + ".go.out"}, mock.Anything, "", "").Return(noopExecutor).Once()
|
||||
step := &stepActionRemote{
|
||||
Step: &model.Step{ID: "step", Uses: "org/repo@ref"},
|
||||
RunContext: &RunContext{
|
||||
Config: &Config{NoActionPatch: true, PresetGitHubContext: &model.GithubContext{}},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
JobContainer: cm,
|
||||
},
|
||||
action: &model.Action{Runs: model.ActionRuns{Using: model.ActionRunsUsingGo, Main: "main.go", Pre: "pre.go", Post: "post.go"}},
|
||||
actionDir: t.TempDir(),
|
||||
env: map[string]string{},
|
||||
remoteAction: mustNewRemoteAction(t, "org/repo@ref"),
|
||||
}
|
||||
switch stage {
|
||||
case "main":
|
||||
require.NoError(t, runActionImpl(step, step.actionDir, step.remoteAction)(t.Context()))
|
||||
case "pre":
|
||||
require.NoError(t, runPreStep(step)(t.Context()))
|
||||
case "post":
|
||||
require.NoError(t, runPostStep(step)(t.Context()))
|
||||
}
|
||||
cm.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeActionCommandPreservesSymlinkedEntrypoint(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlink creation requires privileges on Windows")
|
||||
|
||||
@@ -165,7 +165,7 @@ func (rc *RunContext) refuseUnsecureCommand(ctx context.Context, command string)
|
||||
}
|
||||
caseInsensitive := rc.JobContainer != nil && rc.JobContainer.IsEnvironmentCaseInsensitive()
|
||||
for _, env := range []map[string]string{rc.currentStepEnv(), rc.Env, rc.GlobalEnv} {
|
||||
if value, ok := lookupFold(env, allowUnsecureCommandsVar, caseInsensitive); ok {
|
||||
if value, ok := common.LookupFold(env, allowUnsecureCommandsVar, caseInsensitive); ok {
|
||||
if isTruthyEnv(value) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package runner
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -90,18 +89,18 @@ var hashfiles string
|
||||
|
||||
// NewStepExpressionEvaluator creates a new evaluator with the `inputs` of the enclosing workflow or composite action
|
||||
func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext())
|
||||
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||
}
|
||||
|
||||
// NewActionInputsExpressionEvaluator creates a new evaluator with the step's own with: values as `inputs`
|
||||
func (rc *RunContext) NewActionInputsExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext())
|
||||
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||
}
|
||||
|
||||
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext) *ExpressionEvaluator {
|
||||
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext, github *model.GithubContext) *ExpressionEvaluator {
|
||||
// todo: cleanup EvaluationEnvironment creation
|
||||
ee := &exprparser.EvaluationEnvironment{
|
||||
Github: step.getGithubContext(ctx),
|
||||
Github: github,
|
||||
Env: *step.getEnv(),
|
||||
Job: jobContext,
|
||||
Steps: rc.StepResults,
|
||||
@@ -112,7 +111,7 @@ func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step,
|
||||
Needs: exprparser.NeedsContext(rc.Run),
|
||||
// todo: should be unavailable
|
||||
// but required to interpolate/evaluate the inputs in actions/composite
|
||||
Inputs: getEvaluatorInputs(rc, stepInputs, rc.getGithubContext(ctx)),
|
||||
Inputs: getEvaluatorInputs(rc, stepInputs, github),
|
||||
HashFiles: getHashFilesFunction(ctx, rc),
|
||||
}
|
||||
ee.Runner = rc.getRunnerContext(ctx)
|
||||
@@ -160,7 +159,7 @@ func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Body: hashfiles,
|
||||
}).Then(rc.JobContainer.Exec([]string{"node", path.Join(rc.JobContainer.GetActPath(), name)}, env, "", ""))(timeoutCtx)
|
||||
}).Then(rc.JobContainer.Exec([]string{"node", container.JoinPath(rc.JobContainer.GetActPath(), name)}, env, "", ""))(timeoutCtx)
|
||||
quotedPatterns := expressionStringEscape(strings.Join(patterns, ", "))
|
||||
switch {
|
||||
case err == nil:
|
||||
|
||||
+42
-19
@@ -23,6 +23,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
@@ -30,6 +31,8 @@ import (
|
||||
|
||||
"gitea.dev/actionslib/pkg/exprparser"
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
unicodeencoding "golang.org/x/text/encoding/unicode"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -111,7 +114,7 @@ func printPrepareActions(rc *RunContext, preparers []actionPreparer) common.Exec
|
||||
if !ok || strings.Contains(action.Step.Uses, "${{") || action.action != nil {
|
||||
continue
|
||||
}
|
||||
github := action.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||
github := action.RunContext.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||
remote, err := newRemoteAction(action.Step.Uses, github, action.RunContext.Config.GitHubInstance)
|
||||
if err != nil || action.skipsLocalCheckout(remote, github) {
|
||||
continue
|
||||
@@ -461,7 +464,7 @@ func evaluateJobEnv(ctx context.Context, rc *RunContext) (map[string]string, err
|
||||
}
|
||||
}
|
||||
if workflowEnv != nil {
|
||||
evaluated = mergeMaps(workflowEnv, env)
|
||||
evaluated = rc.mergeEnv(workflowEnv, env)
|
||||
evaluator = rc.NewExpressionEvaluatorWithEnv(ctx, evaluated)
|
||||
}
|
||||
var err error
|
||||
@@ -569,14 +572,34 @@ func tryUploadJobSummary(ctx context.Context, rc *RunContext) {
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
// Reuse a single client across all step uploads so connections can be pooled.
|
||||
client := &http.Client{Timeout: jobSummaryUploadRequestTimeout, Transport: rc.Config.instanceTransportFor(base)}
|
||||
maskers := rc.secretMaskers()
|
||||
longestMask := 0
|
||||
for i := 0; i < len(maskers); i += 2 {
|
||||
longestMask = max(longestMask, len(maskers[i]))
|
||||
}
|
||||
masker := NewSecretReplacer(maskers)
|
||||
limit := maxJobSummaryBytes + longestMask
|
||||
for i := range rc.Run.Job().Steps {
|
||||
summaryPath := path.Join(actPath, "workflow", "step-summary-"+strconv.Itoa(i)+".md")
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, rc.JobContainer, summaryPath, maxJobSummaryBytes)
|
||||
summaryPath := container.JoinPath(actPath, "workflow", "step-summary-"+strconv.Itoa(i)+".md")
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, rc.JobContainer, summaryPath, int64(limit+1))
|
||||
if !ok || len(body) == 0 {
|
||||
continue
|
||||
}
|
||||
// Gitea renders summaries on the run page, so mask before the upload.
|
||||
uploadJobSummary(ctx, client, base+strconv.Itoa(i)+"/summary", runtimeToken, []byte(rc.maskSecrets(string(body))))
|
||||
masked := masker.Replace(string(body))
|
||||
split := len(body) > limit
|
||||
if split { // the read can end inside a secret, so keep only what masks alike without the lookahead
|
||||
unsplit := masker.Replace(string(body[:maxJobSummaryBytes]))
|
||||
kept := 0
|
||||
for kept < len(unsplit) && kept < len(masked) && unsplit[kept] == masked[kept] {
|
||||
kept++
|
||||
}
|
||||
for kept > 0 && kept < len(masked) && !utf8.RuneStart(masked[kept]) {
|
||||
kept--
|
||||
}
|
||||
masked = masked[:kept]
|
||||
}
|
||||
body = truncateJobSummary(ctx, []byte(masked), summaryPath, maxJobSummaryBytes, split)
|
||||
uploadJobSummary(ctx, client, base+strconv.Itoa(i)+"/summary", runtimeToken, body)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -690,26 +713,26 @@ func readSingleFileFromContainerArchive(ctx context.Context, env container.Execu
|
||||
if entryName != requestedPath && entryName != path.Base(requestedPath) {
|
||||
continue
|
||||
}
|
||||
// Summaries larger than the limit are truncated rather than dropped, so the
|
||||
// user still gets the leading content (mirroring how GitHub caps oversized
|
||||
// step summaries instead of discarding them). Read one extra byte so an
|
||||
// over-limit file is detected from the actual stream rather than trusting
|
||||
// header.Size, then cap the returned content at maxBytes.
|
||||
b, err := io.ReadAll(io.LimitReader(tr, maxBytes+1))
|
||||
b, err := io.ReadAll(io.LimitReader(transform.NewReader(tr, unicodeencoding.BOMOverride(unicodeencoding.UTF8.NewDecoder())), maxBytes))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if int64(len(b)) > maxBytes {
|
||||
// Reserve room for the marker so the marked-up result still fits in maxBytes.
|
||||
marker := []byte(jobSummaryTruncationMarker)
|
||||
keep := max(maxBytes-int64(len(marker)), 0)
|
||||
b = append(b[:keep], marker...)
|
||||
common.Logger(ctx).Warnf("job summary truncated: path=%s max=%d", p, maxBytes)
|
||||
}
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
|
||||
func truncateJobSummary(ctx context.Context, body []byte, summaryPath string, maxBytes int, split bool) []byte {
|
||||
if !split && len(body) <= maxBytes {
|
||||
return body
|
||||
}
|
||||
keep := max(min(len(body), maxBytes-len(jobSummaryTruncationMarker)), 0)
|
||||
for keep > 0 && keep < len(body) && !utf8.RuneStart(body[keep]) {
|
||||
keep--
|
||||
}
|
||||
common.Logger(ctx).Warnf("job summary truncated: path=%s max=%d", summaryPath, maxBytes)
|
||||
return append(body[:keep], jobSummaryTruncationMarker...)
|
||||
}
|
||||
|
||||
// useJobStageLogger tags the log entries of a job level executor with the stage they belong to.
|
||||
func useJobStageLogger(stage string, executor common.Executor) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
|
||||
+100
-63
@@ -34,6 +34,8 @@ import (
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
yaml "go.yaml.in/yaml/v4"
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
)
|
||||
|
||||
func TestJobExecutor(t *testing.T) {
|
||||
@@ -932,49 +934,42 @@ func TestExtractJobIDFromRuntimeToken(t *testing.T) {
|
||||
assert.Equal(t, int64(0), extractJobIDFromRuntimeToken(""))
|
||||
}
|
||||
|
||||
func TestReadSingleFileFromContainerArchiveFindsMatchingRegularFile(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cm := &containerMock{}
|
||||
cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(
|
||||
io.NopCloser(bytes.NewReader(tarArchive(t,
|
||||
tarEntry{name: "workflow", typeflag: tar.TypeDir},
|
||||
tarEntry{name: "other.md", body: "wrong"},
|
||||
tarEntry{name: "SUMMARY.md", body: "right"},
|
||||
))),
|
||||
nil,
|
||||
).Once()
|
||||
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, cm, "/var/run/act/workflow/SUMMARY.md", 1024)
|
||||
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, []byte("right"), body)
|
||||
cm.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestReadSingleFileFromContainerArchiveSizeLimit(t *testing.T) {
|
||||
func TestReadSingleFileFromContainerArchive(t *testing.T) {
|
||||
retainedBytes := 200 - len(jobSummaryTruncationMarker)
|
||||
for name, tt := range map[string]struct {
|
||||
content string
|
||||
maxBytes int64
|
||||
want string
|
||||
}{
|
||||
"truncates oversized summary": {strings.Repeat("a", 300), 200, strings.Repeat("a", 200-len(jobSummaryTruncationMarker)) + jobSummaryTruncationMarker},
|
||||
"keeps exact limit": {"abc", 3, "abc"},
|
||||
"truncates oversized summary without reading entire file": {strings.Repeat("a", 100000), 200, strings.Repeat("a", 200-len(jobSummaryTruncationMarker)) + jobSummaryTruncationMarker},
|
||||
"keeps exact limit": {"abc", 3, "abc"},
|
||||
"strips UTF-8 BOM before limit": {"\xef\xbb\xbfabc", 3, "abc"},
|
||||
"decodes UTF-16 before limit": {"\xff\xfea\x00b\x00c\x00", 3, "abc"},
|
||||
"limits expanded UTF-16": {"\xff\xfe" + strings.Repeat("\x2d\x4e", 150), 200, strings.Repeat("中", retainedBytes/3) + jobSummaryTruncationMarker},
|
||||
"keeps complete UTF-8 characters": {strings.Repeat("a", retainedBytes-1) + "€" + strings.Repeat("a", 200), 200, strings.Repeat("a", retainedBytes-1) + jobSummaryTruncationMarker},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
logger, hook := logtest.NewLogger()
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
cm := &containerMock{}
|
||||
archive := bytes.NewReader(tarArchive(t,
|
||||
tarEntry{name: "workflow", typeflag: tar.TypeDir},
|
||||
tarEntry{name: "other.md", body: "wrong"},
|
||||
tarEntry{name: "SUMMARY.md", body: tt.content},
|
||||
))
|
||||
cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(
|
||||
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "SUMMARY.md", body: tt.content}))),
|
||||
io.NopCloser(archive),
|
||||
nil,
|
||||
).Once()
|
||||
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, cm, "/var/run/act/workflow/SUMMARY.md", tt.maxBytes)
|
||||
body, ok := readSingleFileFromContainerArchive(ctx, cm, "/var/run/act/workflow/SUMMARY.md", tt.maxBytes+1)
|
||||
assert.LessOrEqual(t, int64(len(body)), tt.maxBytes+1)
|
||||
body = truncateJobSummary(ctx, body, "/var/run/act/workflow/SUMMARY.md", int(tt.maxBytes), false)
|
||||
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, []byte(tt.want), body)
|
||||
assert.LessOrEqual(t, int64(len(body)), tt.maxBytes)
|
||||
if int64(len(tt.content)) > tt.maxBytes {
|
||||
if strings.HasSuffix(tt.want, jobSummaryTruncationMarker) {
|
||||
if assert.Len(t, hook.AllRecords(), 1) {
|
||||
assert.Contains(t, hook.LastRecord().Message, "job summary truncated")
|
||||
}
|
||||
@@ -1047,23 +1042,25 @@ func makeTestRC(t *testing.T, jobYAML string) *RunContext {
|
||||
|
||||
func TestApplyJobTimeout(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
yaml string
|
||||
wantTimeout bool
|
||||
name string
|
||||
yaml string
|
||||
want time.Duration
|
||||
}{
|
||||
{"empty", "runs-on: ubuntu-latest", false},
|
||||
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", true},
|
||||
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", false},
|
||||
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", false},
|
||||
{"empty", "runs-on: ubuntu-latest", 0},
|
||||
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", 5 * time.Minute},
|
||||
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", 0},
|
||||
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rc := makeTestRC(t, tc.yaml)
|
||||
ctx := t.Context()
|
||||
newCtx, cancel := applyJobTimeout(ctx, rc, rc.Run.Job())
|
||||
newCtx, cancel := applyJobTimeout(t.Context(), rc, rc.Run.Job())
|
||||
defer cancel()
|
||||
_, hasDeadline := newCtx.Deadline()
|
||||
assert.Equal(t, tc.wantTimeout, hasDeadline)
|
||||
deadline, hasDeadline := newCtx.Deadline()
|
||||
assert.Equal(t, tc.want != 0, hasDeadline)
|
||||
if hasDeadline {
|
||||
assert.WithinDuration(t, time.Now().Add(tc.want), deadline, time.Minute)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1111,35 +1108,75 @@ func TestJobSetContinueOnError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTryUploadJobSummaryMasksSecrets(t *testing.T) {
|
||||
var got string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
got = string(body)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
const prefix = "deployed true with s3cr3t and runtime-added via pr0xypw 😀"
|
||||
const maskedPrefix = "deployed true with *** and *** via *** 😀"
|
||||
padding := strings.Repeat("a", maxJobSummaryBytes-len(jobSummaryTruncationMarker)-len(prefix)-2)
|
||||
content := prefix + padding + "s3cr3t" + strings.Repeat("中", 100)
|
||||
want := maskedPrefix + padding + "***" + strings.Repeat("中", (len(prefix)-len(maskedPrefix)-1)/3) + jobSummaryTruncationMarker
|
||||
longSecret := "PREFIX" + strings.Repeat("xy", 100) + "Z"
|
||||
readLimit := maxJobSummaryBytes + base64.StdEncoding.EncodedLen(len(longSecret)) + 1
|
||||
shrinkingPrefix := strings.Repeat(longSecret, (readLimit-len(longSecret)+1)/len(longSecret))
|
||||
for name, testcase := range map[string]struct {
|
||||
encoding encoding.Encoding
|
||||
content, want string
|
||||
masks []string
|
||||
}{
|
||||
"UTF-8": {unicode.UTF8, content, want, nil},
|
||||
"UTF-8 BOM": {unicode.UTF8BOM, content, want, nil},
|
||||
"UTF-16LE BOM": {unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), content, want, nil},
|
||||
"UTF-16BE BOM": {unicode.UTF16(unicode.BigEndian, unicode.UseBOM), content, want, nil},
|
||||
"secret split by the read": {unicode.UTF8, strings.Repeat("s3cr3t", maxJobSummaryBytes/3), "", nil},
|
||||
"split read keeps every masked secret before the limit": {
|
||||
unicode.UTF8,
|
||||
"# public\n" + strings.Repeat(longSecret, readLimit/len(longSecret)+1),
|
||||
"# public\n" + strings.Repeat("***", (maxJobSummaryBytes-len("# public\n"))/len(longSecret)) + jobSummaryTruncationMarker,
|
||||
[]string{longSecret},
|
||||
},
|
||||
"nested secret expands the split tail": {
|
||||
unicode.UTF8,
|
||||
shrinkingPrefix + strings.Repeat("*", readLimit-len(shrinkingPrefix)-len(longSecret)+1) + longSecret,
|
||||
"",
|
||||
[]string{"xy", longSecret},
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var got string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
got = string(body)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cm := &containerMock{}
|
||||
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
|
||||
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{
|
||||
name: "step-summary-0.md", body: "deployed true with s3cr3t and runtime-added via pr0xypw",
|
||||
}))),
|
||||
nil,
|
||||
).Once()
|
||||
content, err := testcase.encoding.NewEncoder().Bytes([]byte(testcase.content))
|
||||
require.NoError(t, err)
|
||||
cm := &containerMock{}
|
||||
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
|
||||
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{
|
||||
name: "step-summary-0.md", body: string(content),
|
||||
}))),
|
||||
nil,
|
||||
).Once()
|
||||
|
||||
rc := newJobSummaryRC(map[string]string{
|
||||
"GITEA_ACTIONS_CAPABILITIES": "job-summary",
|
||||
"ACTIONS_RUNTIME_URL": server.URL,
|
||||
"ACTIONS_RUNTIME_TOKEN": fakeRuntimeToken(34),
|
||||
"GITEA_RUN_ID": "12",
|
||||
}, cm, 1)
|
||||
rc.Config.Secrets = map[string]string{"TOK": "s3cr3t", "ACTIONS_STEP_DEBUG": "true"}
|
||||
rc.Config.ExtraMasks = []string{"pr0xypw"}
|
||||
rc.Masks = []string{"runtime-added"}
|
||||
rc := newJobSummaryRC(map[string]string{
|
||||
"GITEA_ACTIONS_CAPABILITIES": "job-summary",
|
||||
"ACTIONS_RUNTIME_URL": server.URL,
|
||||
"ACTIONS_RUNTIME_TOKEN": fakeRuntimeToken(34),
|
||||
"GITEA_RUN_ID": "12",
|
||||
}, cm, 1)
|
||||
rc.Config.Secrets = map[string]string{"TOK": "s3cr3t", "ACTIONS_STEP_DEBUG": "true"}
|
||||
rc.Config.ExtraMasks = append([]string{"pr0xypw"}, testcase.masks...)
|
||||
rc.Masks = []string{"runtime-added"}
|
||||
|
||||
tryUploadJobSummary(t.Context(), rc)
|
||||
tryUploadJobSummary(t.Context(), rc)
|
||||
|
||||
assert.Equal(t, "deployed true with *** and *** via ***", got)
|
||||
cm.AssertExpectations(t)
|
||||
if testcase.want == "" {
|
||||
assert.Equal(t, jobSummaryTruncationMarker, strings.TrimLeft(got, "*"))
|
||||
} else {
|
||||
assert.Equal(t, testcase.want, got)
|
||||
}
|
||||
cm.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
|
||||
}
|
||||
rawLogger.Infof("::endgroup::")
|
||||
|
||||
env := mergeMaps(rc.containerSpec.Env, rc.GetEnv())
|
||||
env := rc.mergeEnv(rc.caEnv, rc.containerSpec.Env, rc.GetEnv())
|
||||
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
||||
rc.ApplyExtraPath(ctx, &env)
|
||||
|
||||
@@ -82,10 +82,9 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
|
||||
// the first one wrote.
|
||||
func (rc *RunContext) setupHookFileCommands(ctx context.Context, env map[string]string) error {
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
env["GITHUB_ENV"] = path.Join(actPath, hookEnvFileCommand)
|
||||
env["GITHUB_PATH"] = path.Join(actPath, hookPathFileCommand)
|
||||
env["GITEA_ENV"] = env["GITHUB_ENV"]
|
||||
env["GITEA_PATH"] = env["GITHUB_PATH"]
|
||||
env["GITHUB_ENV"] = container.JoinPath(actPath, hookEnvFileCommand)
|
||||
env["GITHUB_PATH"] = container.JoinPath(actPath, hookPathFileCommand)
|
||||
aliasGiteaEnv(env)
|
||||
|
||||
return rc.JobContainer.Copy(actPath,
|
||||
&container.FileEntry{Name: hookEnvFileCommand, Mode: 0o666},
|
||||
@@ -95,7 +94,7 @@ func (rc *RunContext) setupHookFileCommands(ctx context.Context, env map[string]
|
||||
|
||||
func (rc *RunContext) processHookFileCommands(ctx context.Context) error {
|
||||
err := processRunnerEnvFileCommand(ctx, hookEnvFileCommand, rc, rc.setEnvFile)
|
||||
if pathErr := rc.UpdateExtraPath(ctx, path.Join(rc.JobContainer.GetActPath(), hookPathFileCommand)); pathErr != nil && err == nil {
|
||||
if pathErr := rc.UpdateExtraPath(ctx, container.JoinPath(rc.JobContainer.GetActPath(), hookPathFileCommand)); pathErr != nil && err == nil {
|
||||
err = pathErr
|
||||
}
|
||||
return err
|
||||
@@ -110,7 +109,7 @@ func hookCommand(hookPath string) (cmd []string, shell string) {
|
||||
case ".sh":
|
||||
return []string{"bash", "-e", hookPath}, "bash -e {0}"
|
||||
case ".ps1":
|
||||
return []string{"pwsh", "-command", ". '" + hookPath + "'"}, `pwsh -command ". '{0}'"`
|
||||
return []string{"pwsh", "-command", ". '" + expressionStringEscape(hookPath) + "'"}, `pwsh -command ". '{0}'"`
|
||||
default:
|
||||
return []string{hookPath}, ""
|
||||
}
|
||||
|
||||
@@ -121,6 +121,7 @@ func TestRunJobHook(t *testing.T) {
|
||||
assert.Equal(t, "job", jobContainer.env["GITHUB_JOB"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-envs.txt", jobContainer.env["GITHUB_ENV"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-path.txt", jobContainer.env["GITHUB_PATH"])
|
||||
assert.Equal(t, jobContainer.env["GITHUB_PATH"], jobContainer.env["GITEA_PATH"])
|
||||
})
|
||||
|
||||
t.Run("each hook has its own command state", func(t *testing.T) {
|
||||
@@ -245,9 +246,10 @@ func TestHookCommand(t *testing.T) {
|
||||
cmd []string
|
||||
shell string
|
||||
}{
|
||||
"/hooks/started.sh": {[]string{"bash", "-e", "/hooks/started.sh"}, "bash -e {0}"},
|
||||
"/hooks/started.PS1": {[]string{"pwsh", "-command", ". '/hooks/started.PS1'"}, `pwsh -command ". '{0}'"`},
|
||||
"/hooks/started": {[]string{"/hooks/started"}, ""},
|
||||
"/hooks/started.sh": {[]string{"bash", "-e", "/hooks/started.sh"}, "bash -e {0}"},
|
||||
"/hooks/started.PS1": {[]string{"pwsh", "-command", ". '/hooks/started.PS1'"}, `pwsh -command ". '{0}'"`},
|
||||
"/hooks/started": {[]string{"/hooks/started"}, ""},
|
||||
`C:\hooks\O'Brien\started.ps1`: {[]string{"pwsh", "-command", `. 'C:\hooks\O''Brien\started.ps1'`}, `pwsh -command ". '{0}'"`},
|
||||
} {
|
||||
cmd, shell := hookCommand(hookPath)
|
||||
assert.Equal(t, want.cmd, cmd, hookPath)
|
||||
|
||||
@@ -479,11 +479,15 @@ func (f textFormatter) isColored() bool {
|
||||
// such as a container name or a job summary. Masks added at runtime count, so a summary written
|
||||
// after ::add-mask:: is covered too.
|
||||
func (rc *RunContext) maskSecrets(value string) string {
|
||||
return NewSecretReplacer(rc.secretMaskers()).Replace(value)
|
||||
}
|
||||
|
||||
func (rc *RunContext) secretMaskers() []string {
|
||||
oldnew := rc.Config.maskers()
|
||||
for _, mask := range rc.Masks {
|
||||
oldnew = AppendSecretMasker(oldnew, mask)
|
||||
}
|
||||
return NewSecretReplacer(oldnew).Replace(value)
|
||||
return oldnew
|
||||
}
|
||||
|
||||
// maskers is every value this job's configuration says to hide, whatever the sink.
|
||||
|
||||
@@ -256,7 +256,7 @@ func TestActionScriptPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry) {
|
||||
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry, int64) {
|
||||
t.Helper()
|
||||
|
||||
cm := &containerMock{}
|
||||
@@ -274,6 +274,8 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(script), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||
info, err := os.Stat(script)
|
||||
require.NoError(t, err)
|
||||
|
||||
var calls []string
|
||||
var copied []*container.FileEntry
|
||||
@@ -291,22 +293,22 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
source, err := os.ReadFile(script)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gateTSC, string(source), "the shared copy is never written")
|
||||
return calls, copied
|
||||
return calls, copied, int64(info.Mode().Perm())
|
||||
}
|
||||
|
||||
t.Run("patched over the container's copy", func(t *testing.T) {
|
||||
calls, copied := copiedBundles(t, false)
|
||||
calls, copied, mode := copiedBundles(t, false)
|
||||
assert.Equal(t, []string{"CopyDir", "Copy"}, calls)
|
||||
require.Len(t, copied, 1)
|
||||
assert.Equal(t, "dist/cache.js", copied[0].Name)
|
||||
assert.Equal(t, int64(0o600), copied[0].Mode)
|
||||
assert.Equal(t, mode, copied[0].Mode)
|
||||
assert.True(t, gateOpened(copied[0].Body))
|
||||
})
|
||||
|
||||
// The escape hatch, for an action the edit breaks: the artifact actions refuse again, and the
|
||||
// cache client keeps to v1.
|
||||
t.Run("as shipped when the runner is told not to patch", func(t *testing.T) {
|
||||
calls, copied := copiedBundles(t, true)
|
||||
calls, copied, _ := copiedBundles(t, true)
|
||||
assert.Equal(t, []string{"CopyDir"}, calls)
|
||||
assert.Empty(t, copied)
|
||||
})
|
||||
|
||||
@@ -23,13 +23,14 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
return common.NewErrorExecutor(err)
|
||||
}
|
||||
|
||||
if !rc.Config.NoSkipCheckout {
|
||||
preset := rc.Config.PresetGitHubContext
|
||||
if preset == nil {
|
||||
// resolve the local workflow against the workspace root, not the process
|
||||
// working directory, so it is found regardless of where the runner is invoked
|
||||
return newReusableWorkflowExecutor(rc, rc.Config.Workdir, localWorkflow.Path, false)
|
||||
}
|
||||
|
||||
uses := fmt.Sprintf("%s/%s@%s", rc.Config.PresetGitHubContext.Repository, localWorkflow.Path, rc.Config.PresetGitHubContext.Sha)
|
||||
uses := fmt.Sprintf("%s/%s@%s", preset.Repository, localWorkflow.Path, preset.Sha)
|
||||
|
||||
remoteReusableWorkflow, err := newRemoteReusableWorkflow(rc.Config.GitHubInstance, uses)
|
||||
if err != nil {
|
||||
@@ -196,7 +197,7 @@ func (c Config) withInstanceAuth(input git.NewGitCloneExecutorInput, token strin
|
||||
}
|
||||
input.ClientCertFile, input.ClientKeyFile = c.ClientCertFile, c.ClientKeyFile
|
||||
if onInstanceHost(u.Host, c.GitHubInstance) {
|
||||
input.InsecureSkipTLS, input.Transport = c.InsecureSkipTLS, c.InstanceTransport
|
||||
input.InsecureSkipTLS, input.CAFile, input.Transport = c.InsecureSkipTLS, c.CAFile, c.InstanceTransport
|
||||
}
|
||||
if u.User == nil {
|
||||
input.Token = token
|
||||
|
||||
@@ -82,7 +82,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
|
||||
func TestNewReusableWorkflowExecutorHoldsCloneLock(t *testing.T) {
|
||||
workflowDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "reusable.yml"), []byte(":"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "reusable.yml"), []byte("["), 0o644))
|
||||
|
||||
unlock, err := git.AcquireCloneLock(t.Context(), workflowDir)
|
||||
require.NoError(t, err)
|
||||
@@ -118,11 +118,11 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
||||
workdir := filepath.Join(t.TempDir(), "workspace")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(workdir, ".gitea", "workflows"), 0o755))
|
||||
require.NoError(t, os.Mkdir(workdir+".lock", 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, ".gitea", "workflows", "reusable.yml"), []byte(":"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, ".gitea", "workflows", "reusable.yml"), []byte("["), 0o644))
|
||||
|
||||
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
||||
rc := &RunContext{
|
||||
Config: &Config{Workdir: workdir},
|
||||
Config: &Config{Workdir: workdir, NoSkipCheckout: true},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {Uses: uses}}}},
|
||||
}
|
||||
err := newLocalReusableWorkflowExecutor(rc)(t.Context())
|
||||
@@ -130,7 +130,7 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T) {
|
||||
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCertTrustingCA(t *testing.T) {
|
||||
var gotCert bool
|
||||
var gotToken string
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -150,18 +150,19 @@ func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T
|
||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||
rc := &RunContext{
|
||||
Config: &Config{
|
||||
Env: testCheckoutEnv(),
|
||||
GitHubInstance: server.URL,
|
||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
Env: testCheckoutEnv(),
|
||||
GitHubInstance: server.URL,
|
||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||
ClientCertFile: certFile,
|
||||
ClientKeyFile: keyFile,
|
||||
CAFile: certFile,
|
||||
},
|
||||
Run: &model.Run{JobID: "j1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"j1": {}}}},
|
||||
}
|
||||
|
||||
require.Error(t, cloneRemoteReusableWorkflow(rc, server.URL+"/owner/repo", "main", t.TempDir())(t.Context()))
|
||||
require.True(t, gotCert)
|
||||
err = cloneRemoteReusableWorkflow(rc, server.URL+"/owner/repo", "main", t.TempDir())(t.Context())
|
||||
require.Error(t, err)
|
||||
require.True(t, gotCert, "clone failed before receiving a client certificate: %v", err)
|
||||
require.Equal(t, "token", gotToken)
|
||||
}
|
||||
|
||||
@@ -187,11 +188,12 @@ func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
||||
InsecureSkipTLS: true,
|
||||
ClientCertFile: "/cert",
|
||||
ClientKeyFile: "/key",
|
||||
CAFile: "/ca",
|
||||
InstanceTransport: &http.Transport{},
|
||||
}
|
||||
want := git.NewGitCloneExecutorInput{URL: tt.cloneURL, InsecureSkipTLS: tt.insecure}
|
||||
if tt.insecure {
|
||||
want.Transport = conf.InstanceTransport
|
||||
want.CAFile, want.Transport = conf.CAFile, conf.InstanceTransport
|
||||
}
|
||||
if tt.token {
|
||||
want.Token = "token"
|
||||
|
||||
+146
-23
@@ -68,6 +68,7 @@ type RunContext struct {
|
||||
kubernetes bool
|
||||
containerSpec model.ContainerSpec // container:, resolved once with platformImage
|
||||
containerEnv yaml.Node // container.env of the kept containerSpec, decoded once the job env resolves
|
||||
caEnv map[string]string // NODE_EXTRA_CA_CERTS for the runner CA, beneath any env the job and its image set
|
||||
JobName string
|
||||
ActionPath string
|
||||
Parent *RunContext
|
||||
@@ -81,6 +82,7 @@ type RunContext struct {
|
||||
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
||||
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
||||
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
||||
trackingID string // RUNNER_TRACKING_ID of the job, generated on first use
|
||||
runtimeToken string // ACTIONS_RUNTIME_TOKEN of the workflow run, shared with its called workflows and composite actions
|
||||
|
||||
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
||||
@@ -191,7 +193,7 @@ func (rc *RunContext) GetEnv() map[string]string {
|
||||
if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil {
|
||||
job := rc.Run.Job()
|
||||
if job != nil {
|
||||
rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
||||
rc.Env = rc.mergeEnv(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
||||
}
|
||||
}
|
||||
if rc.runtimeToken != "" {
|
||||
@@ -277,11 +279,37 @@ func (rc *RunContext) validVolumes() []string {
|
||||
if rc.dockerProxy != nil {
|
||||
volumes = append(volumes, rc.dockerProxy.Socket)
|
||||
}
|
||||
volumes = append(volumes, rc.Config.containerVolumeSources()...)
|
||||
// TODO: add a new configuration to control whether the docker daemon can be mounted
|
||||
return append(volumes, name, name+"-env",
|
||||
getDockerDaemonSocketMountPath(rc.containerDaemonSocket()))
|
||||
}
|
||||
|
||||
// containerVolumeSources lets workflows mount the container.volumes sources with no more access than they give.
|
||||
func (c *Config) containerVolumeSources() []string {
|
||||
var sources []string
|
||||
for _, spec := range c.ContainerVolumes {
|
||||
parsed, _ := container.ParseVolume(spec) // validated on config load
|
||||
if parsed.ReadOnly {
|
||||
parsed.Source += ":ro"
|
||||
}
|
||||
sources = append(sources, parsed.Source)
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
// mountVolumes sanitizes volumes and adds the container.volumes on the targets they leave unclaimed.
|
||||
func (rc *RunContext) mountVolumes(ctx context.Context, validVolumes, volumes []string) ([]string, map[string]string, map[string]bool) {
|
||||
binds, mounts, claimed := splitVolumes(container.SanitizeBinds(ctx, validVolumes, volumes))
|
||||
for _, spec := range rc.Config.ContainerVolumes {
|
||||
if parsed, _ := container.ParseVolume(spec); !claimed[parsed.Target] { // validated on config load
|
||||
claimed[parsed.Target] = true
|
||||
binds = append(binds, spec)
|
||||
}
|
||||
}
|
||||
return binds, mounts, claimed
|
||||
}
|
||||
|
||||
func (rc *RunContext) workdirMountRoot() string {
|
||||
if rc.Config.PresetGitHubContext != nil {
|
||||
return filepath.Dir(filepath.Dir(rc.Config.Workdir)) // only the daemon presets, its workdir is <parent>/<owner>/<repo>
|
||||
@@ -356,7 +384,7 @@ func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[stri
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
|
||||
// the runner's own mounts below yield to the targets the job claims
|
||||
binds, mounts, claimed := splitVolumes(container.SanitizeBinds(ctx, rc.validVolumes(), rc.containerSpec.Volumes))
|
||||
binds, mounts, claimed := rc.mountVolumes(ctx, rc.validVolumes(), rc.containerSpec.Volumes)
|
||||
|
||||
if rc.containerDaemonSocket() != "-" && !claimed["/var/run/docker.sock"] {
|
||||
binds = append(binds, rc.jobDockerSocket()+":/var/run/docker.sock")
|
||||
@@ -429,27 +457,58 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||
Workdir: rc.Config.Workdir,
|
||||
CleanWorkdir: rc.Config.CleanWorkdir,
|
||||
ActPath: actPath,
|
||||
CleanUp: func() {
|
||||
os.RemoveAll(miscpath)
|
||||
},
|
||||
StdOut: logWriter,
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
Root: miscpath,
|
||||
TrackingID: rc.runnerTrackingID(),
|
||||
StdOut: logWriter,
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
}
|
||||
rc.cleanUpJobContainer = rc.JobContainer.Remove()
|
||||
rc.setRunnerContextEnv(ctx)
|
||||
for _, env := range os.Environ() {
|
||||
if k, v, ok := strings.Cut(env, "="); ok {
|
||||
// don't override
|
||||
if _, ok := rc.Env[k]; !ok {
|
||||
rc.Env[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
rc.importHostEnv(os.Environ())
|
||||
|
||||
return rc.copyWorkflowFiles()(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) runnerTrackingID() string {
|
||||
job := rc.topLevelRunContext()
|
||||
if job.trackingID == "" {
|
||||
job.trackingID = "gitea_" + rand.Text()
|
||||
}
|
||||
return job.trackingID
|
||||
}
|
||||
|
||||
func (rc *RunContext) isEnvCaseInsensitive() bool {
|
||||
if rc.JobContainer != nil {
|
||||
return rc.JobContainer.IsEnvironmentCaseInsensitive()
|
||||
}
|
||||
return runtime.GOOS == "windows" && rc.IsHostEnv()
|
||||
}
|
||||
|
||||
func (rc *RunContext) importHostEnv(environ []string) {
|
||||
for _, env := range environ {
|
||||
if key, value, ok := strings.Cut(env, "="); ok {
|
||||
if key == "RUNNER_TRACKING_ID" || rc.isEnvCaseInsensitive() && strings.EqualFold(key, "RUNNER_TRACKING_ID") {
|
||||
continue
|
||||
}
|
||||
_, exists := common.LookupFold(rc.Env, key, rc.isEnvCaseInsensitive())
|
||||
if !exists {
|
||||
rc.Env[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) mergeEnv(envs ...map[string]string) map[string]string {
|
||||
if rc.isEnvCaseInsensitive() {
|
||||
env := map[string]string{}
|
||||
mergeIntoMapCaseInsensitive(env, envs...)
|
||||
return env
|
||||
} else {
|
||||
return mergeMaps(envs...)
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) setRunnerContextEnv(ctx context.Context) {
|
||||
for k, v := range rc.getRunnerContext(ctx) {
|
||||
if v, ok := v.(string); ok {
|
||||
@@ -460,6 +519,15 @@ func (rc *RunContext) setRunnerContextEnv(ctx context.Context) {
|
||||
|
||||
var errKubernetesDocker = errors.New("docker actions and docker:// steps are not supported on Kubernetes")
|
||||
|
||||
var errWindowsHostContainer = errors.New("container actions and docker:// steps are not supported in Windows host jobs")
|
||||
|
||||
var hostOS = runtime.GOOS // replaced in tests
|
||||
|
||||
func (rc *RunContext) isWindowsHost() bool {
|
||||
_, host := rc.JobContainer.(*container.HostEnvironment)
|
||||
return host && hostOS == "windows"
|
||||
}
|
||||
|
||||
func (rc *RunContext) startKubernetesPod() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if rc.containerSpec.Options != "" || len(rc.containerSpec.Volumes) > 0 || rc.containerSpec.Credentials != nil {
|
||||
@@ -664,7 +732,12 @@ func (rc *RunContext) jobServices(ctx context.Context, networkName string) (map[
|
||||
entrypoint = []string{spec.Entrypoint}
|
||||
}
|
||||
|
||||
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes)
|
||||
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes) // as given on Kubernetes, which rejects them
|
||||
validVolumes := rc.Config.ValidVolumes // not validVolumes(), a service gets no docker socket
|
||||
if !rc.kubernetes {
|
||||
validVolumes = append(slices.Clone(validVolumes), rc.Config.containerVolumeSources()...)
|
||||
serviceBinds, serviceMounts, _ = rc.mountVolumes(ctx, validVolumes, spec.Volumes)
|
||||
}
|
||||
|
||||
exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports)
|
||||
if err != nil {
|
||||
@@ -693,7 +766,7 @@ func (rc *RunContext) jobServices(ctx context.Context, networkName string) (map[
|
||||
NetworkAliases: []string{serviceID},
|
||||
ExposedPorts: exposedPorts,
|
||||
PortBindings: portBindings,
|
||||
ValidVolumes: rc.Config.ValidVolumes, // not validVolumes(), a service gets no docker socket
|
||||
ValidVolumes: validVolumes,
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
}
|
||||
}
|
||||
@@ -1078,10 +1151,35 @@ func (svc *serviceContainer) healthOutputSuffix() string {
|
||||
}
|
||||
|
||||
func (rc *RunContext) copyWorkflowFiles() common.Executor {
|
||||
return rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/",
|
||||
&container.FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: rc.EventJSON},
|
||||
&container.FileEntry{Name: "workflow/envs.txt", Mode: 0o666},
|
||||
)
|
||||
return func(ctx context.Context) error {
|
||||
files := []*container.FileEntry{
|
||||
{Name: "workflow/event.json", Mode: 0o644, Body: rc.EventJSON},
|
||||
{Name: "workflow/envs.txt", Mode: 0o666},
|
||||
}
|
||||
if rc.Config.CAFile != "" {
|
||||
ca, err := os.ReadFile(rc.Config.CAFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read runner CA file: %w", err)
|
||||
}
|
||||
files = append(files, &container.FileEntry{Name: "ca.crt", Mode: 0o644, Body: string(ca)})
|
||||
imageEnv := map[string]string{}
|
||||
if err = rc.JobContainer.UpdateFromImageEnv(&imageEnv)(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, set := imageEnv["NODE_EXTRA_CA_CERTS"]; !set {
|
||||
rc.caEnv = map[string]string{"NODE_EXTRA_CA_CERTS": rc.jobCAFile()}
|
||||
}
|
||||
}
|
||||
return rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", files...)(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// jobCAFile is where copyWorkflowFiles puts the runner CA in the job, empty without one.
|
||||
func (rc *RunContext) jobCAFile() string {
|
||||
if rc.Config.CAFile == "" {
|
||||
return ""
|
||||
}
|
||||
return rc.JobContainer.GetActPath() + "/ca.crt"
|
||||
}
|
||||
|
||||
// captureJobContainerInfo is a convenience: failing to describe the container must not
|
||||
@@ -1311,6 +1409,10 @@ func (rc *RunContext) resolvePlatformImage(ctx context.Context) error {
|
||||
}
|
||||
image, rc.kubernetes = strings.CutPrefix(image, "kubernetes://")
|
||||
rc.platformImage = cmp.Or(rc.containerSpec.Image, image)
|
||||
if rc.isEnvCaseInsensitive() {
|
||||
rc.Env = nil // Rebuild scopes after the host platform is known.
|
||||
rc.GetEnv()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1509,7 +1611,6 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext
|
||||
RepositoryOwner: rc.Config.Env["GITHUB_REPOSITORY_OWNER"],
|
||||
RetentionDays: cmp.Or(rc.Config.Env["GITHUB_RETENTION_DAYS"], "0"),
|
||||
RunnerPerflog: cmp.Or(rc.Config.Env["RUNNER_PERFLOG"], "/dev/null"),
|
||||
RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"],
|
||||
Repository: rc.Config.Env["GITHUB_REPOSITORY"],
|
||||
Ref: rc.Config.Env["GITHUB_REF"],
|
||||
Sha: rc.Config.Env["SHA_REF"],
|
||||
@@ -1523,6 +1624,8 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext
|
||||
ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json"
|
||||
}
|
||||
|
||||
ghc.RunnerTrackingID = rc.runnerTrackingID()
|
||||
|
||||
if preset := rc.Config.PresetGitHubContext; preset != nil {
|
||||
merged := *preset
|
||||
merged.Workflow, merged.Job, merged.Workspace, merged.EventPath = ghc.Workflow, ghc.Job, ghc.Workspace, ghc.EventPath
|
||||
@@ -1602,6 +1705,23 @@ func isLocalCheckout(ghc *model.GithubContext, step *model.Step) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// aliasGiteaEnv mirrors the runner's GITHUB_* variables as GITEA_*, call it after the last write to them.
|
||||
func aliasGiteaEnv(env map[string]string) {
|
||||
for _, suffix := range []string{
|
||||
"ACTION", "ACTION_PATH", "ACTION_REF", "ACTION_REPOSITORY", "ACTIONS", "ACTOR", "ACTOR_ID", "API_URL", "BASE_REF",
|
||||
"ENV", "EVENT_NAME", "EVENT_PATH", "GRAPHQL_URL", "HEAD_REF", "JOB", "OUTPUT", "PATH", "REF", "REF_NAME",
|
||||
"REF_PROTECTED", "REF_TYPE", "REPOSITORY", "REPOSITORY_ID", "REPOSITORY_OWNER", "REPOSITORY_OWNER_ID",
|
||||
"RETENTION_DAYS", "RUN_ATTEMPT", "RUN_ID", "RUN_NUMBER", "SERVER_URL", "SHA", "STATE", "STEP_SUMMARY",
|
||||
"TRIGGERING_ACTOR", "WORKFLOW", "WORKFLOW_REF", "WORKFLOW_SHA", "WORKSPACE",
|
||||
} {
|
||||
if value, ok := env["GITHUB_"+suffix]; ok {
|
||||
env["GITEA_"+suffix] = value
|
||||
} else {
|
||||
delete(env, "GITEA_"+suffix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) {
|
||||
if _, ok := env["CI"]; !ok {
|
||||
env["CI"] = "true"
|
||||
@@ -1628,7 +1748,9 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
||||
env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner
|
||||
env["GITHUB_RETENTION_DAYS"] = github.RetentionDays
|
||||
env["RUNNER_PERFLOG"] = github.RunnerPerflog
|
||||
env["RUNNER_TRACKING_ID"] = github.RunnerTrackingID
|
||||
if _, ok := common.LookupFold(env, "RUNNER_TRACKING_ID", rc.isEnvCaseInsensitive()); !ok {
|
||||
env["RUNNER_TRACKING_ID"] = github.RunnerTrackingID
|
||||
}
|
||||
env["GITHUB_BASE_REF"] = github.BaseRef
|
||||
env["GITHUB_HEAD_REF"] = github.HeadRef
|
||||
env["GITHUB_SERVER_URL"] = github.ServerURL
|
||||
@@ -1654,6 +1776,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
||||
|
||||
env["GITHUB_GRAPHQL_URL"] = ""
|
||||
env["GITHUB_RUN_ATTEMPT"] = github.RunAttempt
|
||||
aliasGiteaEnv(env)
|
||||
|
||||
env["RUNNER_NAME"] = rc.Config.RunnerName
|
||||
env["RUNNER_ENVIRONMENT"] = "self-hosted"
|
||||
|
||||
+120
-23
@@ -19,6 +19,7 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -276,25 +277,38 @@ jobs:
|
||||
require.Equal(t, [2]string{"", ""}, credentials["redis:latest"])
|
||||
}
|
||||
|
||||
func TestStartJobContainerGivesServicesTheirVolumes(t *testing.T) {
|
||||
redis := startJobContainerInputs(t, `
|
||||
func TestStartJobContainerMountsWorkflowVolumesAndContainerVolumesOnUnclaimedTargets(t *testing.T) {
|
||||
const ca, certs = "/host/ca.crt:/etc/ssl/certs/ca-certificates.crt:ro", "/host/certs:/certs"
|
||||
inputs := startJobContainerInputs(t, `
|
||||
jobs:
|
||||
job:
|
||||
container:
|
||||
image: node:24
|
||||
volumes:
|
||||
- /job/certs:/certs
|
||||
services:
|
||||
redis:
|
||||
image: redis:latest
|
||||
volumes:
|
||||
- data:/data
|
||||
- data:/backup
|
||||
`, &Config{ValidVolumes: []string{"data"}})[0]
|
||||
- /blocked/ca.crt:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
- blocked:/certs
|
||||
`, &Config{ValidVolumes: []string{"data", "/job/certs"}, ContainerVolumes: []string{ca, certs}})
|
||||
require.Len(t, inputs, 2)
|
||||
redis, job := inputs[0], inputs[1]
|
||||
|
||||
require.Equal(t, "redis:latest", redis.Image)
|
||||
require.Equal(t, []string{"data"}, redis.ValidVolumes)
|
||||
require.Equal(t, []string{"data", "/job/certs", "/host/ca.crt:ro", "/host/certs"}, redis.ValidVolumes)
|
||||
require.Equal(t, map[string]string{"data": "/data"}, redis.Mounts)
|
||||
require.Equal(t, []string{"data:/backup"}, redis.Binds)
|
||||
require.Equal(t, []string{"data:/backup", ca, certs}, redis.Binds)
|
||||
require.Empty(t, redis.WorkingDir)
|
||||
require.Nil(t, redis.Entrypoint)
|
||||
require.Nil(t, redis.Cmd)
|
||||
assert.Subset(t, job.Binds, []string{"/job/certs:/certs", ca})
|
||||
assert.NotContains(t, job.Binds, certs)
|
||||
assert.Subset(t, job.ValidVolumes, []string{"/job/certs", "/host/ca.crt:ro", "/host/certs"})
|
||||
assert.NotContains(t, job.ValidVolumes, "/host/ca.crt", "a workflow must not mount a read-only source writable")
|
||||
}
|
||||
|
||||
func TestStartJobContainerBuildsEvaluatedJobAndServiceInputs(t *testing.T) {
|
||||
@@ -303,7 +317,7 @@ jobs:
|
||||
job:
|
||||
container: ${{ fromJSON('{"image":"docker://node:20","options":"--label ${{ github.job }}","credentials":{"username":"user","password":"${{ vars.TITLE }}"}}') }}
|
||||
services: ${{ fromJSON('{"redis":{"image":"docker://redis:latest","volumes":["data:/data"],"env":{"TITLE":"${{ github.job }}"},"credentials":{"username":"user","password":"${{ github.job }}"},"entrypoint":"/entry.sh","command":"redis-server --port 6380"}}') }}
|
||||
`, &Config{})
|
||||
`, &Config{ValidVolumes: []string{"data"}})
|
||||
redis, job := inputs[0], inputs[1]
|
||||
|
||||
require.Equal(t, "redis:latest", redis.Image)
|
||||
@@ -877,25 +891,25 @@ func TestGetGitHubContext(t *testing.T) {
|
||||
RepositoryID: "34",
|
||||
RepositoryOwnerID: "56",
|
||||
WorkflowRef: "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
WorkflowSha: "workflow-sha",
|
||||
RefProtected: true,
|
||||
}
|
||||
ghc = rc.getGithubContext(t.Context())
|
||||
assert.Equal(t, "preset-actor", ghc.Actor)
|
||||
assert.Equal(t, "triggerer", ghc.TriggeringActor)
|
||||
assert.Equal(t, "job1", ghc.Job)
|
||||
env := map[string]string{}
|
||||
env := map[string]string{"GITHUB_WORKFLOW_SHA": "workflow-sha"}
|
||||
rc.withGithubEnv(t.Context(), ghc, env)
|
||||
for name, value := range map[string]string{
|
||||
"GITHUB_ACTOR_ID": "12",
|
||||
"GITHUB_TRIGGERING_ACTOR": "triggerer",
|
||||
"GITHUB_REPOSITORY_ID": "34",
|
||||
"GITHUB_REPOSITORY_OWNER_ID": "56",
|
||||
"GITHUB_WORKFLOW_REF": "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
"GITHUB_WORKFLOW_SHA": "workflow-sha",
|
||||
"GITHUB_REF_PROTECTED": "true",
|
||||
for suffix, value := range map[string]string{
|
||||
"ACTOR_ID": "12",
|
||||
"TRIGGERING_ACTOR": "triggerer",
|
||||
"REPOSITORY_ID": "34",
|
||||
"REPOSITORY_OWNER_ID": "56",
|
||||
"WORKFLOW_REF": "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
"WORKFLOW_SHA": "workflow-sha",
|
||||
"REF_PROTECTED": "true",
|
||||
} {
|
||||
assert.Equal(t, value, env[name], name)
|
||||
assert.Equal(t, value, env["GITHUB_"+suffix], suffix)
|
||||
assert.Equal(t, value, env["GITEA_"+suffix], suffix)
|
||||
}
|
||||
ghc.RefProtected = false
|
||||
rc.withGithubEnv(t.Context(), ghc, env)
|
||||
@@ -922,12 +936,12 @@ func TestGetGitHubContext(t *testing.T) {
|
||||
github := rc.getGithubContext(t.Context())
|
||||
assert.Equal(t, testcase.serverURL, github.ServerURL)
|
||||
assert.Equal(t, testcase.apiURL, github.APIURL)
|
||||
env := map[string]string{}
|
||||
env := map[string]string{"GITEA_SERVER_URL": "stale", "GITEA_API_URL": "stale"}
|
||||
rc.withGithubEnv(t.Context(), github, env)
|
||||
_, exported := env["GITHUB_SERVER_URL"]
|
||||
assert.Equal(t, testcase.serverURL != "", exported)
|
||||
assert.Equal(t, testcase.serverURL, env["GITHUB_SERVER_URL"])
|
||||
assert.Equal(t, testcase.apiURL, env["GITHUB_API_URL"])
|
||||
assert.Equal(t, testcase.serverURL, env["GITEA_SERVER_URL"])
|
||||
assert.Equal(t, testcase.apiURL, env["GITEA_API_URL"])
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1082,6 +1096,51 @@ func TestRunContextGetEnv(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunContextEnvCaseSensitivity(t *testing.T) {
|
||||
for _, caseInsensitive := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(caseInsensitive), func(t *testing.T) {
|
||||
rc := createIfTestRunContext(map[string]*model.Job{
|
||||
"job1": createJob(t, "runs-on: self-hosted\nenv:\n build_mode: job\n config_value: job", ""),
|
||||
})
|
||||
rc.Run.Workflow.Env = map[string]string{"BUILD_MODE": "workflow", "CONFIG_VALUE": "workflow", "PATH": "workflow-path", "runner_tool_cache": "workflow-cache"}
|
||||
rc.Config.Env = map[string]string{"CONFIG_VALUE": "config", "RUNNER_TOOL_CACHE": "configured-cache"}
|
||||
rc.Config.DisableActEnv, rc.Config.NoSkipCheckout = true, true
|
||||
rc.Env = nil
|
||||
rc.GetEnv()
|
||||
rc.JobContainer = fakeContainer{ExecutionsEnvironment: &container.HostEnvironment{}, caseInsensitive: caseInsensitive}
|
||||
require.NoError(t, rc.resolvePlatformImage(t.Context()))
|
||||
if caseInsensitive {
|
||||
assert.Equal(t, map[string]string{"build_mode": "job", "CONFIG_VALUE": "config", "PATH": "workflow-path", "RUNNER_TOOL_CACHE": "configured-cache"}, rc.GetEnv())
|
||||
} else {
|
||||
assert.Equal(t, map[string]string{
|
||||
"BUILD_MODE": "workflow", "build_mode": "job", "CONFIG_VALUE": "config",
|
||||
"config_value": "job", "PATH": "workflow-path", "runner_tool_cache": "workflow-cache", "RUNNER_TOOL_CACHE": "configured-cache",
|
||||
}, rc.GetEnv())
|
||||
}
|
||||
want := maps.Clone(rc.GetEnv())
|
||||
rc.Run.Workflow.Env = nil
|
||||
require.NoError(t, rc.Run.Workflow.RawEnv.Encode(`${{ fromJSON('{"BUILD_MODE":"workflow","CONFIG_VALUE":"workflow","PATH":"workflow-path","runner_tool_cache":"workflow-cache"}') }}`))
|
||||
rc.Env = nil
|
||||
evaluated, err := evaluateJobEnv(t.Context(), rc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, evaluated)
|
||||
rc.Env = evaluated
|
||||
|
||||
rc.importHostEnv([]string{"Path=host-path", "HOST_ONLY=value", "RUNNER_TRACKING_ID=parent", "runner_tracking_id=parent-lower"})
|
||||
assert.NotContains(t, rc.Env, "RUNNER_TRACKING_ID")
|
||||
assert.Equal(t, "workflow-path", rc.Env["PATH"])
|
||||
assert.Equal(t, "value", rc.Env["HOST_ONLY"])
|
||||
if caseInsensitive {
|
||||
assert.NotContains(t, rc.Env, "Path")
|
||||
assert.NotContains(t, rc.Env, "runner_tracking_id")
|
||||
} else {
|
||||
assert.Equal(t, "host-path", rc.Env["Path"])
|
||||
assert.Equal(t, "parent-lower", rc.Env["runner_tracking_id"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetActionEnvRefreshesInputs(t *testing.T) {
|
||||
rc := &RunContext{}
|
||||
rc.setActionEnv(map[string]string{"INPUT_MSG": "pre"})
|
||||
@@ -1404,14 +1463,52 @@ func TestGetJobContextReportsContainers(t *testing.T) {
|
||||
}, jobContext.Services)
|
||||
}
|
||||
|
||||
type imageEnvContainer struct {
|
||||
*container.HostEnvironment
|
||||
imageEnv map[string]string
|
||||
}
|
||||
|
||||
func (c imageEnvContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
return func(_ctx context.Context) error { maps.Copy(*env, c.imageEnv); return nil }
|
||||
}
|
||||
|
||||
func TestCopyWorkflowFilesHandsNodeTheRunnerCAUnlessTheJobOrImageSetsIt(t *testing.T) {
|
||||
caFile, actPath := filepath.Join(t.TempDir(), "ca.crt"), t.TempDir()
|
||||
require.NoError(t, os.WriteFile(caFile, []byte("PEM\n"), 0o600))
|
||||
own := map[string]string{"NODE_EXTRA_CA_CERTS": "/own.pem"}
|
||||
stepEnv := func(caFile string, jobEnv, containerEnv, imageEnv map[string]string) map[string]string {
|
||||
rc := &RunContext{
|
||||
Config: &Config{CAFile: caFile},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
Env: maps.Clone(jobEnv),
|
||||
JobContainer: imageEnvContainer{&container.HostEnvironment{Path: t.TempDir(), ActPath: actPath, StdOut: io.Discard}, imageEnv},
|
||||
}
|
||||
rc.containerSpec.Env = containerEnv
|
||||
require.NoError(t, rc.copyWorkflowFiles()(t.Context()))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, setupEnv(t.Context(), &stepRun{RunContext: rc, Step: &model.Step{}, env: env}))
|
||||
return env
|
||||
}
|
||||
|
||||
assert.Equal(t, filepath.ToSlash(actPath)+"/ca.crt", stepEnv(caFile, nil, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||
ca, err := os.ReadFile(filepath.Join(actPath, "ca.crt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "PEM\n", string(ca))
|
||||
assert.Equal(t, "/own.pem", stepEnv(caFile, own, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||
assert.Equal(t, "/own.pem", stepEnv(caFile, nil, own, nil)["NODE_EXTRA_CA_CERTS"])
|
||||
assert.NotContains(t, stepEnv(caFile, nil, nil, own), "NODE_EXTRA_CA_CERTS")
|
||||
assert.NotContains(t, stepEnv("", nil, nil, nil), "NODE_EXTRA_CA_CERTS")
|
||||
}
|
||||
|
||||
func TestCaptureJobContainerInfoExportsDockerWorkspace(t *testing.T) {
|
||||
workdir := t.TempDir()
|
||||
job := &containerMock{}
|
||||
job.On("Inspect", mock.Anything).Return(&container.Info{
|
||||
ID: "job-container-id",
|
||||
Mounts: map[string]string{"/workspace": "/var/lib/docker/volumes/job/_data"},
|
||||
Mounts: map[string]string{job.ToContainerPath(workdir): "/var/lib/docker/volumes/job/_data"},
|
||||
}, nil)
|
||||
rc := &RunContext{
|
||||
Config: &Config{Workdir: "/workspace/owner/repo/"},
|
||||
Config: &Config{Workdir: filepath.Join(workdir, "owner", "repo") + string(filepath.Separator)},
|
||||
Env: map[string]string{},
|
||||
JobContainer: job,
|
||||
}
|
||||
@@ -1421,7 +1518,7 @@ func TestCaptureJobContainerInfoExportsDockerWorkspace(t *testing.T) {
|
||||
assert.Equal(t, "job-container-id", rc.jobContainerID)
|
||||
assert.Equal(t, "/var/lib/docker/volumes/job/_data/owner/repo", rc.Env["GITEA_DOCKER_WORKSPACE"])
|
||||
|
||||
rc.Config.Workdir = "/elsewhere"
|
||||
rc.Config.Workdir = filepath.Dir(workdir)
|
||||
rc.Env = map[string]string{}
|
||||
require.NoError(t, rc.captureJobContainerInfo()(t.Context()))
|
||||
assert.NotContains(t, rc.Env, "GITEA_DOCKER_WORKSPACE")
|
||||
|
||||
@@ -83,9 +83,11 @@ type Config struct {
|
||||
InstanceTransport http.RoundTripper
|
||||
JobLoggerLevel *logging.Level // the level of job logger
|
||||
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
||||
ContainerVolumes []string // mounted into every job, service and docker action container
|
||||
SharedToolCache bool // one tool cache for all jobs instead of one per job
|
||||
InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance
|
||||
ClientCertFile, ClientKeyFile string // PEM client certificate for the Gitea instance
|
||||
CAFile string // PEM CA certificates the Gitea instance's certificate chains to
|
||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package runner
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRunEventHostEnvironmentUTF8(t *testing.T) {
|
||||
handler := logtest.New()
|
||||
t.Cleanup(func() {
|
||||
if t.Failed() {
|
||||
for _, record := range handler.AllRecords() {
|
||||
stepIDs, _ := record.Attr("stepID")
|
||||
t.Logf("stepID=%v: %s", stepIDs, record.Message)
|
||||
}
|
||||
}
|
||||
})
|
||||
table := TestJobFileInfo{
|
||||
workdir: workdir,
|
||||
workflowPath: "windows-utf8",
|
||||
eventName: "push",
|
||||
platforms: map[string]string{"windows-latest": "-self-hosted"},
|
||||
}
|
||||
table.runTest(WithJobLoggerFactory(t.Context(), factoryFunc(func() logging.Handler { return handler })), t, &Config{Env: testCheckoutEnv()})
|
||||
var outputSteps []string
|
||||
for _, record := range handler.AllRecords() {
|
||||
if strings.TrimRight(record.Message, "\r\n") == "你好,世界" {
|
||||
stepIDs, ok := record.Attr("stepID")
|
||||
require.True(t, ok)
|
||||
outputSteps = append(outputSteps, stepIDs.([]string)...)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{"cmd", "pwsh", "powershell"}, outputSteps)
|
||||
}
|
||||
+40
-32
@@ -27,12 +27,24 @@ type step interface {
|
||||
post() common.Executor
|
||||
|
||||
getRunContext() *RunContext
|
||||
getGithubContext(ctx context.Context) *model.GithubContext
|
||||
getStepModel() *model.Step
|
||||
getEnv() *map[string]string
|
||||
getIfExpression(context context.Context, stage stepStage) string
|
||||
}
|
||||
|
||||
func stepActionIdentity(step step) (repository, ref string) {
|
||||
if action, ok := step.(interface{ actionIdentity() (string, string) }); ok {
|
||||
return action.actionIdentity()
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func getStepGithubContext(ctx context.Context, step step) *model.GithubContext {
|
||||
github := step.getRunContext().getGithubContext(ctx)
|
||||
github.ActionRepository, github.ActionRef = stepActionIdentity(step)
|
||||
return github
|
||||
}
|
||||
|
||||
type stepStage int
|
||||
|
||||
const (
|
||||
@@ -57,10 +69,15 @@ func (s stepStage) String() string {
|
||||
}
|
||||
|
||||
func processRunnerEnvFileCommand(ctx context.Context, fileName string, rc *RunContext, setter func(context.Context, map[string]string, string)) error {
|
||||
if rc.JobContainer.IsEnvironmentCaseInsensitive() {
|
||||
return container.ParseEnvFile(rc.JobContainer, container.JoinPath(rc.JobContainer.GetActPath(), fileName), func(name, value string) {
|
||||
setter(ctx, map[string]string{"name": name}, value)
|
||||
}, true)(ctx)
|
||||
}
|
||||
env := map[string]string{}
|
||||
err := rc.JobContainer.UpdateFromEnv(path.Join(rc.JobContainer.GetActPath(), fileName), &env)(ctx)
|
||||
for k, v := range env {
|
||||
setter(ctx, map[string]string{"name": k}, v)
|
||||
err := rc.JobContainer.UpdateFromEnv(container.JoinPath(rc.JobContainer.GetActPath(), fileName), &env)(ctx)
|
||||
for key, value := range env {
|
||||
setter(ctx, map[string]string{"name": key}, value)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -92,6 +109,8 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
runStep, err = isStepEnabled(ctx, ifExpression, step, stage)
|
||||
}
|
||||
if err == nil && runStep {
|
||||
env := *step.getEnv()
|
||||
env["GITHUB_ACTION_REPOSITORY"], env["GITHUB_ACTION_REF"] = stepActionIdentity(step)
|
||||
err = setupInputs(ctx, step)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -125,16 +144,16 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
|
||||
outputFileCommand := path.Join("workflow", "outputcmd.txt")
|
||||
(*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand)
|
||||
(*step.getEnv())["GITHUB_OUTPUT"] = container.JoinPath(actPath, outputFileCommand)
|
||||
|
||||
stateFileCommand := path.Join("workflow", "statecmd.txt")
|
||||
(*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand)
|
||||
(*step.getEnv())["GITHUB_STATE"] = container.JoinPath(actPath, stateFileCommand)
|
||||
|
||||
pathFileCommand := path.Join("workflow", "pathcmd.txt")
|
||||
(*step.getEnv())["GITHUB_PATH"] = path.Join(actPath, pathFileCommand)
|
||||
(*step.getEnv())["GITHUB_PATH"] = container.JoinPath(actPath, pathFileCommand)
|
||||
|
||||
envFileCommand := path.Join("workflow", "envs.txt")
|
||||
(*step.getEnv())["GITHUB_ENV"] = path.Join(actPath, envFileCommand)
|
||||
(*step.getEnv())["GITHUB_ENV"] = container.JoinPath(actPath, envFileCommand)
|
||||
|
||||
// Per-step summary file. Composite sub-steps share the outer job step's index
|
||||
// via the Parent chain so all writes from within a composite action accumulate
|
||||
@@ -142,16 +161,8 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
topRC := rc.topLevelRunContext()
|
||||
stepSummaryIndex := topRC.CurrentStepIndex
|
||||
summaryFileCommand := path.Join("workflow", "step-summary-"+strconv.Itoa(stepSummaryIndex)+".md")
|
||||
(*step.getEnv())["GITHUB_STEP_SUMMARY"] = path.Join(actPath, summaryFileCommand)
|
||||
|
||||
{
|
||||
// For Gitea
|
||||
(*step.getEnv())["GITEA_OUTPUT"] = (*step.getEnv())["GITHUB_OUTPUT"]
|
||||
(*step.getEnv())["GITEA_STATE"] = (*step.getEnv())["GITHUB_STATE"]
|
||||
(*step.getEnv())["GITEA_PATH"] = (*step.getEnv())["GITHUB_PATH"]
|
||||
(*step.getEnv())["GITEA_ENV"] = (*step.getEnv())["GITHUB_ENV"]
|
||||
(*step.getEnv())["GITEA_STEP_SUMMARY"] = (*step.getEnv())["GITHUB_STEP_SUMMARY"]
|
||||
}
|
||||
(*step.getEnv())["GITHUB_STEP_SUMMARY"] = container.JoinPath(actPath, summaryFileCommand)
|
||||
aliasGiteaEnv(*step.getEnv())
|
||||
|
||||
// Reset the per-phase file-command files. GITHUB_STEP_SUMMARY is intentionally
|
||||
// excluded here and initialized below at most once per step so writes from later
|
||||
@@ -197,7 +208,7 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, outputFileCommand, rc, rc.setOutput); fileErr != nil && err == nil {
|
||||
err = fileErr
|
||||
}
|
||||
if fileErr := rc.UpdateExtraPath(fileCommandCtx, path.Join(actPath, pathFileCommand)); fileErr != nil && err == nil {
|
||||
if fileErr := rc.UpdateExtraPath(fileCommandCtx, container.JoinPath(actPath, pathFileCommand)); fileErr != nil && err == nil {
|
||||
err = fileErr
|
||||
}
|
||||
}
|
||||
@@ -252,8 +263,8 @@ func setupEnv(ctx context.Context, step step) error {
|
||||
|
||||
env := step.getEnv()
|
||||
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
||||
mergeIntoMap(step, env, rc.containerSpec.Env, rc.GetEnv())
|
||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env)
|
||||
mergeIntoMap(step, env, rc.caEnv, rc.containerSpec.Env, rc.GetEnv())
|
||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *env)
|
||||
|
||||
var err error
|
||||
exprEval := rc.NewExpressionEvaluator(ctx)
|
||||
@@ -302,7 +313,7 @@ func isStepEnabled(ctx context.Context, expr string, step step, stage stepStage)
|
||||
if rc.Parent != nil && stage == stepStageMain && jobContext.Status != "cancelled" {
|
||||
jobContext.Status = rc.ownStatus()
|
||||
}
|
||||
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext), expr, defaultStatusCheck)
|
||||
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext, rc.getGithubContext(ctx)), expr, defaultStatusCheck)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("if-expression %q evaluation failed: %s", expr, err)
|
||||
}
|
||||
@@ -345,17 +356,14 @@ func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]st
|
||||
for k := range target {
|
||||
foldKeys[strings.ToLower(k)] = k
|
||||
}
|
||||
toKey := func(s string) string {
|
||||
foldKey := strings.ToLower(s)
|
||||
if k, ok := foldKeys[foldKey]; ok {
|
||||
return k
|
||||
}
|
||||
foldKeys[foldKey] = s
|
||||
return s
|
||||
}
|
||||
for _, m := range maps {
|
||||
for k, v := range m {
|
||||
target[toKey(k)] = v
|
||||
for key, value := range m {
|
||||
foldKey := strings.ToLower(key)
|
||||
if previous, ok := foldKeys[foldKey]; ok {
|
||||
delete(target, previous)
|
||||
}
|
||||
foldKeys[foldKey] = key
|
||||
target[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
rc.ApplyExtraPath(ctx, &sab.env)
|
||||
return sab.run(ctx, &action.Context{
|
||||
Container: rc.JobContainer,
|
||||
Github: rc.getGithubContext(ctx),
|
||||
Github: getStepGithubContext(ctx, sab),
|
||||
Inputs: sab.Step.With,
|
||||
HostWorkdir: rc.Config.Workdir,
|
||||
Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir),
|
||||
@@ -49,6 +49,7 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
InsecureSkipTLS: rc.Config.InsecureSkipTLS,
|
||||
ClientCertFile: rc.Config.ClientCertFile,
|
||||
ClientKeyFile: rc.Config.ClientKeyFile,
|
||||
CAFile: rc.jobCAFile(),
|
||||
Env: sab.env,
|
||||
})
|
||||
})
|
||||
@@ -58,10 +59,6 @@ func (sab *stepActionBuiltin) post() common.Executor { return common.NewPipeline
|
||||
|
||||
func (sab *stepActionBuiltin) getRunContext() *RunContext { return sab.RunContext }
|
||||
|
||||
func (sab *stepActionBuiltin) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sab.RunContext.getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sab *stepActionBuiltin) getStepModel() *model.Step { return sab.Step }
|
||||
|
||||
func (sab *stepActionBuiltin) getEnv() *map[string]string { return &sab.env }
|
||||
|
||||
@@ -95,10 +95,6 @@ func (sal *stepActionLocal) getRunContext() *RunContext {
|
||||
return sal.RunContext
|
||||
}
|
||||
|
||||
func (sal *stepActionLocal) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sal.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sal *stepActionLocal) getStepModel() *model.Step {
|
||||
return sal.Step
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type stepActionMocks struct {
|
||||
@@ -53,7 +54,7 @@ func TestStepActionLocalTest(t *testing.T) {
|
||||
readAction: salm.readAction,
|
||||
runAction: salm.runAction,
|
||||
RunContext: &RunContext{
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
StepResults: map[string]*model.StepResult{"previous": {Outputs: map[string]string{"value": "previous"}}},
|
||||
ExprEval: &expressionEvaluator{},
|
||||
Config: &Config{
|
||||
Workdir: "/tmp",
|
||||
@@ -68,25 +69,42 @@ func TestStepActionLocalTest(t *testing.T) {
|
||||
},
|
||||
},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/action", "GITHUB_ACTION_REF": "v1"},
|
||||
Matrix: map[string]any{"value": "matrix"},
|
||||
actionInputs: map[string]any{"shared": "input"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Uses: "./path/to/action",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/action' && github.action_ref == 'v1' && " +
|
||||
"env.STEP_VALUE == 'step' && env.GITHUB_ACTION_REPOSITORY == 'forged/repo' && env.GITHUB_ACTION_REF == 'forged-ref' && " +
|
||||
"steps.previous.outputs.value == 'previous' && inputs.shared == 'input' && matrix.value == 'matrix'"},
|
||||
},
|
||||
}
|
||||
require.NoError(t, sal.Step.Env.Encode(map[string]string{"STEP_VALUE": "step", "GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref"}))
|
||||
require.NoError(t, sal.Step.RawWith.Encode(map[string]string{"identity": "${{ github.action_repository }}|${{ github.action_ref }}"}))
|
||||
|
||||
salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything).
|
||||
Return(&model.Action{}, nil)
|
||||
|
||||
expectStepFileCommands(t, cm)
|
||||
|
||||
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(noopExecutor)
|
||||
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error {
|
||||
sal.RunContext.setOutput(ctx, map[string]string{"name": "value"}, "ran")
|
||||
return nil
|
||||
})
|
||||
|
||||
err := sal.pre()(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = sal.main()(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sal.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sal.env["GITHUB_ACTION_REF"])
|
||||
assert.Equal(t, "|", sal.Step.With["identity"])
|
||||
assert.Equal(t, "ran", sal.RunContext.StepResults["1"].Outputs["value"])
|
||||
assert.Equal(t, "other/action", sal.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sal.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
cm.AssertExpectations(t)
|
||||
salm.AssertExpectations(t)
|
||||
|
||||
@@ -75,7 +75,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
}
|
||||
sar.Step.Uses = uses
|
||||
|
||||
github := sar.getGithubContext(ctx) // read before remoteAction is set, so `$/` resolves against the enclosing action
|
||||
github := sar.RunContext.getGithubContext(ctx)
|
||||
if sar.remoteAction, err = newRemoteAction(sar.Step.Uses, github, sar.RunContext.Config.GitHubInstance); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func (sar *stepActionRemote) main() common.Executor {
|
||||
return common.NewPipelineExecutor(
|
||||
sar.prepareActionExecutor(),
|
||||
runStepExecutor(sar, stepStageMain, func(ctx context.Context) error {
|
||||
github := sar.getGithubContext(ctx)
|
||||
github := getStepGithubContext(ctx, sar)
|
||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||
printRunActionHeader(ctx, sar.Step, nil, sar.env, sar.RunContext)
|
||||
if sar.RunContext.Config.BindWorkdir {
|
||||
@@ -181,17 +181,11 @@ func (sar *stepActionRemote) getRunContext() *RunContext {
|
||||
return sar.RunContext
|
||||
}
|
||||
|
||||
func (sar *stepActionRemote) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
ghc := sar.getRunContext().getGithubContext(ctx)
|
||||
|
||||
// extend github context if we already have an initialized remoteAction
|
||||
remoteAction := sar.remoteAction
|
||||
if remoteAction != nil {
|
||||
ghc.ActionRepository = fmt.Sprintf("%s/%s", remoteAction.Org, remoteAction.Repo)
|
||||
ghc.ActionRef = remoteAction.Ref
|
||||
func (sar *stepActionRemote) actionIdentity() (string, string) {
|
||||
if sar.remoteAction == nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
return ghc
|
||||
return sar.remoteAction.Org + "/" + sar.remoteAction.Repo, sar.remoteAction.Ref
|
||||
}
|
||||
|
||||
func (sar *stepActionRemote) getStepModel() *model.Step {
|
||||
@@ -205,7 +199,7 @@ func (sar *stepActionRemote) getEnv() *map[string]string {
|
||||
func (sar *stepActionRemote) getIfExpression(ctx context.Context, stage stepStage) string {
|
||||
switch stage {
|
||||
case stepStagePre:
|
||||
github := sar.getGithubContext(ctx)
|
||||
github := getStepGithubContext(ctx, sar)
|
||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||
// skip local checkout pre step
|
||||
return "false"
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestStepActionRemote(t *testing.T) {
|
||||
status model.StepStatus
|
||||
runError error
|
||||
}{
|
||||
{name: "run-successful", status: model.StepStatusSuccess},
|
||||
{name: "run-successful", condition: "github.action_repository == 'other/composite' && github.action_ref == 'v2'", status: model.StepStatusSuccess},
|
||||
{name: "run-skipped", condition: "false", status: model.StepStatusSkipped},
|
||||
{name: "run-error", status: model.StepStatusFailure, runError: assert.AnError},
|
||||
} {
|
||||
@@ -110,6 +110,7 @@ func TestStepActionRemote(t *testing.T) {
|
||||
},
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v2"},
|
||||
},
|
||||
Step: &model.Step{ID: "step", Uses: "remote/action@v1", If: yaml.Node{Value: tt.condition}},
|
||||
readAction: sarm.readAction,
|
||||
@@ -127,6 +128,8 @@ func TestStepActionRemote(t *testing.T) {
|
||||
require.NoError(t, sar.pre()(ctx))
|
||||
assert.Equal(t, tt.runError, sar.main()(ctx))
|
||||
assert.True(t, clonedAction)
|
||||
assert.Equal(t, "remote/action", sar.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sar.env["GITHUB_ACTION_REF"])
|
||||
assert.Equal(t, &model.StepResult{Conclusion: tt.status, Outcome: tt.status, Outputs: map[string]string{}}, sar.RunContext.StepResults["step"])
|
||||
|
||||
sarm.AssertExpectations(t)
|
||||
@@ -182,7 +185,7 @@ func TestStepActionRemotePrepare(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
root, err := prepare(job, "org/repo@v1")
|
||||
require.NoError(t, err)
|
||||
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job}, "org/repo/save@v1")
|
||||
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job, Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "org/repo", "GITHUB_ACTION_REF": "v1"}}, "$/save")
|
||||
require.NoError(t, err)
|
||||
other, err := prepare(job, "org/repo@v2")
|
||||
require.NoError(t, err)
|
||||
@@ -346,9 +349,6 @@ func Test_newRemoteActionSelfRepo(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{name: "unknown instance", action: "$/.gitea/actions/build", github: &model.GithubContext{Repository: "owner/workflow-repo", Sha: "abc123"}},
|
||||
{name: "empty path", action: "$/", github: workflow},
|
||||
{name: "ref suffix is not allowed", action: "$/.gitea/actions/build@v1", github: workflow},
|
||||
{name: "path traversal", action: "$/../escape", github: workflow},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -35,10 +35,6 @@ func (sd *stepDocker) getRunContext() *RunContext {
|
||||
return sd.RunContext
|
||||
}
|
||||
|
||||
func (sd *stepDocker) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sd.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sd *stepDocker) getStepModel() *model.Step {
|
||||
return sd.Step
|
||||
}
|
||||
@@ -56,6 +52,9 @@ func (sd *stepDocker) runUsesContainer() common.Executor {
|
||||
step := sd.Step
|
||||
|
||||
return func(ctx context.Context) error {
|
||||
if rc.isWindowsHost() {
|
||||
return errWindowsHostContainer
|
||||
}
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestStepDockerMain(t *testing.T) {
|
||||
@@ -57,13 +58,16 @@ func TestStepDockerMain(t *testing.T) {
|
||||
},
|
||||
},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Uses: "docker://node:14",
|
||||
WorkingDirectory: "workdir",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||
},
|
||||
}
|
||||
require.NoError(t, sd.Step.RawWith.Encode(map[string]string{"args": "identity:${{ github.action_repository }}:${{ github.action_ref }}"}))
|
||||
sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx)
|
||||
|
||||
cm.On("Pull", false).Return(noopExecutor)
|
||||
@@ -80,6 +84,11 @@ func TestStepDockerMain(t *testing.T) {
|
||||
require.NoError(t, sd.main()(ctx))
|
||||
|
||||
assert.Equal(t, "node:14", input.Image)
|
||||
assert.Equal(t, []string{"identity::"}, input.Cmd)
|
||||
assert.Contains(t, input.Env, "GITHUB_ACTION_REPOSITORY=")
|
||||
assert.Contains(t, input.Env, "GITHUB_ACTION_REF=")
|
||||
assert.Equal(t, "other/composite", sd.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sd.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
assert.Empty(t, input.Username)
|
||||
assert.Empty(t, input.Password)
|
||||
@@ -150,3 +159,33 @@ func TestStepDockerNewStepContainer(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWindowsHostContainerActions(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name, hostOS string
|
||||
jobContainer container.ExecutionsEnvironment
|
||||
wantError bool
|
||||
}{
|
||||
{name: "Windows host", hostOS: "windows", jobContainer: &container.HostEnvironment{}, wantError: true},
|
||||
{name: "Linux host", hostOS: "linux", jobContainer: &container.HostEnvironment{}},
|
||||
{name: "macOS host", hostOS: "darwin", jobContainer: &container.HostEnvironment{}},
|
||||
{name: "Windows with Linux job container", hostOS: "windows", jobContainer: &containerMock{}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
original := hostOS
|
||||
hostOS = test.hostOS
|
||||
t.Cleanup(func() { hostOS = original })
|
||||
rc := &RunContext{JobContainer: test.jobContainer, kubernetes: true}
|
||||
wantError := errKubernetesDocker.Error()
|
||||
if test.wantError {
|
||||
wantError = errWindowsHostContainer.Error()
|
||||
}
|
||||
t.Run("docker step", func(t *testing.T) {
|
||||
require.EqualError(t, (&stepDocker{RunContext: rc}).runUsesContainer()(t.Context()), wantError)
|
||||
})
|
||||
t.Run("container action", func(t *testing.T) {
|
||||
require.EqualError(t, execAsDocker(t.Context(), &stepActionRemote{RunContext: rc}, "action", "", "", false, stepStageMain), wantError)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ type stepFactoryImpl struct{}
|
||||
func (sf *stepFactoryImpl) newStep(stepModel *model.Step, rc *RunContext) (step, error) {
|
||||
switch stepModel.Type() {
|
||||
case model.StepTypeInvalid:
|
||||
return nil, fmt.Errorf("invalid run/uses syntax for job:%s step:%+v", rc.Run, stepModel)
|
||||
return nil, fmt.Errorf("invalid run/uses syntax for job:%s step:%+v", rc.Name, stepModel)
|
||||
case model.StepTypeRun:
|
||||
return &stepRun{
|
||||
Step: stepModel,
|
||||
@@ -40,7 +40,7 @@ func (sf *stepFactoryImpl) newStep(stepModel *model.Step, rc *RunContext) (step,
|
||||
run, ok := builtinActions[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown built-in action %q (known: %v) for job:%s step:%+v",
|
||||
name, slices.Sorted(maps.Keys(builtinActions)), rc.Run, stepModel)
|
||||
name, slices.Sorted(maps.Keys(builtinActions)), rc.Name, stepModel)
|
||||
}
|
||||
return &stepActionBuiltin{
|
||||
Step: stepModel,
|
||||
@@ -61,5 +61,5 @@ func (sf *stepFactoryImpl) newStep(stepModel *model.Step, rc *RunContext) (step,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unable to determine how to run job:%s step:%+v", rc.Run, stepModel)
|
||||
return nil, fmt.Errorf("unable to determine how to run job:%s step:%+v", rc.Name, stepModel)
|
||||
}
|
||||
@@ -37,7 +37,7 @@ func TestStepFactoryInvalidStep(t *testing.T) {
|
||||
{Uses: "remote/action@v1", Run: "cmd"},
|
||||
{Uses: "builtin:unknown"},
|
||||
} {
|
||||
_, err := (&stepFactoryImpl{}).newStep(stepModel, &RunContext{})
|
||||
assert.Error(t, err, stepModel.Uses)
|
||||
_, err := (&stepFactoryImpl{}).newStep(stepModel, &RunContext{Name: "build"})
|
||||
assert.ErrorContains(t, err, "job:build ", stepModel.Uses)
|
||||
}
|
||||
}
|
||||
+31
-6
@@ -100,7 +100,7 @@ func printRunActionHeader(ctx context.Context, step *model.Step, action *model.A
|
||||
maps.Copy(inputs, step.With)
|
||||
if action != nil {
|
||||
for name := range action.Inputs {
|
||||
if _, provided := lookupFold(step.With, name, true); !provided {
|
||||
if _, provided := common.LookupFold(step.With, name, true); !provided {
|
||||
inputs[name] = env[inputEnvKey(name)]
|
||||
}
|
||||
}
|
||||
@@ -222,10 +222,6 @@ func (sr *stepRun) getRunContext() *RunContext {
|
||||
return sr.RunContext
|
||||
}
|
||||
|
||||
func (sr *stepRun) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sr.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sr *stepRun) getStepModel() *model.Step {
|
||||
return sr.Step
|
||||
}
|
||||
@@ -296,7 +292,7 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string,
|
||||
name += ".sh"
|
||||
case "pwsh", "powershell":
|
||||
name += ".ps1"
|
||||
runPrepend = "$ErrorActionPreference = 'stop'"
|
||||
runPrepend = "$ErrorActionPreference = 'stop'\n$OutputEncoding = [System.Text.UTF8Encoding]::new($false)\ntry { [Console]::OutputEncoding = $OutputEncoding } catch {}"
|
||||
runAppend = "if ((Test-Path -LiteralPath variable:/LASTEXITCODE)) { exit $LASTEXITCODE }"
|
||||
case "cmd":
|
||||
name += ".cmd"
|
||||
@@ -306,6 +302,12 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string,
|
||||
}
|
||||
|
||||
script = fmt.Sprintf("%s\n%s\n%s", runPrepend, script, runAppend)
|
||||
switch shellCommand {
|
||||
case "powershell":
|
||||
script = "\ufeff" + script
|
||||
case "cmd":
|
||||
script = strings.ReplaceAll(strings.ReplaceAll(script, "\r\n", "\n"), "\n", "\r\n")
|
||||
}
|
||||
|
||||
if !strings.Contains(script, "::add-mask::") && !sr.RunContext.Config.InsecureSecrets {
|
||||
logger.Debugf("Wrote command \n%s\n to '%s'", script, name)
|
||||
@@ -315,7 +317,30 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string,
|
||||
|
||||
rc := sr.getRunContext()
|
||||
scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name)
|
||||
if step.Shell == "pwsh" || step.Shell == "powershell" {
|
||||
scriptPath = expressionStringEscape(scriptPath)
|
||||
}
|
||||
if step.Shell == "bash" || step.Shell == "sh" || step.Shell == "python" {
|
||||
if rc.isWindowsHost() {
|
||||
scriptPath = `"` + scriptPath + `"`
|
||||
} else {
|
||||
scriptPath = shellquote.Join(scriptPath)
|
||||
}
|
||||
}
|
||||
sr.cmdline = strings.ReplaceAll(scCmd, `{0}`, scriptPath)
|
||||
if rc.isWindowsHost() {
|
||||
command := strings.TrimSpace(sr.cmdline)
|
||||
if strings.HasPrefix(command, `"`) {
|
||||
executable, _, closed := strings.Cut(command[1:], `"`)
|
||||
if !closed || executable == "" {
|
||||
return "", "", fmt.Errorf("invalid shell option %q: expected a double-quoted executable", step.Shell)
|
||||
}
|
||||
sr.cmd = []string{executable}
|
||||
} else {
|
||||
sr.cmd = strings.Fields(command)[:1]
|
||||
}
|
||||
return name, script, nil
|
||||
}
|
||||
sr.cmd, err = shellquote.Split(sr.cmdline)
|
||||
|
||||
return name, script, err
|
||||
|
||||
+102
-25
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type shellContainerMock struct{ *containerMock }
|
||||
@@ -36,7 +38,6 @@ func TestStepRun(t *testing.T) {
|
||||
sr := &stepRun{
|
||||
RunContext: &RunContext{
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
ExprEval: &expressionEvaluator{},
|
||||
Config: &Config{Env: testCheckoutEnv()},
|
||||
Run: &model.Run{
|
||||
JobID: "1",
|
||||
@@ -48,32 +49,49 @@ func TestStepRun(t *testing.T) {
|
||||
},
|
||||
JobContainer: cm,
|
||||
jobRunDefaults: model.RunDefaults{Shell: "bash"},
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Run: "cmd",
|
||||
Run: "cmd${{ github.action_repository }}${{ github.action_ref }}",
|
||||
WorkingDirectory: "workdir",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||
},
|
||||
}
|
||||
sr.RunContext.ExprEval = sr.RunContext.NewExpressionEvaluator(t.Context())
|
||||
require.NoError(t, sr.Step.Env.Encode(map[string]string{
|
||||
"GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref",
|
||||
"GITEA_ACTION_REPOSITORY": "forged/repo", "GITEA_ACTION_REF": "forged-ref",
|
||||
}))
|
||||
|
||||
cm.On("Copy", "/var/run/act", []*container.FileEntry{fileEntry}).Return(noopExecutor)
|
||||
cm.On("Exec", []string{"bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "/var/run/act/workflow/1.sh"}, mock.AnythingOfType("map[string]string"), "", "workdir").Return(noopExecutor)
|
||||
|
||||
expectStepFileCommands(t, cm)
|
||||
require.NoError(t, sr.main()(t.Context()))
|
||||
assert.Empty(t, sr.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sr.env["GITHUB_ACTION_REF"])
|
||||
assert.Empty(t, sr.env["GITEA_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sr.env["GITEA_ACTION_REF"])
|
||||
assert.Equal(t, sr.env["GITHUB_STEP_SUMMARY"], sr.env["GITEA_STEP_SUMMARY"])
|
||||
assert.Equal(t, "other/composite", sr.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sr.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
cm.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestStepRunShellParity(t *testing.T) {
|
||||
tests := []struct {
|
||||
const powershellScript = "$ErrorActionPreference = 'stop'\n$OutputEncoding = [System.Text.UTF8Encoding]::new($false)\ntry { [Console]::OutputEncoding = $OutputEncoding } catch {}\necho 你好,世界\nif ((Test-Path -LiteralPath variable:/LASTEXITCODE)) { exit $LASTEXITCODE }"
|
||||
for _, test := range []struct {
|
||||
name, run, shell, workingDir string
|
||||
hostOS, wantCmdline string
|
||||
actPath string
|
||||
env map[string]string
|
||||
host bool
|
||||
probeErr error
|
||||
wantExt string
|
||||
wantCmd []string
|
||||
wantScript string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
@@ -91,24 +109,79 @@ func TestStepRunShellParity(t *testing.T) {
|
||||
wantCmd: []string{"sh", "-e", "/var/run/act/workflow/1.sh"},
|
||||
},
|
||||
{
|
||||
name: "builtin pwsh",
|
||||
shell: "pwsh",
|
||||
actPath: "/var/run/act with spaces",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"pwsh", "-command", ". '/var/run/act with spaces/workflow/1.ps1'"},
|
||||
name: "builtin pwsh",
|
||||
run: "echo 你好,世界",
|
||||
shell: "pwsh",
|
||||
actPath: "/var/run/act with spaces/O'Brien",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"pwsh", "-command", ". '/var/run/act with spaces/O''Brien/workflow/1.ps1'"},
|
||||
wantScript: powershellScript,
|
||||
},
|
||||
{
|
||||
name: "builtin powershell",
|
||||
shell: "powershell",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"powershell", "-command", ". '/var/run/act/workflow/1.ps1'"},
|
||||
name: "builtin powershell",
|
||||
actPath: "/var/run/act/O'Brien",
|
||||
run: "echo 你好,世界",
|
||||
shell: "powershell",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"powershell", "-command", ". '/var/run/act/O''Brien/workflow/1.ps1'"},
|
||||
wantScript: "\ufeff" + powershellScript,
|
||||
},
|
||||
{
|
||||
name: "custom pwsh template",
|
||||
shell: "pwsh -NoProfile -File {0}",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"pwsh", "-NoProfile", "-File", "/var/run/act/workflow/1.ps1"},
|
||||
name: "custom pwsh template",
|
||||
run: "echo 你好,世界",
|
||||
shell: "pwsh -NoProfile -File {0}",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"pwsh", "-NoProfile", "-File", "/var/run/act/workflow/1.ps1"},
|
||||
wantScript: powershellScript,
|
||||
},
|
||||
{
|
||||
name: "builtin cmd reads and emits UTF-8 without a BOM",
|
||||
run: "echo 你好,世界",
|
||||
shell: "cmd",
|
||||
wantExt: ".cmd",
|
||||
wantCmd: []string{"cmd", "/D", "/E:ON", "/V:OFF", "/S", "/C", "CALL /var/run/act/workflow/1.cmd"},
|
||||
wantScript: "@echo off\r\necho 你好,世界\r\n",
|
||||
},
|
||||
{
|
||||
name: "Windows custom executable preserves backslashes and raw arguments",
|
||||
shell: `C:\Python313\python.exe -X 'utf8' "{0}"`,
|
||||
actPath: `C:\runner\O'Brien`,
|
||||
hostOS: "windows",
|
||||
wantCmd: []string{`C:\Python313\python.exe`},
|
||||
wantCmdline: `C:\Python313\python.exe -X 'utf8' "{0}"`,
|
||||
},
|
||||
{
|
||||
name: "Windows quoted custom executable preserves spaces and backslashes",
|
||||
shell: `"C:\Program Files\Python313\python.exe" "{0}"`,
|
||||
actPath: `C:\runner`,
|
||||
hostOS: "windows",
|
||||
wantCmd: []string{`C:\Program Files\Python313\python.exe`},
|
||||
wantCmdline: `"C:\Program Files\Python313\python.exe" "{0}"`,
|
||||
},
|
||||
{
|
||||
name: "Windows PowerShell escapes apostrophes in the raw command line",
|
||||
run: "echo 你好,世界",
|
||||
shell: "pwsh",
|
||||
actPath: `C:\runner\O'Brien`,
|
||||
hostOS: "windows",
|
||||
wantExt: ".ps1",
|
||||
wantCmd: []string{"pwsh"},
|
||||
wantCmdline: `pwsh -command ". '{0}'"`,
|
||||
wantScript: powershellScript,
|
||||
},
|
||||
{
|
||||
name: "Windows unclosed executable quote",
|
||||
shell: `"C:\Program Files\python.exe {0}`,
|
||||
actPath: `C:\runner`,
|
||||
hostOS: "windows",
|
||||
wantErr: "expected a double-quoted executable",
|
||||
},
|
||||
{name: "linux builtin bash quotes script path", shell: "bash", actPath: `C:/Users/Jane Doe/O'Brien`, wantCmd: []string{"bash", "--noprofile", "--norc", "-e", "-o", "pipefail", `C:/Users/Jane Doe/O'Brien/workflow/1.sh`}},
|
||||
{name: "linux builtin sh quotes script path", shell: "sh", actPath: `C:/Users/Jane Doe/O'Brien`, wantCmd: []string{"sh", "-e", `C:/Users/Jane Doe/O'Brien/workflow/1.sh`}},
|
||||
{name: "linux builtin python quotes script path", shell: "python", actPath: `C:/Users/Jane Doe/O'Brien`, wantExt: ".py", wantCmd: []string{"python", `C:/Users/Jane Doe/O'Brien/workflow/1.py`}},
|
||||
{name: "windows builtin bash quotes script path", shell: "bash", hostOS: "windows", actPath: `C:\Users\Jane Doe\O'Brien`, wantExt: ".sh", wantCmd: []string{"bash"}, wantCmdline: `bash --noprofile --norc -e -o pipefail "{0}"`},
|
||||
{name: "windows builtin sh quotes script path", shell: "sh", hostOS: "windows", actPath: `C:\Users\Jane Doe\O'Brien`, wantExt: ".sh", wantCmd: []string{"sh"}, wantCmdline: `sh -e "{0}"`},
|
||||
{name: "windows builtin python quotes script path", shell: "python", hostOS: "windows", actPath: `C:\Users\Jane Doe\O'Brien`, wantExt: ".py", wantCmd: []string{"python"}, wantCmdline: `python "{0}"`},
|
||||
{
|
||||
name: "missing placeholder",
|
||||
shell: "bash -e",
|
||||
@@ -142,9 +215,7 @@ func TestStepRunShellParity(t *testing.T) {
|
||||
workingDir: "${{ DIR }}",
|
||||
wantErr: "unable to interpolate the working directory:",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cm := &containerMock{}
|
||||
var jobContainer container.ExecutionsEnvironment = &shellContainerMock{cm}
|
||||
@@ -175,22 +246,28 @@ func TestStepRunShellParity(t *testing.T) {
|
||||
env: test.env,
|
||||
}
|
||||
|
||||
original := hostOS
|
||||
hostOS = cmp.Or(test.hostOS, "linux")
|
||||
t.Cleanup(func() { hostOS = original })
|
||||
name, script, err := sr.setupShellCommand(t.Context())
|
||||
if test.wantErr != "" {
|
||||
require.ErrorContains(t, err, test.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
if test.wantExt == "" {
|
||||
if test.wantExt == "" && test.hostOS != "windows" {
|
||||
test.wantExt = ".sh"
|
||||
}
|
||||
wantScript := "\necho hi\n"
|
||||
if test.wantExt == ".ps1" {
|
||||
wantScript = "$ErrorActionPreference = 'stop'\necho hi\nif ((Test-Path -LiteralPath variable:/LASTEXITCODE)) { exit $LASTEXITCODE }"
|
||||
}
|
||||
assert.Equal(t, "workflow/1"+test.wantExt, name)
|
||||
assert.Equal(t, wantScript, script)
|
||||
assert.Equal(t, cmp.Or(test.wantScript, "\necho hi\n"), script)
|
||||
assert.Equal(t, test.wantCmd, sr.cmd)
|
||||
if test.wantCmdline != "" {
|
||||
scriptPath := jobContainer.GetActPath() + "/workflow/1" + test.wantExt
|
||||
if test.shell == "pwsh" || test.shell == "powershell" {
|
||||
scriptPath = strings.ReplaceAll(scriptPath, "'", "''")
|
||||
}
|
||||
assert.Equal(t, strings.ReplaceAll(test.wantCmdline, "{0}", scriptPath), sr.cmdline)
|
||||
}
|
||||
assert.Equal(t, test.env["DIR"], sr.WorkingDirectory)
|
||||
cm.AssertExpectations(t)
|
||||
})
|
||||
|
||||
+45
-11
@@ -155,18 +155,14 @@ func TestSetupEnv(t *testing.T) {
|
||||
require.NoError(t, setupEnv(t.Context(), testStep))
|
||||
require.NoError(t, setupInputs(t.Context(), testStep))
|
||||
|
||||
delete(env, "GITHUB_REF")
|
||||
delete(env, "GITHUB_REF_NAME")
|
||||
delete(env, "GITHUB_REF_TYPE")
|
||||
delete(env, "GITHUB_SHA")
|
||||
delete(env, "GITHUB_WORKSPACE")
|
||||
delete(env, "GITHUB_REPOSITORY")
|
||||
delete(env, "GITHUB_REPOSITORY_OWNER")
|
||||
delete(env, "GITHUB_ACTOR")
|
||||
for _, suffix := range []string{"REF", "REF_NAME", "REF_TYPE", "SHA", "WORKSPACE", "REPOSITORY", "REPOSITORY_OWNER", "ACTOR"} {
|
||||
delete(env, "GITHUB_"+suffix)
|
||||
delete(env, "GITEA_"+suffix)
|
||||
}
|
||||
delete(env, "RUNNER_NAME")
|
||||
delete(env, "RUNNER_WORKSPACE")
|
||||
|
||||
assert.Equal(t, map[string]string{
|
||||
expected := map[string]string{
|
||||
"ACT": "true",
|
||||
"ACT_SKIP_CHECKOUT": "true",
|
||||
"CI": "true",
|
||||
@@ -192,8 +188,23 @@ func TestSetupEnv(t *testing.T) {
|
||||
"RC_KEY": "rcvalue",
|
||||
"RUNNER_ENVIRONMENT": "self-hosted",
|
||||
"RUNNER_PERFLOG": "/dev/null",
|
||||
"RUNNER_TRACKING_ID": "",
|
||||
}, env)
|
||||
"RUNNER_TRACKING_ID": env["RUNNER_TRACKING_ID"],
|
||||
}
|
||||
assert.Regexp(t, `^gitea_[A-Z2-7]{26}$`, env["RUNNER_TRACKING_ID"])
|
||||
for name, value := range expected {
|
||||
if suffix, ok := strings.CutPrefix(name, "GITHUB_"); ok {
|
||||
expected["GITEA_"+suffix] = value
|
||||
}
|
||||
}
|
||||
assert.Equal(t, expected, env)
|
||||
|
||||
t.Run("keeps an inherited RUNNER_TRACKING_ID opt-out", func(t *testing.T) {
|
||||
rc.Env["RUNNER_TRACKING_ID"] = ""
|
||||
optedOut := map[string]string{}
|
||||
require.NoError(t, setupEnv(t.Context(), &stepRun{RunContext: rc, Step: step, env: optedOut}))
|
||||
assert.Contains(t, optedOut, "RUNNER_TRACKING_ID")
|
||||
assert.Empty(t, optedOut["RUNNER_TRACKING_ID"])
|
||||
})
|
||||
|
||||
for _, expression := range []string{"inputs.args", "env.ARGS"} {
|
||||
t.Run("deferred inputs from "+expression, func(t *testing.T) {
|
||||
@@ -408,6 +419,29 @@ func (c *fileCommandTestContainer) GetContainerArchive(ctx context.Context, srcP
|
||||
return c.HostEnvironment.GetContainerArchive(ctx, srcPath)
|
||||
}
|
||||
|
||||
type windowsFileCommandTestContainer struct {
|
||||
*container.HostEnvironment
|
||||
}
|
||||
|
||||
func (*windowsFileCommandTestContainer) IsEnvironmentCaseInsensitive() bool { return true }
|
||||
|
||||
func TestProcessRunnerEnvFileCommandOrder(t *testing.T) {
|
||||
rc := &RunContext{
|
||||
JobContainer: &windowsFileCommandTestContainer{&container.HostEnvironment{ActPath: t.TempDir()}},
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rc.JobContainer.GetActPath(), "envs.txt"), []byte("BUILD_MODE=first\nbuild_mode<<EOF\nmiddle\nEOF\nBUILD_MODE=last\n"), 0o600))
|
||||
var entries [][2]string
|
||||
require.NoError(t, processRunnerEnvFileCommand(t.Context(), "envs.txt", rc, func(ctx context.Context, params map[string]string, value string) {
|
||||
entries = append(entries, [2]string{params["name"], value})
|
||||
rc.setEnvFile(ctx, params, value)
|
||||
}))
|
||||
assert.Equal(t, [][2]string{
|
||||
{"BUILD_MODE", "first"}, {"build_mode", "middle"}, {"BUILD_MODE", "last"},
|
||||
}, entries)
|
||||
assert.Equal(t, map[string]string{"BUILD_MODE": "last"}, rc.Env)
|
||||
assert.Equal(t, rc.Env, rc.GlobalEnv)
|
||||
}
|
||||
|
||||
func TestRunStepExecutorParity(t *testing.T) {
|
||||
newStep := func(t *testing.T, stepModel *model.Step) *stepRun {
|
||||
rc := createRunContext(t)
|
||||
|
||||
Loaded 100 of 186 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user