mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 06:45:44 +00:00
Compare commits
41
Commits
v5.2.0
..
lean-tests
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e6bf67d7d | ||
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 | ||
|
|
f2fbd67962 | ||
|
|
d6f5b68a0d | ||
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 | ||
|
|
c719a88f36 | ||
|
|
d882311ec6 | ||
|
|
923a0bfb59 | ||
|
|
6f0d4f1b48 | ||
|
|
430f7bb2cf | ||
|
|
befb1b4bb9 | ||
|
|
1a7f09224f |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
||||
|
||||
/bin
|
||||
/dist
|
||||
/.tmp
|
||||
/.git/hooks
|
||||
/.git/logs
|
||||
/.git/rr-cache
|
||||
/.env
|
||||
/.runner
|
||||
/.runner.lock
|
||||
/config.yaml
|
||||
/Dockerfile
|
||||
.DS_Store
|
||||
@@ -0,0 +1,29 @@
|
||||
name: go-setup
|
||||
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
id: setup-go
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: false
|
||||
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||
@@ -0,0 +1,26 @@
|
||||
name: cache-seeder
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- go.mod
|
||||
- go.sum
|
||||
- .golangci.yml
|
||||
- Makefile
|
||||
- .gitea/actions/go-setup/action.yml
|
||||
- .gitea/workflows/cache-seeder.yml
|
||||
|
||||
jobs:
|
||||
seed:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- run: make deps-tools lint build
|
||||
- name: compile the test binaries without running them
|
||||
run: |
|
||||
make test GOTEST_FLAGS='-race -exec true'
|
||||
go test -tags e2e -exec true ./e2e/...
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
node-version: 26
|
||||
- run: make lint-pr-title
|
||||
env:
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
@@ -11,7 +11,6 @@ on:
|
||||
|
||||
env:
|
||||
DOCKER_ORG: gitea
|
||||
DOCKER_LATEST: nightly
|
||||
|
||||
jobs:
|
||||
goreleaser:
|
||||
@@ -35,8 +34,15 @@ jobs:
|
||||
with:
|
||||
go-version-file: "go.mod"
|
||||
check-latest: true
|
||||
- name: Import GPG key
|
||||
id: import_gpg
|
||||
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||
with:
|
||||
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||
passphrase: ${{ secrets.PASSPHRASE }}
|
||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||
- name: goreleaser
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
||||
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||
with:
|
||||
distribution: goreleaser-pro
|
||||
args: release --nightly
|
||||
@@ -48,6 +54,7 @@ jobs:
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
GORELEASER_FORCE_TOKEN: "gitea"
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||
|
||||
release-image:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -80,7 +87,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
passphrase: ${{ secrets.PASSPHRASE }}
|
||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||
- name: goreleaser
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
||||
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||
with:
|
||||
distribution: goreleaser-pro
|
||||
args: release
|
||||
@@ -69,7 +69,6 @@ jobs:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
env:
|
||||
DOCKER_ORG: gitea
|
||||
DOCKER_LATEST: latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
@@ -83,7 +82,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
+37
-15
@@ -17,25 +17,24 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
check-latest: true
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- name: prepare anonymous docker config
|
||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
||||
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||
- name: pre-pull test images
|
||||
env:
|
||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:d6aa754f16b3197301076f047b5def2f02ea1dbbc2ca920407d46d7ec7f87b20 # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
||||
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||
run: |
|
||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||
for attempt in 1 2 3; do
|
||||
docker pull "$image" && break
|
||||
[ "$attempt" = 3 ] || sleep 5
|
||||
done
|
||||
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||
for attempt in 1 2 3; do
|
||||
docker pull "$image" && break
|
||||
[ "$attempt" = 3 ] || sleep 5
|
||||
done
|
||||
fi
|
||||
docker tag "$image" "${image%@*}"
|
||||
done
|
||||
- name: lint
|
||||
@@ -58,6 +57,32 @@ jobs:
|
||||
make coverage-report
|
||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
windows:
|
||||
name: test on windows
|
||||
runs-on: ubuntu-latest-kvm
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- name: prepare anonymous docker config
|
||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||
- name: windows vm cache key
|
||||
id: windows-vm
|
||||
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
- name: test on windows
|
||||
id: test-windows
|
||||
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||
env:
|
||||
WINDOWS_TEST_TIMEOUT: 45
|
||||
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
|
||||
e2e:
|
||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||
runs-on: ubuntu-latest
|
||||
@@ -68,10 +93,7 @@ jobs:
|
||||
- gitea/gitea:main-nightly
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
check-latest: true
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- name: prepare anonymous docker config
|
||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||
- name: e2e compatibility
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/gitea-runner
|
||||
/gitea-runner.exe
|
||||
.env
|
||||
!/act/runner/testdata/secrets/.env
|
||||
.runner
|
||||
|
||||
@@ -39,6 +39,8 @@ linters:
|
||||
deny:
|
||||
- pkg: golang.org/x/exp
|
||||
desc: it's experimental and unreliable
|
||||
- pkg: unsafe
|
||||
desc: use typed APIs instead
|
||||
logging:
|
||||
files:
|
||||
- $all
|
||||
|
||||
+5
-22
@@ -28,38 +28,16 @@ builds:
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: arm
|
||||
- goos: darwin
|
||||
goarch: ppc64le
|
||||
- goos: darwin
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: ppc64le
|
||||
- goos: windows
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "5"
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "6"
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "7"
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: ppc64le
|
||||
- goos: freebsd
|
||||
goarch: s390x
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "5"
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "6"
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "7"
|
||||
- goos: freebsd
|
||||
goarch: arm64
|
||||
flags:
|
||||
@@ -102,6 +80,7 @@ builds:
|
||||
publishers:
|
||||
- name: cloudflare-r2
|
||||
checksum: true
|
||||
signature: true
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
- glob: ./**.sha256
|
||||
@@ -122,6 +101,10 @@ checksum:
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
|
||||
signs:
|
||||
- artifacts: checksum
|
||||
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||
|
||||
snapshot:
|
||||
version_template: "{{ .Branch }}-devel"
|
||||
|
||||
|
||||
+22
-4
@@ -28,8 +28,8 @@ depending on the prefix:
|
||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
||||
encoded forms too.
|
||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
|
||||
decodes exactly those two when folding a location into an annotation.
|
||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||
|
||||
## Logging
|
||||
|
||||
@@ -57,7 +57,25 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
||||
|
||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||
parallel.
|
||||
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||
|
||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
||||
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||
Docker and Kubernetes.
|
||||
|
||||
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||
|
||||
## Windows tests
|
||||
|
||||
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
|
||||
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
|
||||
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||
+5
-3
@@ -12,12 +12,14 @@ ENV GOPROXY=${GOPROXY:-}
|
||||
COPY . /opt/src/runner
|
||||
WORKDIR /opt/src/runner
|
||||
|
||||
RUN make clean && make build
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
make clean && make build
|
||||
|
||||
### DIND VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind AS dind
|
||||
FROM docker:29.9.2-dind AS dind
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -39,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
### DIND-ROOTLESS VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
||||
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
DIST := dist
|
||||
EXECUTABLE := gitea-runner
|
||||
GO ?= go
|
||||
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||
|
||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||
|
||||
DOCKER_IMAGE ?= gitea/runner
|
||||
DOCKER_TAG ?= nightly
|
||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
||||
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||
|
||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||
|
||||
@@ -34,15 +34,12 @@ endif
|
||||
|
||||
ifeq ($(OS), Windows_NT)
|
||||
GOFLAGS := -v -buildmode=exe
|
||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||
else ifeq ($(OS), Windows)
|
||||
GOFLAGS := -v -buildmode=exe
|
||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||
else
|
||||
GOFLAGS := -v
|
||||
EXECUTABLE ?= $(EXECUTABLE)
|
||||
GO_ENV_WINDOWS := GOOS=windows
|
||||
endif
|
||||
|
||||
@@ -154,11 +151,15 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||
|
||||
.PHONY: test-windows
|
||||
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||
|
||||
E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||
|
||||
.PHONY: install
|
||||
install: $(GOFILES) ## install the runner binary via `go install`
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
# Gitea Runner
|
||||
|
||||
Gitea Runner runs the jobs of Gitea Actions workflows. Register it with a Gitea instance to run jobs in Docker containers, Kubernetes pods or directly on the host.
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
On the runner:
|
||||
- Linux, macOS or Windows, or FreeBSD.
|
||||
- `git` 2.34.1 or newer.
|
||||
- `docker`, or `podman` with `podman.socket` enabled, for `docker` labels and Docker actions. See Docker's [install instructions](https://docs.docker.com/engine/install/).
|
||||
|
||||
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
|
||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
In the job image, or on the host for host jobs:
|
||||
|
||||
Where steps run, in the job image or on the runner in host mode:
|
||||
|
||||
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
|
||||
- `bash` or `sh`, or `pwsh` or `powershell` on Windows.
|
||||
- `node` for JavaScript actions and `hashFiles()`.
|
||||
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
|
||||
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
|
||||
- `git-lfs` for `builtin:checkout` with `lfs: true`.
|
||||
- `ssh` for `builtin:checkout` with `ssh-key`.
|
||||
|
||||
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
|
||||
|
||||
### Download pre-built binary
|
||||
|
||||
Visit [here](https://dl.gitea.com/gitea-runner/) and download the right version for your platform.
|
||||
Download the binary for your platform from [dl.gitea.com](https://dl.gitea.com/gitea-runner/).
|
||||
|
||||
### Build from source
|
||||
|
||||
@@ -36,7 +38,7 @@ make docker
|
||||
|
||||
## Quickstart
|
||||
|
||||
Actions are disabled by default, so you need to add the following to the configuration file of your Gitea instance to enable it:
|
||||
Enable Actions in the configuration file of your Gitea instance:
|
||||
|
||||
```ini
|
||||
[actions]
|
||||
@@ -49,70 +51,72 @@ ENABLED=true
|
||||
./gitea-runner register
|
||||
```
|
||||
|
||||
And you will be asked to input:
|
||||
It asks for:
|
||||
|
||||
1. Gitea instance URL, like `http://192.168.8.8:3000/`. You should use your gitea instance ROOT_URL as the instance argument
|
||||
and you should not use `localhost` or `127.0.0.1` as instance IP;
|
||||
2. Runner token, you can get it from `http://192.168.8.8:3000/admin/actions/runners`;
|
||||
3. Runner name, you can just leave it blank;
|
||||
4. Runner labels, you can just leave it blank.
|
||||
1. The Gitea instance URL. Use its `ROOT_URL`, not `localhost` or `127.0.0.1`.
|
||||
2. A runner token, from `/admin/actions/runners` on your instance.
|
||||
3. A runner name, the hostname when left blank.
|
||||
4. The runner [labels](#labels), the defaults when left blank.
|
||||
|
||||
The process looks like:
|
||||
|
||||
```text
|
||||
INFO Registering runner, arch=amd64, os=darwin, version=0.1.5.
|
||||
WARN Runner in user-mode.
|
||||
INFO Enter the Gitea instance URL (for example, https://gitea.com/):
|
||||
http://192.168.8.8:3000/
|
||||
INFO Enter the runner token:
|
||||
fe884e8027dc292970d4e0303fe82b14xxxxxxxx
|
||||
INFO Enter the runner name (if set empty, use hostname: Test.local):
|
||||
|
||||
INFO Enter the runner labels, leave blank to use the default labels (comma-separated, for example, ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest):
|
||||
|
||||
INFO Registering runner, name=Test.local, instance=http://192.168.8.8:3000/, labels=[ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04 ubuntu-20.04:docker://docker.gitea.com/runner-images:ubuntu-20.04].
|
||||
DEBU Successfully pinged the Gitea instance server
|
||||
INFO Runner registered successfully.
|
||||
```
|
||||
|
||||
You can also register with command line arguments.
|
||||
To register without prompts:
|
||||
|
||||
```bash
|
||||
./gitea-runner register --instance http://192.168.8.8:3000 --token <my_runner_token> --no-interactive
|
||||
./gitea-runner register --instance https://gitea.example.com --token <token> --no-interactive
|
||||
```
|
||||
|
||||
If the registry succeed, it will run immediately. Next time, you could run the runner directly.
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
./gitea-runner daemon
|
||||
```
|
||||
|
||||
## Deployment
|
||||
|
||||
### Windows service
|
||||
|
||||
Register the runner first, keeping `gitea-runner.exe`, `config.yaml` and the `.runner` file in one directory. Then install and start the service from an elevated Command Prompt:
|
||||
|
||||
```bat
|
||||
sc.exe create gitea-runner binPath= "\"C:\gitea-runner\gitea-runner.exe\" daemon --config \"C:\gitea-runner\config.yaml\"" start= auto
|
||||
sc.exe start gitea-runner
|
||||
```
|
||||
|
||||
The service logs to `gitea-runner.log` next to the executable. Its account needs write access to that directory and access to the tools jobs use.
|
||||
|
||||
`sc.exe stop gitea-runner` lets running jobs finish within `runner.shutdown_timeout`. To allow the same on shutdown, raise the service's preshutdown timeout above it, here to six minutes. This takes effect after the next reboot:
|
||||
|
||||
```bat
|
||||
reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\gitea-runner /v PreshutdownTimeout /t REG_DWORD /d 360000 /f
|
||||
```
|
||||
|
||||
### Run with docker
|
||||
|
||||
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
|
||||
|
||||
```bash
|
||||
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:latest-dind
|
||||
```
|
||||
|
||||
The volumes keep the runner's registration and the job image cache across container recreation.
|
||||
The volumes keep the registration and the image cache across container recreation. The image reads more variables, such as `GITEA_RUNNER_LABELS` and `CONFIG_FILE`, see [scripts/run.sh](scripts/run.sh) and the [Docker example](examples/docker/README.md).
|
||||
|
||||
### Image flavours
|
||||
#### Image flavours
|
||||
|
||||
| Tag | Docker daemon for jobs |
|
||||
| --- | --- |
|
||||
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as UID 1000 for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Its image cache is in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's `/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
|
||||
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
|
||||
Versioned and `nightly` tags take the same suffixes, like `nightly-dind`. The images target Docker, not Podman.
|
||||
|
||||
### Configuration
|
||||
### Example deployments
|
||||
|
||||
The runner reads a YAML file. Without one, every option keeps its default.
|
||||
See the [examples](examples) directory for more deployments, among them systemd, Kubernetes and VMs.
|
||||
|
||||
## Configuration
|
||||
|
||||
The runner reads an optional YAML file:
|
||||
|
||||
```bash
|
||||
./gitea-runner config init # write config.yaml, with no option set
|
||||
@@ -120,37 +124,68 @@ The runner reads a YAML file. Without one, every option keeps its default.
|
||||
./gitea-runner -c config.yaml daemon # -c also works on register and cache-server
|
||||
```
|
||||
|
||||
`config generate` prints [config.example.yaml](internal/pkg/config/config.example.yaml). Every value in it is commented out, so copy the lines you want to change into your own file and uncomment them.
|
||||
`config generate` prints [config.example.yaml](internal/pkg/config/config.example.yaml), which documents every option. Copy the lines you need into your file and uncomment them.
|
||||
|
||||
#### Editing a config file
|
||||
### Editing a config file
|
||||
|
||||
`config` edits a file in place, which is handy in provisioning scripts:
|
||||
`config` edits a file in place and keeps its comments, which suits provisioning scripts:
|
||||
|
||||
```bash
|
||||
./gitea-runner config set runner.capacity 4
|
||||
./gitea-runner config set runner.timeout 90m # written as 1h30m0s
|
||||
./gitea-runner config set runner.envs.MY_VAR value
|
||||
./gitea-runner config add runner.labels 'ubuntu:docker://node:22'
|
||||
./gitea-runner config remove runner.labels 'ubuntu:docker://node:22'
|
||||
./gitea-runner config get runner.labels
|
||||
```
|
||||
|
||||
A key is its dotted YAML path. An unknown key, a value of the wrong type, or `add`/`remove` on anything but a list is refused before the file is touched. `set` replaces a whole list when you give it several values.
|
||||
Keys are dotted YAML paths. Unknown keys and invalid values are refused without touching the file. Without `-c`, it uses `config.yaml` from the working directory, then from the binary's directory.
|
||||
|
||||
An edit keeps the comments and the key order of the file. Indentation becomes two spaces, and a blank line between two values is dropped.
|
||||
### Environment variables
|
||||
|
||||
`config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path.
|
||||
Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset variable fails the load, and `$${NAME}` stays a literal `${NAME}`.
|
||||
|
||||
#### Tool cache
|
||||
The override variables of earlier releases, such as `GITEA_RUNNER_CAPACITY`, are gone. Set those options in the file instead.
|
||||
|
||||
Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`, which is `/opt/hostedtoolcache` inside a job. `runner.tool_cache_mode` selects what backs it:
|
||||
### Labels
|
||||
|
||||
| Mode | Tool cache | Trade-off |
|
||||
| --- | --- | --- |
|
||||
| `none` (default) | Per job, provided by the job image | A version the image lacks is downloaded in every job |
|
||||
| `shared` | One volume reused by every job | Two jobs writing the same tool version at once corrupt it, so use it only with `runner.capacity: 1` |
|
||||
Labels decide which jobs a runner accepts and how it runs them. A job's `runs-on` is matched against the label names, and the first match picks where it runs.
|
||||
|
||||
With `none`, tools must come from the job image. Install them into `/opt/hostedtoolcache/<tool>/<version>/<arch>`, with an empty `<arch>.complete` file next to the directory:
|
||||
A label is written as `<name>[:<schema>[:<args>]]`, with one of these schemas:
|
||||
|
||||
- `docker://<image>` runs the job in a container of `<image>`.
|
||||
- `kubernetes://<image>` runs it as a Kubernetes pod, see [docs/kubernetes.md](docs/kubernetes.md).
|
||||
- `host`, the default, runs its steps directly on the runner's machine.
|
||||
|
||||
With these labels, `runs-on: ubuntu-latest` runs in a container and `runs-on: macos` on the host:
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
```
|
||||
|
||||
[gitea/runner-images](https://gitea.com/gitea/runner-images) has suitable images. Jobs matching no label run in `runner.default_image`, or on the host when docker is unavailable.
|
||||
|
||||
Host jobs still need a Docker daemon for Docker actions and `docker://` steps. Windows hosts do not support those, as on GitHub. A step's background processes end with the job unless it opts out, see [host mode](docs/host-mode.md).
|
||||
|
||||
#### Registration vs config labels
|
||||
|
||||
Labels are set at registration, with `--labels` or the prompt. To change them later, use one of these, the leftmost set one wins:
|
||||
|
||||
```
|
||||
daemon --labels / GITEA_RUNNER_LABELS > runner.labels in the config file > labels in the .runner file
|
||||
```
|
||||
|
||||
Changed labels are saved to the `.runner` file and sent to Gitea on startup. When `runner.labels` is set, `register` uses it and ignores `--labels`.
|
||||
|
||||
### Tool cache
|
||||
|
||||
Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`. `runner.tool_cache_mode` selects what backs it:
|
||||
|
||||
| Mode | Tool cache |
|
||||
| --- | --- |
|
||||
| `none` (default) | Provided by the job image. Versions it lacks are downloaded in every job. |
|
||||
| `shared` | One volume for all jobs. Only safe with `runner.capacity: 1`. |
|
||||
|
||||
To ship tools in a job image, install them to `/opt/hostedtoolcache/<tool>/<version>/<arch>` with an empty `<arch>.complete` file next to it, as the `-full` [runner images](https://gitea.com/gitea/runner-images) do:
|
||||
|
||||
```dockerfile
|
||||
RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1\.26\.[0-9]*' | head -1); \
|
||||
@@ -161,83 +196,29 @@ RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1
|
||||
touch "${DIR}.complete"
|
||||
```
|
||||
|
||||
A workflow requesting a minor version, `go-version: "1.26"`, resolves to the newest matching version in the cache, so a patch update in the image still hits it.
|
||||
### Proxy
|
||||
|
||||
Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavour is the one that ships tools in this layout.
|
||||
Set the proxy in the runner's environment, for example with systemd `Environment=` or `docker run -e`:
|
||||
|
||||
`gitea-runner exec` reads no config file and takes `--tool-cache-mode` instead, defaulting to `none`.
|
||||
|
||||
#### Environment variables
|
||||
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
|
||||
### Labels
|
||||
|
||||
Labels decide **which jobs a runner accepts** and **how it runs them**. A job's `runs-on` is matched against the runner's label names; the first match wins and selects the execution environment for that job.
|
||||
|
||||
A label is written as:
|
||||
|
||||
```text
|
||||
<name>[:<schema>[:<args>]]
|
||||
```sh
|
||||
http_proxy=http://proxy.example:3128
|
||||
https_proxy=http://proxy.example:3128
|
||||
no_proxy=gitea.internal,.example.local
|
||||
```
|
||||
|
||||
| Part | Meaning |
|
||||
| --- | --- |
|
||||
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
|
||||
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
|
||||
The runner uses it and passes it on to jobs. Jobs always reach the cache server, `localhost`, their service containers and a `tcp://` Docker daemon directly. Add Gitea to `no_proxy` if it should be reached directly too.
|
||||
|
||||
Three schemas are supported:
|
||||
To change the proxy for jobs, use `runner.envs`, a step's `env:` or `$GITHUB_ENV`. Workflow and job level `env:` have no effect on it.
|
||||
|
||||
- **`docker://<image>`** — the job runs inside a container created from `<image>`:
|
||||
The Docker daemon pulls images with its own proxy settings. The `dind` images pass the variables above to it, for other daemons see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/).
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
A password in the proxy URL is hidden in logs, but every step can read it.
|
||||
|
||||
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
|
||||
## Jobs
|
||||
|
||||
```text
|
||||
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
### Service containers
|
||||
|
||||
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
|
||||
|
||||
```text
|
||||
macos:host
|
||||
```
|
||||
|
||||
So with the labels
|
||||
|
||||
```text
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
```
|
||||
|
||||
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
|
||||
|
||||
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
|
||||
|
||||
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
|
||||
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
|
||||
|
||||
#### Registration vs config labels
|
||||
|
||||
If `runner.labels` is set in the YAML file, those labels are used during `register` and the `--labels` CLI flag is ignored.
|
||||
|
||||
The `daemon` command also accepts `--labels` (which defaults to the `GITEA_RUNNER_LABELS` environment variable), so the labels of an already registered runner can be changed without deleting its registration file. The most explicit source wins:
|
||||
|
||||
```
|
||||
--labels / GITEA_RUNNER_LABELS > runner.labels in the config file > labels in the .runner file
|
||||
```
|
||||
|
||||
Whenever the resulting labels differ from the ones in the registration file, they are written back to it and re-declared to the Gitea instance on startup.
|
||||
|
||||
> **Note:** A runner that only exposes `host` labels still needs access to a Docker daemon (e.g. a mounted `/var/run/docker.sock`) whenever a job uses a `docker://` action or a service container. `host` labels only change where the job's own steps run; container-based steps and actions are still executed with Docker.
|
||||
|
||||
#### Service containers
|
||||
|
||||
A job's `services` are started before its steps run. When a service's image or its `options` declare a healthcheck, the runner waits for it to report healthy, so a workflow does not have to poll for its own services:
|
||||
When a service declares a healthcheck, in its image or `options`, the job waits for it to become healthy:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
@@ -249,63 +230,32 @@ services:
|
||||
--health-retries 10
|
||||
```
|
||||
|
||||
A service that reports unhealthy fails the job right away, with its container log. One that never becomes healthy fails it after `container.service_ready_timeout` (default `5m`, negative disables the wait). A service that exits without declaring a healthcheck only gets its log and a warning.
|
||||
A service that turns unhealthy, or is not healthy within `container.service_ready_timeout`, fails the job. Steps reach a service by its id and port, like `psql -h postgres -p 5432`.
|
||||
|
||||
A job in a container reaches a service by its id on the job network, on the port the service listens on, for example `psql -h postgres -p 5432`. The started containers also fill the `job` context: `job.container.{id,network}` and `job.services.<id>.{id,network,ports}`, where `ports` maps a container port to the host port Docker published it on, for the services that publish one.
|
||||
Unlike GitHub, jobs running on the host start no services. Give them a `container:` to use services.
|
||||
|
||||
Unlike GitHub, a job whose steps run on the host (a `host` label without `container:`) starts no service containers, so `job.services` and `job.container` stay empty. Give such a job a `container:` when it needs services.
|
||||
### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
|
||||
|
||||
#### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
|
||||
|
||||
Containers a job starts through the Docker socket can bind-mount the workspace by the job's own path, as on a host, for example `docker run -v "$PWD:/src"`. Without the Docker proxy, for example with a remote `DOCKER_HOST`, use `GITEA_DOCKER_WORKSPACE`, the path the daemon sees, as the prefix of workspace binds, with `.` as the fallback for local use:
|
||||
Containers a job starts can bind-mount the workspace by its path, like `docker run -v "$PWD:/src"`. Where the Docker daemon sees another path, for example a remote `DOCKER_HOST`, prefix the bind with `GITEA_DOCKER_WORKSPACE`:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
|
||||
In most Linux container jobs, the containers, networks and volumes a job creates are removed when it ends, `docker compose` services included. Set `container.sweep: false` if jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
#### Proxy
|
||||
### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
|
||||
`uses: builtin:checkout` checks out a repository without downloading `actions/checkout` or needing Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md).
|
||||
|
||||
```sh
|
||||
http_proxy=http://proxy.example:3128
|
||||
https_proxy=http://proxy.example:3128
|
||||
no_proxy=gitea.internal,.example.local
|
||||
```
|
||||
### Caching (`actions/cache`)
|
||||
|
||||
The runner uses them for its own requests and gives them to every job, in lower and upper case.
|
||||
Each runner runs its own cache server. As on GitHub, a job restores caches of its own ref, its pull request's base branch and the default branch.
|
||||
|
||||
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
|
||||
#### Sharing a cache between runners
|
||||
|
||||
- the built-in cache server, whose address is assigned at startup
|
||||
- `localhost`, `127.0.0.1` and `::1`
|
||||
- the job's service containers
|
||||
- the Docker daemon, when it is reached over `tcp://`
|
||||
|
||||
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
|
||||
|
||||
For one step, set it in the step's `env:`, for the rest of a job, write it to `$GITHUB_ENV`. Setting it at workflow or job level or in `container.env` has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
|
||||
|
||||
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
|
||||
|
||||
Dockerfile actions are built with these variables as build arguments, so their `RUN` steps can reach the network.
|
||||
|
||||
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
|
||||
|
||||
#### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
Use `uses: builtin:checkout` to check out a repository without downloading `actions/checkout` or needing Node in the job image, see **[docs/builtin-actions.md](docs/builtin-actions.md)**.
|
||||
|
||||
#### Caching (`actions/cache`)
|
||||
|
||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||
|
||||
**Sharing a cache between runners**
|
||||
|
||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||
Run one `gitea-runner cache-server` and point every runner at it with the same secret, for example from `openssl rand -hex 32`:
|
||||
|
||||
```yaml
|
||||
# cache-server.yaml
|
||||
@@ -326,13 +276,11 @@ cache:
|
||||
external_secret: "<secret>"
|
||||
```
|
||||
|
||||
`--dir`, `--host` and `--port` override the matching `cache` keys. Eviction settings take effect on the cache server, not on the runners.
|
||||
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower requests.
|
||||
|
||||
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower cache requests.
|
||||
#### Storing the cache in an S3-compatible bucket
|
||||
|
||||
**Storing the cache in an S3-compatible bucket**
|
||||
|
||||
The cache can also be stored in an S3-compatible bucket such as AWS S3, MinIO or RustFS, with no cache server. Runners with the same settings use the same cache:
|
||||
Runners can share a cache in an S3-compatible bucket, such as AWS S3, MinIO or RustFS, without a cache server:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
@@ -346,66 +294,48 @@ cache:
|
||||
secret_key_file: "/run/secrets/cache-s3-secret-key"
|
||||
```
|
||||
|
||||
A cache saved on one runner can be restored on the others within 30 seconds. The eviction settings below do not apply to the bucket, so add lifecycle rules to it that expire old caches and abort incomplete multipart uploads.
|
||||
Eviction does not apply to the bucket. Add lifecycle rules that expire old caches and abort incomplete multipart uploads.
|
||||
|
||||
**Eviction**
|
||||
#### Eviction
|
||||
|
||||
Entries not read or written within `retention` (default `168h`) are removed. A repository over `repo_size_limit` (default `10GB`) loses its least recently used entries, and `size_limit` (off by default) caps the whole cache the same way. Entries in use are never removed. The cache also keeps 1024 MiB free on its volume, or `health_check.min_free_disk_space_mb` when health checks are enabled. See [config.example.yaml](internal/pkg/config/config.example.yaml) for all options.
|
||||
Entries unused for `cache.retention` (default `168h`) are removed. A repository over `cache.repo_size_limit` (default `10GB`) loses its least recently used entries. See [config.example.yaml](internal/pkg/config/config.example.yaml) for more limits.
|
||||
|
||||
**Cache service v2**
|
||||
#### Cache service v2
|
||||
|
||||
`actions/cache` v3.4.0, v4.2.0 and later use the cache service v2 API, which the runner serves by default. These actions fall back to v1 on hosts they do not recognize as GitHub, so the runner removes that check from them while a job runs. This also lets the stock `actions/upload-artifact` v4.4.0 and `actions/download-artifact` v4.1.8 and later work without the `gitea-upload-artifact` fork. With `runner.patch_actions: false`, the cache stays on v1 and the artifact actions fail.
|
||||
The runner serves the cache service v2 API used by current `actions/cache` releases. It patches them to use it on Gitea, which also makes the stock `actions/upload-artifact` and `actions/download-artifact` v4 work. Artifact calls then go through the cache server, so jobs must be able to reach it.
|
||||
|
||||
With v2, artifact calls also go through the cache server, so jobs must be able to reach it. `cache.v2: false` sends them to Gitea directly, unless the Gitea URL has a path or `runner.insecure` is set with HTTPS.
|
||||
### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
|
||||
|
||||
#### Official Docker image
|
||||
Hook scripts run where the job's steps run, before the first step and after the last. They match GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`.
|
||||
|
||||
Besides `GITEA_INSTANCE_URL` and `GITEA_RUNNER_REGISTRATION_TOKEN`, the image entrypoint supports optional variables such as `CONFIG_FILE` (passed through as `-c`), `GITEA_RUNNER_LABELS`, `GITEA_RUNNER_EPHEMERAL`, `GITEA_RUNNER_ONCE`, `GITEA_RUNNER_NAME`, `GITEA_MAX_REG_ATTEMPTS`, `RUNNER_STATE_FILE`, and `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`. See [scripts/run.sh](scripts/run.sh) for exact behavior.
|
||||
Use them for per-job setup no workflow should carry, like registry logins or masking runner-wide secrets. They block the job, and a non-zero exit fails it. See [docs/job-hooks.md](docs/job-hooks.md).
|
||||
|
||||
For a fuller container-oriented walkthrough, see [examples/docker](examples/docker/README.md).
|
||||
### Secret masking
|
||||
|
||||
While the runner is idle it cleans up after earlier jobs:
|
||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||
Secrets and `::add-mask::` values are hidden in job logs, [local job logs](#local-job-logs-logjobdir), job summaries and container names. As on GitHub, a job output containing one is dropped with a warning.
|
||||
|
||||
#### Post-task script (`runner.post_task_script`)
|
||||
## Operation
|
||||
|
||||
Optional host script that runs **after** each task's built-in cleanup (post-steps, container teardown, bind-workdir removal). Use it for extra machine housekeeping — Docker pruning, disk cleanup, and similar.
|
||||
### Idle cleanup
|
||||
|
||||
**While the script runs, the runner stops task heartbeats and stays offline from Gitea's perspective until the script exits (or hits `runner.post_task_script_timeout`, default `5m`).** A script that blocks without exiting keeps the runner from taking new work for up to that timeout. Script output goes to the runner log, not the job log; a non-zero exit is warned but does not change the job result.
|
||||
While idle, the runner removes leftovers of earlier jobs every `runner.idle_cleanup_interval` (default `10m`):
|
||||
|
||||
On Windows, use `.exe`, `.bat`, or `.cmd` paths; **PowerShell (`.ps1`) is not supported yet** as the configured path — wrap commands in a `.cmd` file instead.
|
||||
- Workspaces older than `runner.workdir_cleanup_age` (default `24h`), when `container.bind_workdir` is on. This assumes no other runner shares `container.workdir_parent`.
|
||||
- Unused Docker networks and volumes of jobs the runner did not tear down.
|
||||
- Kubernetes job pods older than `runner.workdir_cleanup_age`, with their Secrets.
|
||||
|
||||
See **[docs/post-task-script.md](docs/post-task-script.md)** for lifecycle details, environment variables, timeout interaction, and platform notes.
|
||||
Set either option to `0` to turn cleanup off.
|
||||
|
||||
#### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
|
||||
### Post-task script (`runner.post_task_script`)
|
||||
|
||||
Optional scripts that run **inside the job environment** (the job container, or the host in host mode), before the job's first step and after its last one. They are the equivalent of GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`, which are read when the settings are unset.
|
||||
A host script that runs after each task's cleanup, for housekeeping like Docker pruning. The runner takes no new jobs until it exits or hits `runner.post_task_script_timeout` (default `5m`). Its output goes to the runner log.
|
||||
|
||||
Because they run where the steps run and see the job's environment, they are the place for per-job setup no workflow should have to carry: registry logins, mirror configuration, or masking runner-wide secrets with `::add-mask::`. Their output is part of the job log and is scanned for workflow commands, and they can export to the job through `$GITHUB_ENV` and `$GITHUB_PATH`.
|
||||
On Windows, use an `.exe`, `.bat` or `.cmd` file, PowerShell scripts are not supported. See [docs/post-task-script.md](docs/post-task-script.md).
|
||||
|
||||
Both hooks are synchronous and block the job while they run. Either one exiting non-zero fails the job, and there is no per-hook timeout.
|
||||
### Local job logs (`log.job.dir`)
|
||||
|
||||
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
|
||||
Set `log.job.dir` to keep a copy of every task's log, masked as in Gitea, with the job's result on the last line. Copies are kept for `log.job.retention` (default `168h`), so keep it above `runner.timeout`. Only the runner's user can read them.
|
||||
|
||||
#### OpenTelemetry (`OTEL_*`)
|
||||
### OpenTelemetry (`OTEL_*`)
|
||||
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
|
||||
|
||||
#### Local job logs (`log.job.dir`)
|
||||
|
||||
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
|
||||
|
||||
#### Secret masking
|
||||
|
||||
A job's secrets and its `::add-mask::` values are hidden from what the runner writes and uploads: the job log, the local copy above, job summaries, and the names of the containers it creates. A job output carrying one is skipped with a warning rather than sent masked, as GitHub does, so a downstream `needs.<job>.outputs.<name>` reading it is empty.
|
||||
|
||||
`log.job.retention` (default `168h`) is how long a log is kept, expired ones being deleted as new tasks start, and `log.job.max_size` (default `1GB`) caps one log. Keep `retention` above `runner.timeout` so a long job cannot outlive its own log, and prefer local disk, the file is written while the job runs. Only the runner's own user can read it.
|
||||
|
||||
### Example Deployments
|
||||
|
||||
Check out the [examples](examples) directory for sample deployment types.
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send job traces and runner metrics to an OpenTelemetry collector. See [docs/telemetry.md](docs/telemetry.md).
|
||||
@@ -6,7 +6,5 @@
|
||||
//
|
||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||
//
|
||||
// TODO: Authorization
|
||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||
package artifactcache
|
||||
+368
-197
@@ -15,6 +15,7 @@ import (
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
@@ -32,8 +33,8 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/internal/pkg/disk"
|
||||
|
||||
"github.com/timshannon/bolthold"
|
||||
"go.etcd.io/bbolt"
|
||||
berrors "go.etcd.io/bbolt/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -44,6 +45,8 @@ const (
|
||||
// Short enough that a leaked URL is near-worthless; long enough to let the
|
||||
// @actions/cache client download a big blob that was returned from /cache.
|
||||
artifactURLTTL = 10 * time.Minute
|
||||
|
||||
dbOpenTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
type credKey struct{}
|
||||
@@ -56,6 +59,10 @@ type credKey struct{}
|
||||
type JobCredential struct {
|
||||
Repo string `json:"repo"`
|
||||
|
||||
Ref string `json:"ref"`
|
||||
BaseRef string `json:"base_ref"`
|
||||
DefaultRef string `json:"default_ref"`
|
||||
|
||||
// Results is the instance whose artifact service this server forwards for the job, and
|
||||
// InsecureTLS how the runner reaches it; see results.go.
|
||||
Results string `json:"-"`
|
||||
@@ -65,6 +72,10 @@ type JobCredential struct {
|
||||
PublicURL string `json:"public_url"`
|
||||
}
|
||||
|
||||
func (cred JobCredential) owns(cache *Cache) bool {
|
||||
return cache.Repo == cred.Repo && cache.Ref == cred.Ref
|
||||
}
|
||||
|
||||
// credEntry holds a registered job's credential along with an active
|
||||
// registration count. RegisterJob is reference-counted so that if two tasks
|
||||
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
||||
@@ -79,19 +90,22 @@ type Handler struct {
|
||||
dir string
|
||||
storage cacheStorage
|
||||
s3 *s3Storage
|
||||
router *http.ServeMux
|
||||
listener net.Listener
|
||||
port int
|
||||
server *http.Server
|
||||
logger *logging.Logger
|
||||
|
||||
dbLock chan struct{} // queues store openers without polling bolt's file lock
|
||||
entries cacheEntries // guarded by dbLock
|
||||
|
||||
gcing atomic.Bool
|
||||
gcAt time.Time
|
||||
|
||||
stopSync context.CancelFunc
|
||||
syncing sync.WaitGroup
|
||||
|
||||
outboundIP string
|
||||
outboundIP string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||
// runner can use these endpoints to pre-register per-job
|
||||
@@ -111,7 +125,7 @@ type Handler struct {
|
||||
|
||||
policy Policy
|
||||
|
||||
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
|
||||
// freeDisk is a field so tests can drive makeRoom without a full volume.
|
||||
freeDisk func(string) (uint64, error)
|
||||
}
|
||||
|
||||
@@ -130,18 +144,21 @@ type Options struct {
|
||||
// in-process RegisterJob method directly.
|
||||
InternalSecret string
|
||||
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
InstanceTransport http.RoundTripper // reaches the instance artifact requests are forwarded to, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
||||
func StartHandler(opts Options) (*Handler, error) {
|
||||
dir, logger := opts.Dir, opts.Logger
|
||||
h := &Handler{
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
instanceTransport: opts.InstanceTransport,
|
||||
}
|
||||
|
||||
if logger == nil {
|
||||
@@ -194,11 +211,9 @@ func StartHandler(opts Options) (*Handler, error) {
|
||||
h.storage = h.s3
|
||||
}
|
||||
|
||||
secret, err := loadOrCreateSecret(dir)
|
||||
if err != nil {
|
||||
if h.secret, err = loadOrCreateSecret(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.secret = secret
|
||||
|
||||
router := http.NewServeMux()
|
||||
router.HandleFunc("GET "+apiPath+"/cache", h.bearerAuth(h.find))
|
||||
@@ -217,8 +232,6 @@ func StartHandler(opts Options) (*Handler, error) {
|
||||
h.registerV2Routes(router)
|
||||
router.HandleFunc(artifactServicePath, h.forwardOrNotFound)
|
||||
|
||||
h.router = router
|
||||
|
||||
h.gcCache()
|
||||
|
||||
if err := h.serve(opts.Port, router); err != nil {
|
||||
@@ -327,16 +340,7 @@ func (h *Handler) RegisterJob(token string, cred JobCredential) func() {
|
||||
}
|
||||
}
|
||||
h.credMu.Unlock()
|
||||
return func() {
|
||||
h.credMu.Lock()
|
||||
if entry, ok := h.creds[token]; ok {
|
||||
entry.refs--
|
||||
if entry.refs <= 0 {
|
||||
delete(h.creds, token)
|
||||
}
|
||||
}
|
||||
h.credMu.Unlock()
|
||||
}
|
||||
return func() { h.RevokeJob(token) }
|
||||
}
|
||||
|
||||
// RevokeJob explicitly revokes one registration of token, mirroring one call
|
||||
@@ -399,10 +403,7 @@ func (h *Handler) Close() error {
|
||||
}
|
||||
var retErr error
|
||||
if h.server != nil {
|
||||
err := h.server.Close()
|
||||
if err != nil {
|
||||
retErr = err
|
||||
}
|
||||
retErr = h.server.Close()
|
||||
h.server = nil
|
||||
}
|
||||
if h.listener != nil {
|
||||
@@ -418,16 +419,170 @@ func (h *Handler) Close() error {
|
||||
return retErr
|
||||
}
|
||||
|
||||
func (h *Handler) openDB() (*bolthold.Store, error) {
|
||||
return bolthold.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bolthold.Options{
|
||||
Encoder: func(value any) ([]byte, error) { return json.Marshal(value) },
|
||||
Decoder: func(data []byte, value any) error { return json.Unmarshal(data, value) },
|
||||
Options: &bbolt.Options{
|
||||
Timeout: 5 * time.Second,
|
||||
NoGrowSync: bbolt.DefaultOptions.NoGrowSync,
|
||||
FreelistType: bbolt.DefaultOptions.FreelistType,
|
||||
},
|
||||
// cacheBucket holds each entry as JSON under its decimal ID, the layout bolthold wrote, so older stores stay readable.
|
||||
var cacheBucket = []byte("Cache")
|
||||
|
||||
var errCacheNotFound = errors.New("cache not found")
|
||||
|
||||
type cacheDB struct {
|
||||
bolt *bbolt.DB
|
||||
close func() error
|
||||
entries *cacheEntries
|
||||
}
|
||||
|
||||
// cacheEntries mirrors the bucket as of transaction tx, so queries decode it again only after another process wrote.
|
||||
type cacheEntries struct {
|
||||
tx int
|
||||
byID map[uint64]*Cache
|
||||
}
|
||||
|
||||
func (db *cacheDB) Close() error {
|
||||
return db.close()
|
||||
}
|
||||
|
||||
func cacheKey(id uint64) []byte {
|
||||
return strconv.AppendUint(nil, id, 10)
|
||||
}
|
||||
|
||||
func (db *cacheDB) get(id uint64) (*Cache, error) {
|
||||
if err := db.bolt.View(db.load); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cache, ok := db.entries.byID[id]
|
||||
if !ok {
|
||||
return nil, errCacheNotFound
|
||||
}
|
||||
copied := *cache
|
||||
return &copied, nil
|
||||
}
|
||||
|
||||
// find returns copies of the matching entries in ID order, so ties break the same way every time.
|
||||
func (db *cacheDB) find(match func(*Cache) bool) ([]*Cache, error) {
|
||||
if err := db.bolt.View(db.load); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var caches []*Cache
|
||||
for _, cache := range db.entries.byID {
|
||||
if match(cache) {
|
||||
copied := *cache
|
||||
caches = append(caches, &copied)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(caches, func(left, right *Cache) int { return cmp.Compare(left.ID, right.ID) })
|
||||
return caches, nil
|
||||
}
|
||||
|
||||
func (db *cacheDB) load(tx *bbolt.Tx) error {
|
||||
if db.entries.byID != nil && db.entries.tx == tx.ID() {
|
||||
return nil
|
||||
}
|
||||
byID := make(map[uint64]*Cache)
|
||||
if bucket := tx.Bucket(cacheBucket); bucket != nil {
|
||||
if err := bucket.ForEach(func(key, data []byte) (err error) {
|
||||
cache := &Cache{}
|
||||
if err := json.Unmarshal(data, cache); err != nil {
|
||||
return err
|
||||
}
|
||||
cache.ID, err = strconv.ParseUint(string(key), 10, 64) // an interrupted legacy insert left "id":0 in the value
|
||||
byID[cache.ID] = cache
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
db.entries.byID, db.entries.tx = byID, tx.ID()
|
||||
return nil
|
||||
}
|
||||
|
||||
// update keeps the mirror in step with its own write, and drops it when another process wrote in between or the write rolled back.
|
||||
func (db *cacheDB) update(fn func(*bbolt.Bucket) error) error {
|
||||
err := db.bolt.Update(func(tx *bbolt.Tx) error {
|
||||
if db.entries.tx != tx.ID()-1 {
|
||||
db.entries.byID = nil
|
||||
}
|
||||
db.entries.tx = tx.ID()
|
||||
bucket, err := tx.CreateBucketIfNotExists(cacheBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fn(bucket)
|
||||
})
|
||||
if err != nil {
|
||||
db.entries.byID = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *cacheDB) put(bucket *bbolt.Bucket, cache *Cache) error {
|
||||
data, err := json.Marshal(cache)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if db.entries.byID != nil {
|
||||
copied := *cache
|
||||
db.entries.byID[cache.ID] = &copied
|
||||
}
|
||||
return bucket.Put(cacheKey(cache.ID), data)
|
||||
}
|
||||
|
||||
func (db *cacheDB) delete(bucket *bbolt.Bucket, id uint64) error {
|
||||
delete(db.entries.byID, id)
|
||||
return bucket.Delete(cacheKey(id))
|
||||
}
|
||||
|
||||
// dropLegacyIndexes deletes the index buckets bolthold kept, all created together and never recreated, checking first so later runs do not write.
|
||||
func (db *cacheDB) dropLegacyIndexes() error {
|
||||
var found bool
|
||||
_ = db.bolt.View(func(tx *bbolt.Tx) error {
|
||||
found = tx.Bucket([]byte("_index:Cache:Key")) != nil
|
||||
return nil
|
||||
})
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return db.bolt.Update(func(tx *bbolt.Tx) error {
|
||||
for _, field := range []string{"Repo", "Key", "Version", "Complete", "UsedAt", "CreatedAt"} {
|
||||
if err := tx.DeleteBucket([]byte("_index:Cache:" + field)); err != nil && !errors.Is(err, berrors.ErrBucketNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// save rewrites a stored entry, failing once a sweep has removed it so a late commit cannot resurrect it.
|
||||
func (db *cacheDB) save(cache *Cache) error {
|
||||
return db.update(func(bucket *bbolt.Bucket) error {
|
||||
if bucket.Get(cacheKey(cache.ID)) == nil {
|
||||
return errCacheNotFound
|
||||
}
|
||||
return db.put(bucket, cache)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) openDB() (store *cacheDB, err error) {
|
||||
select {
|
||||
case h.dbLock <- struct{}{}:
|
||||
case <-time.After(dbOpenTimeout):
|
||||
return nil, berrors.ErrTimeout
|
||||
}
|
||||
defer func() {
|
||||
if store == nil { // only an opened store hands the lock to Close, even when Open panics
|
||||
<-h.dbLock
|
||||
}
|
||||
}()
|
||||
db, err := bbolt.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bbolt.Options{
|
||||
Timeout: dbOpenTimeout,
|
||||
NoGrowSync: bbolt.DefaultOptions.NoGrowSync,
|
||||
FreelistType: bbolt.DefaultOptions.FreelistType,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cacheDB{bolt: db, entries: &h.entries, close: sync.OnceValue(func() error {
|
||||
defer func() { <-h.dbLock }()
|
||||
return db.Close()
|
||||
})}, nil
|
||||
}
|
||||
|
||||
// syncMetadata mirrors the bucket into the store until ctx ends, so no request waits on S3 for it.
|
||||
@@ -452,7 +607,7 @@ func (h *Handler) importMetadata(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
local := h.caches(db, bolthold.Where("Complete").Eq(true).Index("Complete"))
|
||||
local := h.caches(db, func(cache *Cache) bool { return cache.Complete })
|
||||
db.Close()
|
||||
|
||||
published, err := h.s3.published(ctx)
|
||||
@@ -480,15 +635,15 @@ func (h *Handler) importMetadata(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
return db.Bolt().Update(func(tx *bbolt.Tx) error {
|
||||
return db.update(func(bucket *bbolt.Bucket) error {
|
||||
for _, cache := range added {
|
||||
if err := db.TxUpsert(tx, cache.ID, cache); err != nil {
|
||||
if err := db.put(bucket, cache); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, cache := range gone {
|
||||
_ = h.s3.Storage.Remove(cache.ID) // a blob kept on disk before the runner used S3
|
||||
if err := db.TxDelete(tx, cache.ID, cache); err != nil && !errors.Is(err, bolthold.ErrNotFound) {
|
||||
if err := db.delete(bucket, cache.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -509,7 +664,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, keys, version)
|
||||
cache, err := h.lookupCache(db, cred, keys, version)
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -525,26 +680,27 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
|
||||
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
|
||||
// entry is dropped on the way out.
|
||||
func (h *Handler) lookupCache(db *bolthold.Store, repo string, keys []string, version string) (*Cache, error) {
|
||||
cache, err := findCache(db, repo, keys, version)
|
||||
if err != nil || cache == nil {
|
||||
return nil, err
|
||||
// lookupCache returns the entry to restore or (nil, nil), dropping matches whose blob was pruned.
|
||||
func (h *Handler) lookupCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
for {
|
||||
cache, err := findCache(db, cred, keys, version)
|
||||
if err != nil || cache == nil {
|
||||
return nil, err
|
||||
}
|
||||
ok, err := h.storage.Exist(cache.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||
// this call and the GET that follows it.
|
||||
h.touch(db, cache)
|
||||
return cache, nil
|
||||
}
|
||||
if !h.deleteCache(db, cache) {
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
}
|
||||
ok, err := h.storage.Exist(cache.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
h.deleteCache(db, cache)
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||
// this call and the GET that follows it.
|
||||
h.touch(db, cache)
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// POST /_apis/artifactcache/caches
|
||||
@@ -556,7 +712,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
cache := &Cache{Repo: cred.Repo, Ref: cred.Ref, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
if cache.Size == 0 {
|
||||
cache.Size = -1
|
||||
}
|
||||
@@ -567,9 +723,10 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().Unix()
|
||||
cache.CreatedAt = now
|
||||
cache.UsedAt = now
|
||||
now := time.Now()
|
||||
cache.CreatedAt = now.Unix()
|
||||
cache.CreatedAtNanos = now.Nanosecond()
|
||||
cache.UsedAt = now.Unix()
|
||||
if err := h.insertCache(db, cache); err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -588,15 +745,15 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
cache := &Cache{}
|
||||
db, err := h.openDB()
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.Get(id, cache); err != nil {
|
||||
if errors.Is(err, bolthold.ErrNotFound) {
|
||||
cache, err := db.get(uint64(id))
|
||||
if err != nil {
|
||||
if errors.Is(err, errCacheNotFound) {
|
||||
h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id))
|
||||
return
|
||||
}
|
||||
@@ -604,7 +761,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -614,11 +771,12 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
db.Close()
|
||||
start, _, err := parseContentRange(r.Header.Get("Content-Range"))
|
||||
start, err := parseContentRange(r.Header.Get("Content-Range"))
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 400, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -635,16 +793,27 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, 400, err)
|
||||
return
|
||||
}
|
||||
var request struct {
|
||||
Size *int64 `json:"size"`
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||
if err == nil && len(body) > 0 {
|
||||
err = json.Unmarshal(body, &request)
|
||||
}
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
cache := &Cache{}
|
||||
db, err := h.openDB()
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.Get(id, cache); err != nil {
|
||||
if errors.Is(err, bolthold.ErrNotFound) {
|
||||
cache, err := db.get(uint64(id))
|
||||
if err != nil {
|
||||
if errors.Is(err, errCacheNotFound) {
|
||||
h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id))
|
||||
return
|
||||
}
|
||||
@@ -652,7 +821,7 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -662,6 +831,14 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if request.Size != nil {
|
||||
if *request.Size < 0 || cache.Size >= 0 && cache.Size != *request.Size {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, errors.New("invalid cache size"))
|
||||
return
|
||||
}
|
||||
cache.Size = *request.Size
|
||||
}
|
||||
|
||||
db.Close()
|
||||
|
||||
if err := h.commitCache(cache); err != nil {
|
||||
@@ -696,17 +873,14 @@ func (h *Handler) commitCache(cache *Cache) error {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.Update(cache.ID, cache); err != nil {
|
||||
if err := db.save(cache); err != nil {
|
||||
return err
|
||||
}
|
||||
if h.s3 != nil {
|
||||
return nil // lifecycle rules on the bucket bound its size
|
||||
}
|
||||
// A commit is the only thing that grows the store, so the only thing that can push the
|
||||
// volume under the floor.
|
||||
h.evictRepo(db, cache.Repo)
|
||||
h.evictTotal(db)
|
||||
h.evictForFreeSpace(db)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -881,10 +1055,8 @@ func bearerToken(r *http.Request) string {
|
||||
}
|
||||
|
||||
func credFromContext(ctx context.Context) JobCredential {
|
||||
if cred, ok := ctx.Value(credKey{}).(JobCredential); ok {
|
||||
return cred
|
||||
}
|
||||
return JobCredential{}
|
||||
cred, _ := ctx.Value(credKey{}).(JobCredential)
|
||||
return cred
|
||||
}
|
||||
|
||||
// computeSignature signs a URL for one cache entry and expiry. purpose is mixed into the
|
||||
@@ -909,34 +1081,49 @@ func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time
|
||||
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
||||
}
|
||||
|
||||
// if not found, return (nil, nil) instead of an error.
|
||||
func findCache(db *bolthold.Store, repo string, keys []string, version string) (*Cache, error) {
|
||||
cache := &Cache{}
|
||||
for _, prefix := range keys {
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
exact, err := findExactCache(db, repo, prefix, version, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// findCache tries every key in one ref before the next ref, and returns (nil, nil) on a miss.
|
||||
func findCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
refs := []string{cred.Ref}
|
||||
for _, ref := range []string{cred.BaseRef, cred.DefaultRef} {
|
||||
if ref != "" && !slices.Contains(refs, ref) {
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
if exact != nil {
|
||||
return exact, nil
|
||||
}
|
||||
if err := db.FindOne(cache,
|
||||
bolthold.Where("Repo").Eq(repo).Index("Repo").
|
||||
And("Key").MatchFunc(func(key string) (bool, error) { return strings.HasPrefix(key, prefix), nil }).
|
||||
And("Version").Eq(version).
|
||||
And("Complete").Eq(true).
|
||||
SortBy("CreatedAt").Reverse()); err != nil {
|
||||
if errors.Is(err, bolthold.ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Repo == cred.Repo && cache.Version == version && cache.Complete && slices.Contains(refs, cache.Ref) &&
|
||||
slices.ContainsFunc(keys, func(prefix string) bool { return strings.HasPrefix(cache.Key, prefix) })
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
for _, ref := range refs {
|
||||
for _, prefix := range keys {
|
||||
var exact, partial []*Cache
|
||||
for _, cache := range caches {
|
||||
if cache.Ref == ref && cache.Key == prefix {
|
||||
exact = append(exact, cache)
|
||||
} else if cache.Ref == ref && strings.HasPrefix(cache.Key, prefix) {
|
||||
partial = append(partial, cache)
|
||||
}
|
||||
}
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
if cache := cmp.Or(latest(exact, (*Cache).createdAtNano), latest(partial, (*Cache).createdAtNano)); cache != nil {
|
||||
return cache, nil
|
||||
}
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
||||
}
|
||||
|
||||
func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
||||
if len(caches) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.MaxFunc(caches, func(left, right *Cache) int {
|
||||
return cmp.Or(cmp.Compare(stamp(left), stamp(right)), cmp.Compare(left.ID, right.ID))
|
||||
})
|
||||
}
|
||||
|
||||
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
|
||||
// none. Unlike findCache it never falls back to a prefix (restore-key) match, which is what both
|
||||
// its callers need: a new key that is only a prefix of an existing key is not the same entry.
|
||||
@@ -944,43 +1131,33 @@ func findCache(db *bolthold.Store, repo string, keys []string, version string) (
|
||||
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
||||
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
||||
// touches UsedAt on every part.
|
||||
func findExactCache(db *bolthold.Store, repo, key, version string, complete bool) (*Cache, error) {
|
||||
sortBy := "UsedAt"
|
||||
if complete {
|
||||
sortBy = "CreatedAt"
|
||||
}
|
||||
cache := &Cache{}
|
||||
err := db.FindOne(cache,
|
||||
bolthold.Where("Key").Eq(key).Index("Key").
|
||||
And("Repo").Eq(repo).
|
||||
And("Version").Eq(version).
|
||||
And("Complete").Eq(complete).
|
||||
SortBy(sortBy).Reverse())
|
||||
if errors.Is(err, bolthold.ErrNotFound) {
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
func findExactCache(db *cacheDB, cred JobCredential, key, version string, complete bool) (*Cache, error) {
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cred.owns(cache) && cache.Key == key && cache.Version == version && cache.Complete == complete
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
return cache, nil
|
||||
stamp := func(cache *Cache) int64 { return cache.UsedAt }
|
||||
if complete {
|
||||
stamp = (*Cache).createdAtNano
|
||||
}
|
||||
return latest(caches, stamp), nil
|
||||
}
|
||||
|
||||
func (h *Handler) insertCache(db *bolthold.Store, cache *Cache) error {
|
||||
if h.s3 != nil {
|
||||
var id [8]byte
|
||||
_, _ = rand.Read(id[:])
|
||||
cache.ID = binary.BigEndian.Uint64(id[:]) >> 11 // IDs must be unique across runners, and v1 clients parse an ID as a JS number
|
||||
return db.Insert(cache.ID, cache)
|
||||
}
|
||||
return db.Bolt().Update(func(tx *bbolt.Tx) error {
|
||||
if err := db.TxInsert(tx, bolthold.NextSequence(), cache); err != nil {
|
||||
func (h *Handler) insertCache(db *cacheDB, cache *Cache) error {
|
||||
return db.update(func(bucket *bbolt.Bucket) (err error) {
|
||||
if h.s3 != nil {
|
||||
var id [8]byte
|
||||
_, _ = rand.Read(id[:])
|
||||
cache.ID = binary.BigEndian.Uint64(id[:]) >> 11 // IDs must be unique across runners, and v1 clients parse an ID as a JS number
|
||||
} else if cache.ID, err = bucket.NextSequence(); err != nil {
|
||||
return fmt.Errorf("insert cache: %w", err)
|
||||
}
|
||||
// write back id to db
|
||||
if err := db.TxUpdate(tx, cache.ID, cache); err != nil {
|
||||
return fmt.Errorf("write back id to db: %w", err)
|
||||
if bucket.Get(cacheKey(cache.ID)) != nil {
|
||||
return fmt.Errorf("insert cache: id %d is taken", cache.ID)
|
||||
}
|
||||
return nil
|
||||
return db.put(bucket, cache)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -996,11 +1173,8 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache := &Cache{}
|
||||
if err := db.Get(id, cache); err != nil {
|
||||
if errors.Is(err, bolthold.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
cache, err := db.get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if requireIncomplete && cache.Complete {
|
||||
@@ -1010,7 +1184,7 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
||||
return nil
|
||||
}
|
||||
cache.UsedAt = time.Now().Unix()
|
||||
return db.Update(cache.ID, cache)
|
||||
return db.save(cache)
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -1056,9 +1230,6 @@ func (p Policy) withDefaults() Policy {
|
||||
}
|
||||
|
||||
func (h *Handler) gcCache() {
|
||||
if h.gcing.Load() {
|
||||
return
|
||||
}
|
||||
if !h.gcing.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
@@ -1077,6 +1248,9 @@ func (h *Handler) gcCache() {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.dropLegacyIndexes(); err != nil {
|
||||
h.logger.Warnf("drop legacy indexes: %v", err)
|
||||
}
|
||||
h.evictIncomplete(db)
|
||||
if h.s3 != nil {
|
||||
return // lifecycle rules on the bucket expire its entries
|
||||
@@ -1084,23 +1258,29 @@ func (h *Handler) gcCache() {
|
||||
h.evictExpired(db)
|
||||
h.evictSuperseded(db)
|
||||
h.evictOversized(db)
|
||||
h.evictForFreeSpace(db)
|
||||
_ = db.Close()
|
||||
h.makeRoom(0)
|
||||
}
|
||||
|
||||
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
|
||||
// accounted for.
|
||||
func (h *Handler) evictForFreeSpace(db *bolthold.Store) {
|
||||
// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
|
||||
func (h *Handler) makeRoom(need int64) {
|
||||
free, err := h.freeDisk(h.dir)
|
||||
if err != nil {
|
||||
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
||||
return
|
||||
}
|
||||
if free >= uint64(h.policy.MinFreeDisk) {
|
||||
shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
|
||||
if h.s3 != nil || shortfall <= 0 {
|
||||
return
|
||||
}
|
||||
db, err := h.openDB()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
caches := h.completedByUse(db)
|
||||
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
|
||||
total := totalSize(caches)
|
||||
if total <= shortfall {
|
||||
// Say so, or shedding everything and still being short reads as the backstop working.
|
||||
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
||||
@@ -1109,38 +1289,32 @@ func (h *Handler) evictForFreeSpace(db *bolthold.Store) {
|
||||
}
|
||||
|
||||
// evictIncomplete removes uploads that stopped part way, which are most likely broken.
|
||||
func (h *Handler) evictIncomplete(db *bolthold.Store) {
|
||||
h.sweep(db, bolthold.
|
||||
Where("UsedAt").Lt(time.Now().Add(-uploadStallTimeout).Unix()).
|
||||
And("Complete").Eq(false).
|
||||
Index("UsedAt"))
|
||||
func (h *Handler) evictIncomplete(db *cacheDB) {
|
||||
stalled := time.Now().Add(-uploadStallTimeout).Unix()
|
||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < stalled && !cache.Complete })
|
||||
}
|
||||
|
||||
func (h *Handler) evictExpired(db *bolthold.Store) {
|
||||
func (h *Handler) evictExpired(db *cacheDB) {
|
||||
if h.policy.Retention <= 0 {
|
||||
return
|
||||
}
|
||||
// Never below inUseGrace, or a short retention would outrun a signed URL already issued.
|
||||
window := max(h.policy.Retention+touchStale, inUseGrace)
|
||||
h.sweep(db, bolthold.Where("UsedAt").Lt(time.Now().Add(-window).Unix()).Index("UsedAt"))
|
||||
expired := time.Now().Add(-max(h.policy.Retention+touchStale, inUseGrace)).Unix()
|
||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
||||
}
|
||||
|
||||
// evictSuperseded removes entries a newer one with the same key and version replaced. The
|
||||
// aggregation includes Repo so two repos sharing a (key, version) do not evict each other.
|
||||
func (h *Handler) evictSuperseded(db *bolthold.Store) {
|
||||
results, err := db.FindAggregate(&Cache{}, bolthold.Where("Complete").Eq(true).Index("Complete"), "Repo", "Key", "Version")
|
||||
if err != nil {
|
||||
h.logger.Warnf("find aggregate caches: %v", err)
|
||||
return
|
||||
// evictSuperseded removes entries a newer one with the same repo, ref, key and version replaced.
|
||||
func (h *Handler) evictSuperseded(db *cacheDB) {
|
||||
type entry struct{ repo, ref, key, version string }
|
||||
groups := make(map[entry][]*Cache)
|
||||
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
||||
group := entry{cache.Repo, cache.Ref, cache.Key, cache.Version}
|
||||
groups[group] = append(groups[group], cache)
|
||||
}
|
||||
var caches []*Cache
|
||||
for _, result := range results {
|
||||
if result.Count() <= 1 {
|
||||
continue
|
||||
}
|
||||
result.Sort("CreatedAt")
|
||||
caches = caches[:0]
|
||||
result.Reduction(&caches)
|
||||
for _, caches := range groups {
|
||||
slices.SortFunc(caches, func(left, right *Cache) int {
|
||||
return cmp.Or(cmp.Compare(left.createdAtNano(), right.createdAtNano()), cmp.Compare(left.ID, right.ID))
|
||||
})
|
||||
for _, cache := range caches[:len(caches)-1] {
|
||||
if inUse(cache) {
|
||||
continue
|
||||
@@ -1152,7 +1326,7 @@ func (h *Handler) evictSuperseded(db *bolthold.Store) {
|
||||
|
||||
// evictOversized applies the per-repository limit, then the whole-store one. Only completed
|
||||
// entries count, since only those carry a size measured at commit rather than claimed.
|
||||
func (h *Handler) evictOversized(db *bolthold.Store) {
|
||||
func (h *Handler) evictOversized(db *cacheDB) {
|
||||
if h.policy.RepoSizeLimit > 0 {
|
||||
byRepo := make(map[string][]*Cache)
|
||||
for _, cache := range h.completedByUse(db) {
|
||||
@@ -1167,7 +1341,7 @@ func (h *Handler) evictOversized(db *bolthold.Store) {
|
||||
|
||||
// evictTotal caps the store as a whole. It re-queries because the per-repo pass may have
|
||||
// deleted rows an earlier result still holds.
|
||||
func (h *Handler) evictTotal(db *bolthold.Store) {
|
||||
func (h *Handler) evictTotal(db *cacheDB) {
|
||||
if h.policy.SizeLimit <= 0 {
|
||||
return
|
||||
}
|
||||
@@ -1175,15 +1349,15 @@ func (h *Handler) evictTotal(db *bolthold.Store) {
|
||||
}
|
||||
|
||||
// evictRepo reclaims space when a commit pushes a repo over, rather than at the next sweep.
|
||||
func (h *Handler) evictRepo(db *bolthold.Store, repo string) {
|
||||
func (h *Handler) evictRepo(db *cacheDB, repo string) {
|
||||
if h.policy.RepoSizeLimit <= 0 {
|
||||
return
|
||||
}
|
||||
h.evictTo(db, h.cachesByUse(db, bolthold.Where("Repo").Eq(repo).And("Complete").Eq(true).Index("Repo")), h.policy.RepoSizeLimit, "repository "+repo)
|
||||
h.evictTo(db, h.cachesByUse(db, func(cache *Cache) bool { return cache.Repo == repo && cache.Complete }), h.policy.RepoSizeLimit, "repository "+repo)
|
||||
}
|
||||
|
||||
// evictTo deletes until caches fit limit. caches must be ordered by UsedAt ascending.
|
||||
func (h *Handler) evictTo(db *bolthold.Store, caches []*Cache, limit int64, scope string) {
|
||||
func (h *Handler) evictTo(db *cacheDB, caches []*Cache, limit int64, scope string) {
|
||||
// An entry bigger than the limit never fits, so it goes on its own account instead of
|
||||
// dragging every neighbour out first and then following them next sweep.
|
||||
fits := caches[:0]
|
||||
@@ -1230,41 +1404,39 @@ func touchNeeded(cache *Cache) bool {
|
||||
// touch stamps UsedAt through the caller's store, a bolt write on the read path. It cannot
|
||||
// go through touchCache, which opens its own store and would block on the exclusive lock
|
||||
// for as long as the caller holds one.
|
||||
func (h *Handler) touch(db *bolthold.Store, cache *Cache) {
|
||||
func (h *Handler) touch(db *cacheDB, cache *Cache) {
|
||||
if !touchNeeded(cache) {
|
||||
return
|
||||
}
|
||||
cache.UsedAt = time.Now().Unix()
|
||||
if err := db.Update(cache.ID, cache); err != nil {
|
||||
if err := db.save(cache); err != nil {
|
||||
h.logger.Warnf("touch cache: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) sweep(db *bolthold.Store, query *bolthold.Query) {
|
||||
for _, cache := range h.caches(db, query) {
|
||||
func (h *Handler) sweep(db *cacheDB, match func(*Cache) bool) {
|
||||
for _, cache := range h.caches(db, match) {
|
||||
h.deleteCache(db, cache)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) caches(db *bolthold.Store, query *bolthold.Query) []*Cache {
|
||||
var caches []*Cache
|
||||
if err := db.Find(&caches, query); err != nil {
|
||||
func (h *Handler) caches(db *cacheDB, match func(*Cache) bool) []*Cache {
|
||||
caches, err := db.find(match)
|
||||
if err != nil {
|
||||
h.logger.Warnf("find caches: %v", err)
|
||||
}
|
||||
return caches
|
||||
}
|
||||
|
||||
// cachesByUse returns matches least recently accessed first, sorting here rather than with
|
||||
// bolthold's SortBy, which reflects over every field it compares.
|
||||
func (h *Handler) cachesByUse(db *bolthold.Store, query *bolthold.Query) []*Cache {
|
||||
caches := h.caches(db, query)
|
||||
func (h *Handler) cachesByUse(db *cacheDB, match func(*Cache) bool) []*Cache {
|
||||
caches := h.caches(db, match)
|
||||
slices.SortFunc(caches, func(a, b *Cache) int { return cmp.Compare(a.UsedAt, b.UsedAt) })
|
||||
return caches
|
||||
}
|
||||
|
||||
// completedByUse returns every entry the size limits count, least recently accessed first.
|
||||
func (h *Handler) completedByUse(db *bolthold.Store) []*Cache {
|
||||
return h.cachesByUse(db, bolthold.Where("Complete").Eq(true).Index("Complete"))
|
||||
func (h *Handler) completedByUse(db *cacheDB) []*Cache {
|
||||
return h.cachesByUse(db, func(cache *Cache) bool { return cache.Complete })
|
||||
}
|
||||
|
||||
func totalSize(caches []*Cache) int64 {
|
||||
@@ -1277,7 +1449,7 @@ func totalSize(caches []*Cache) int64 {
|
||||
|
||||
// deleteCache drops an entry and its bytes, reporting whether it went fully. The blob goes
|
||||
// first, so a failed unlink leaves the row for the next sweep instead of orphaning bytes.
|
||||
func (h *Handler) deleteCache(db *bolthold.Store, cache *Cache) bool {
|
||||
func (h *Handler) deleteCache(db *cacheDB, cache *Cache) bool {
|
||||
storage := h.storage
|
||||
if h.s3 != nil && !cache.Complete {
|
||||
storage = h.s3.Storage // an upload not yet committed has nothing in the bucket
|
||||
@@ -1286,7 +1458,7 @@ func (h *Handler) deleteCache(db *bolthold.Store, cache *Cache) bool {
|
||||
h.logger.Warnf("remove cache blob: %v", err)
|
||||
return false
|
||||
}
|
||||
if err := db.Delete(cache.ID, cache); err != nil {
|
||||
if err := db.update(func(bucket *bbolt.Bucket) error { return db.delete(bucket, cache.ID) }); err != nil {
|
||||
h.logger.Warnf("delete cache: %v", err)
|
||||
return false
|
||||
}
|
||||
@@ -1311,18 +1483,17 @@ func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int,
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
|
||||
func parseContentRange(s string) (int64, int64, error) {
|
||||
func parseContentRange(s string) (int64, error) {
|
||||
// support the format like "bytes 11-22/*" only
|
||||
s, _, _ = strings.Cut(strings.TrimPrefix(s, "bytes "), "/")
|
||||
s1, s2, _ := strings.Cut(s, "-")
|
||||
|
||||
start, err := strconv.ParseInt(s1, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("parse %q: %w", s, err)
|
||||
return 0, fmt.Errorf("parse %q: %w", s, err)
|
||||
}
|
||||
stop, err := strconv.ParseInt(s2, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("parse %q: %w", s, err)
|
||||
if _, err := strconv.ParseInt(s2, 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("parse %q: %w", s, err)
|
||||
}
|
||||
return start, stop, nil
|
||||
return start, nil
|
||||
}
|
||||
+379
-733
File diff suppressed because it is too large.
Load diff
@@ -73,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||
// existing and silently never saved.
|
||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
||||
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if existing != nil {
|
||||
@@ -84,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||
owner := hashedToken(bearerToken(r))
|
||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
||||
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||
@@ -100,15 +100,17 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
now := time.Now()
|
||||
cache := &Cache{
|
||||
Repo: cred.Repo,
|
||||
Key: req.Key,
|
||||
Version: req.Version,
|
||||
Owner: owner,
|
||||
Size: -1, // the size is only known at finalize time
|
||||
CreatedAt: now,
|
||||
UsedAt: now,
|
||||
Repo: cred.Repo,
|
||||
Ref: cred.Ref,
|
||||
Key: req.Key,
|
||||
Version: req.Version,
|
||||
Owner: owner,
|
||||
Size: -1, // the size is only known at finalize time
|
||||
CreatedAt: now.Unix(),
|
||||
CreatedAtNanos: now.Nanosecond(),
|
||||
UsedAt: now.Unix(),
|
||||
}
|
||||
if err := h.insertCache(db, cache); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
@@ -136,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
||||
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||
cache = nil // not the reservation this job made, so not this job's to commit
|
||||
}
|
||||
if err == nil && cache == nil {
|
||||
// A retry whose first response was lost finds the entry already committed.
|
||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
||||
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||
}
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
@@ -185,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
||||
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
@@ -216,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
|
||||
query := r.URL.Query()
|
||||
switch strings.ToLower(query.Get("comp")) {
|
||||
|
||||
@@ -14,13 +14,12 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// v2Call posts a twirp request to the cache service and returns the decoded response.
|
||||
// Field names are the proto ones, which is what the toolkit's client sends.
|
||||
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
@@ -37,7 +36,6 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
|
||||
return got
|
||||
}
|
||||
|
||||
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
|
||||
func putBlob(t *testing.T, url string, content []byte) int {
|
||||
t.Helper()
|
||||
|
||||
@@ -47,8 +45,6 @@ func putBlob(t *testing.T, url string, content []byte) int {
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusCreated {
|
||||
// The Azure SDK client dereferences this header without checking, so a blob upload that
|
||||
// omits it panics the caller rather than failing it.
|
||||
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
||||
}
|
||||
return resp.StatusCode
|
||||
@@ -67,8 +63,6 @@ func getURL(t *testing.T, url string) []byte {
|
||||
return body
|
||||
}
|
||||
|
||||
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
|
||||
// with the upload URL it used.
|
||||
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -84,24 +78,22 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
||||
}), uploadURL
|
||||
}
|
||||
|
||||
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
|
||||
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
|
||||
// or to replace a finalized entry.
|
||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler := newTestHandler(t, Policy{})
|
||||
content := []byte("the cached archive")
|
||||
|
||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||
|
||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
assert.NotEmpty(t, finalized["entry_id"])
|
||||
|
||||
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
|
||||
// and it is an upload URL only: nothing reads a blob back through it.
|
||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
||||
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
|
||||
resp, err := http.Get(uploadURL)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
||||
@@ -114,9 +106,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
assert.Equal(t, content, getURL(t, downloadURL))
|
||||
}
|
||||
|
||||
// A large archive is staged as blocks and only put in order by the final block list, so
|
||||
// blocks that arrive out of order must still be assembled the way the client asked.
|
||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler := newTestHandler(t, Policy{})
|
||||
|
||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||
@@ -130,7 +121,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
blocks[blockID] = []byte(part)
|
||||
order = append(order, blockID)
|
||||
}
|
||||
// Upload in an order that is not the block list order.
|
||||
for _, blockID := range []string{order[2], order[0], order[1]} {
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
||||
}
|
||||
@@ -165,6 +155,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler := newTestHandler(t, Policy{})
|
||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||
require.Equal(t, true, saved["ok"])
|
||||
@@ -174,8 +165,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// The toolkit serialises with the proto field names; the camelCase spellings of the same
|
||||
// proto JSON mapping are accepted alongside them.
|
||||
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
||||
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
||||
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
||||
@@ -191,9 +180,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, again["ok"])
|
||||
})
|
||||
|
||||
// A key that is only a prefix of an existing one is a different entry, so the
|
||||
// reservation check must be exact and not a restore-key prefix match, or the shorter
|
||||
// key would be reported as existing and silently never saved.
|
||||
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
||||
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
||||
require.Equal(t, true, reserved["ok"])
|
||||
@@ -219,7 +205,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// The size the client declares is what Commit validates the assembled archive against.
|
||||
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
||||
@@ -230,8 +215,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// Both API versions are served from one store, so an entry written through v2 is a hit for
|
||||
// a v1 client asking for the same key and version.
|
||||
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
||||
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
||||
require.NoError(t, err)
|
||||
@@ -244,7 +227,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.NotEmpty(t, got["archiveLocation"])
|
||||
})
|
||||
|
||||
// The cache of one repository must stay invisible to another, as it does for the v1 API.
|
||||
t.Run("another repository sees nothing", func(t *testing.T) {
|
||||
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
||||
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
||||
@@ -252,9 +234,28 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||
clients := map[string]*http.Client{"x": testClient}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
require.Equal(t, true, created["ok"])
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||
}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||
}
|
||||
for content, client := range clients {
|
||||
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler := newTestHandler(t, Policy{})
|
||||
|
||||
tests := []struct {
|
||||
@@ -291,6 +292,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler := newTestHandler(t, Policy{})
|
||||
content := []byte("the cached archive")
|
||||
|
||||
@@ -321,20 +323,12 @@ func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||
"another job's upload is not this job's to finalize")
|
||||
|
||||
base := handler.ExternalURL() + apiPath
|
||||
reserveV1, err := json.Marshal(&Request{Key: "v1-made", Version: "v1", Size: int64(len(content))})
|
||||
require.NoError(t, err)
|
||||
resp, err := testClient.Post(base+"/caches", "application/json", bytes.NewReader(reserveV1))
|
||||
require.NoError(t, err)
|
||||
var madeByV1 struct {
|
||||
CacheID uint64 `json:"cacheId"`
|
||||
}
|
||||
require.NoError(t, json.UnmarshalRead(resp.Body, &madeByV1))
|
||||
resp.Body.Close()
|
||||
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
|
||||
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
||||
fmt.Sprintf("%s/caches/%d", base, madeByV1.CacheID), bytes.NewReader(content))
|
||||
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
|
||||
require.NoError(t, err)
|
||||
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
||||
resp, err = testClient.Do(upload)
|
||||
resp, err := testClient.Do(upload)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
package artifactcache
|
||||
|
||||
import "time"
|
||||
|
||||
type Request struct {
|
||||
Key string `json:"key" `
|
||||
Version string `json:"version"`
|
||||
@@ -11,13 +13,19 @@ type Request struct {
|
||||
}
|
||||
|
||||
type Cache struct {
|
||||
ID uint64 `json:"id" boltholdKey:"ID"`
|
||||
Repo string `json:"repo" boltholdIndex:"Repo"`
|
||||
Key string `json:"key" boltholdIndex:"Key"`
|
||||
Version string `json:"version" boltholdIndex:"Version"`
|
||||
Size int64 `json:"cacheSize"`
|
||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||
Complete bool `json:"complete" boltholdIndex:"Complete"`
|
||||
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
|
||||
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
|
||||
ID uint64 `json:"id"`
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref,omitzero"`
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
Size int64 `json:"cacheSize"`
|
||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||
Complete bool `json:"complete"`
|
||||
UsedAt int64 `json:"usedAt"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||
}
|
||||
|
||||
func (cache *Cache) createdAtNano() int64 {
|
||||
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -66,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||
},
|
||||
}
|
||||
if cred.InsecureTLS {
|
||||
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||
proxy.Transport = insecureTransport
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
@@ -74,14 +75,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
||||
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
||||
if cred.InsecureTLS && target.Scheme == "https" {
|
||||
return true
|
||||
}
|
||||
scheme := r.Header.Get("X-Forwarded-Proto")
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
return scheme != target.Scheme
|
||||
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
|
||||
}
|
||||
|
||||
func mergeQuery(target, request string) string {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
)
|
||||
|
||||
func TestFrontResultsService(t *testing.T) {
|
||||
t.Parallel()
|
||||
var gotHost, gotPath, gotProto string
|
||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||
@@ -132,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||
t.Parallel()
|
||||
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||
defer gitea.Close()
|
||||
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
@@ -24,6 +24,10 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// S3Options configures the bucket that stores completed cache entries.
|
||||
@@ -47,7 +51,7 @@ type cacheStorage interface {
|
||||
Remove(uint64) error
|
||||
}
|
||||
|
||||
const s3PartSize = 5 << 30 // the largest part S3 accepts
|
||||
const s3PartSize = 64 << 20
|
||||
|
||||
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
||||
type s3Storage struct {
|
||||
@@ -69,6 +73,7 @@ func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
|
||||
}
|
||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||
transport = transport.Clone()
|
||||
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
|
||||
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
||||
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
||||
if opts.PathStyle != nil {
|
||||
@@ -209,26 +214,43 @@ func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// upload always goes in parts, the only way S3 takes an object over 5 GiB.
|
||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) error {
|
||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
|
||||
if size <= s3PartSize {
|
||||
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
|
||||
return err
|
||||
}
|
||||
var created struct {
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if uploadErr != nil && created.UploadID != "" {
|
||||
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
|
||||
}
|
||||
}()
|
||||
var completion struct {
|
||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||
Parts []s3Part `xml:"Part"`
|
||||
}
|
||||
for offset := int64(0); offset == 0 || offset < size; offset += s3PartSize {
|
||||
number, length := len(completion.Parts)+1, min(s3PartSize, size-offset)
|
||||
query := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {created.UploadID}}
|
||||
header, err := s.call(ctx, http.MethodPut, key, query, io.NewSectionReader(file, offset, length), nil)
|
||||
if err != nil {
|
||||
partSize := max(int64(s3PartSize), (size-1)/10000+1)
|
||||
completion.Parts = make([]s3Part, (size-1)/partSize+1)
|
||||
workers, workerCtx := errgroup.WithContext(ctx)
|
||||
workers.SetLimit(4)
|
||||
for part := range completion.Parts {
|
||||
workers.Go(func() (err error) {
|
||||
defer common.RecoverPanic(&err)
|
||||
offset := int64(part) * partSize
|
||||
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
|
||||
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
|
||||
return err
|
||||
}
|
||||
completion.Parts = append(completion.Parts, s3Part{number, header.Get("ETag")})
|
||||
})
|
||||
}
|
||||
if err := workers.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := xml.Marshal(completion)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,40 +4,135 @@
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/johannesboyne/gofakes3"
|
||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||
backend := s3mem.New()
|
||||
require.NoError(t, backend.CreateBucket("cache"))
|
||||
server := httptest.NewServer(gofakes3.New(backend).Server())
|
||||
t.Parallel()
|
||||
var objects sync.Map
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
key := strings.TrimPrefix(request.URL.Path, "/cache/")
|
||||
switch {
|
||||
case request.Method == http.MethodPut:
|
||||
if strings.Contains(key, "/blobs/") {
|
||||
uploads.Add(1)
|
||||
}
|
||||
body, err := io.ReadAll(request.Body)
|
||||
assert.NoError(t, err)
|
||||
objects.Store(key, body)
|
||||
case request.URL.Query().Has("list-type"):
|
||||
_, _ = io.WriteString(writer, "<ListBucketResult>")
|
||||
objects.Range(func(stored, _ any) bool {
|
||||
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
|
||||
return true
|
||||
})
|
||||
_, _ = io.WriteString(writer, "</ListBucketResult>")
|
||||
default:
|
||||
if body, ok := objects.Load(key); ok {
|
||||
_, _ = writer.Write(body.([]byte))
|
||||
} else {
|
||||
writer.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||
|
||||
started := time.Now()
|
||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
assert.EqualValues(t, 1, uploads.Load())
|
||||
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||
require.NoError(t, err)
|
||||
cache, err := saver.s3.fetch(t.Context(), id)
|
||||
require.NoError(t, err)
|
||||
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||
require.Equal(t, true, found["ok"])
|
||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||
|
||||
require.NoError(t, backend.ForceDeleteBucket("cache"))
|
||||
require.NoError(t, backend.CreateBucket("cache"))
|
||||
for _, cache := range []*Cache{
|
||||
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||
} {
|
||||
require.NoError(t, saver.s3.publish(cache))
|
||||
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||
}
|
||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||
require.Equal(t, true, found["ok"])
|
||||
assert.Equal(t, "deps-new", found["matched_key"])
|
||||
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||
|
||||
objects.Clear()
|
||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||
}
|
||||
|
||||
type s3TestTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return transport(req)
|
||||
}
|
||||
|
||||
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, failingPart := range []string{"", "2"} {
|
||||
var parts, aborts atomic.Int32
|
||||
var completion struct {
|
||||
Parts []s3Part `xml:"Part"`
|
||||
}
|
||||
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
|
||||
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
|
||||
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
|
||||
switch {
|
||||
case query.Has("uploads"):
|
||||
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
|
||||
case req.Method == http.MethodPut:
|
||||
parts.Add(1)
|
||||
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
|
||||
if query.Get("partNumber") == failingPart {
|
||||
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
|
||||
}
|
||||
case req.Method == http.MethodPost:
|
||||
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
|
||||
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
|
||||
case req.Method == http.MethodDelete:
|
||||
aborts.Add(1)
|
||||
}
|
||||
return response, nil
|
||||
})}
|
||||
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
|
||||
assert.EqualValues(t, 3, parts.Load())
|
||||
if failingPart == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
|
||||
assert.Zero(t, aborts.Load())
|
||||
} else {
|
||||
require.ErrorContains(t, err, "InvalidPart")
|
||||
assert.EqualValues(t, 1, aborts.Load())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||
t.Parallel()
|
||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -32,13 +33,11 @@ func NewStorage(rootDir string) (*Storage, error) {
|
||||
}
|
||||
|
||||
func (s *Storage) Exist(id uint64) (bool, error) {
|
||||
name := s.filename(id)
|
||||
if _, err := os.Stat(name); os.IsNotExist(err) {
|
||||
_, err := os.Stat(s.filename(id))
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
||||
@@ -53,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(file, reader)
|
||||
if err = errors.Join(err, file.Close()); err != nil {
|
||||
_ = os.Remove(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -92,9 +92,7 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
||||
}
|
||||
|
||||
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
defer func() {
|
||||
_ = os.RemoveAll(s.tempDir(id))
|
||||
}()
|
||||
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
|
||||
|
||||
name := s.filename(id)
|
||||
tempNames, err := s.tempNames(id)
|
||||
@@ -106,55 +104,50 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
return 0, err
|
||||
}
|
||||
written, err := assemble(name, tempNames)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||
}
|
||||
// If size is less than 0, it means the size is unknown.
|
||||
// We can't check the size of the file, just skip the check.
|
||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
||||
if size >= 0 && written != size {
|
||||
if err != nil {
|
||||
_ = os.Remove(name)
|
||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
||||
return 0, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
||||
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||
func assemble(name string, tempNames []string) (int64, error) {
|
||||
if len(tempNames) == 1 {
|
||||
info, err := os.Stat(tempNames[0])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), os.Rename(tempNames[0], name)
|
||||
if len(tempNames) == 0 {
|
||||
return 0, errors.New("nothing was uploaded")
|
||||
}
|
||||
|
||||
file, err := os.Create(name)
|
||||
if err := os.Rename(tempNames[0], name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var written int64
|
||||
for _, v := range tempNames {
|
||||
f, err := os.Open(v)
|
||||
for _, v := range tempNames[1:] {
|
||||
part, err := os.Open(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := io.Copy(file, f)
|
||||
_ = f.Close()
|
||||
_, err = io.Copy(file, part)
|
||||
_ = part.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written += n
|
||||
_ = os.Remove(v)
|
||||
}
|
||||
return written, nil
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), file.Close()
|
||||
}
|
||||
|
||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||
name := s.filename(id)
|
||||
http.ServeFile(w, r, name)
|
||||
http.ServeFile(w, r, s.filename(id))
|
||||
}
|
||||
|
||||
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
||||
|
||||
+24
-17
@@ -13,6 +13,7 @@ import (
|
||||
"io/fs"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -79,22 +80,30 @@ func uploads(router *http.ServeMux, baseDir string) {
|
||||
itemPath += gzipExtension
|
||||
}
|
||||
|
||||
safeRunPath := safeResolve(baseDir, req.PathValue("runId"))
|
||||
safePath := safeResolve(safeRunPath, itemPath)
|
||||
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
||||
appendUpload := req.Header.Get("Content-Range")
|
||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
||||
var start int64
|
||||
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
flags := os.O_CREATE | os.O_WRONLY
|
||||
if start == 0 {
|
||||
flags |= os.O_TRUNC
|
||||
}
|
||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if req.Body == nil {
|
||||
panic(errors.New("no body given"))
|
||||
}
|
||||
@@ -120,10 +129,8 @@ func downloads(router *http.ServeMux, baseDir string) {
|
||||
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||
runID := req.PathValue("runId")
|
||||
|
||||
safePath := safeResolve(baseDir, runID)
|
||||
|
||||
entries, err := os.ReadDir(safePath)
|
||||
if err != nil {
|
||||
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -148,6 +155,9 @@ func downloads(router *http.ServeMux, baseDir string) {
|
||||
|
||||
var files []ContainerItem
|
||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
rel, err := filepath.Rel(safePath, path)
|
||||
if err != nil {
|
||||
@@ -156,20 +166,16 @@ func downloads(router *http.ServeMux, baseDir string) {
|
||||
|
||||
// if it was upload as gzip
|
||||
rel = strings.TrimSuffix(rel, gzipExtension)
|
||||
path := filepath.Join(itemPath, rel)
|
||||
|
||||
rel = filepath.ToSlash(rel)
|
||||
path = filepath.ToSlash(path)
|
||||
|
||||
files = append(files, ContainerItem{
|
||||
Path: path,
|
||||
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||
ItemType: "file",
|
||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
|
||||
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -212,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(addr, port),
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
||||
router := http.NewServeMux()
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
@@ -48,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
||||
return resp.StatusCode, data
|
||||
}
|
||||
|
||||
t.Run("no-artifacts", func(t *testing.T) {
|
||||
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
var list NamedFileContainerResourceURLResponse
|
||||
require.NoError(t, json.Unmarshal(data, &list))
|
||||
require.Zero(t, list.Count)
|
||||
require.Empty(t, list.Value)
|
||||
})
|
||||
|
||||
t.Run("upload-and-download", func(t *testing.T) {
|
||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
||||
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||
|
||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
@@ -81,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
||||
require.NoError(t, json.Unmarshal(data, &items))
|
||||
require.Len(t, items.Value, 1)
|
||||
require.Equal(t, "file", items.Value[0].ItemType)
|
||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
||||
require.Equal(t, item, items.Value[0].Path)
|
||||
|
||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, content, string(data))
|
||||
|
||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, string(stored))
|
||||
})
|
||||
@@ -95,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
||||
t.Run("content-range", func(t *testing.T) {
|
||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
||||
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
|
||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
for range 2 {
|
||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
}
|
||||
|
||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "first-second", string(stored))
|
||||
|
||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||
require.Equal(t, http.StatusOK, status, string(data))
|
||||
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, stored)
|
||||
})
|
||||
|
||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||
@@ -156,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, content, string(data))
|
||||
})
|
||||
|
||||
t.Run("v4", func(t *testing.T) {
|
||||
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||
call := func(method, body string) (int, map[string]any) {
|
||||
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||
var out map[string]any
|
||||
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||
return status, out
|
||||
}
|
||||
put := func(rawURL, body string) int {
|
||||
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||
return status
|
||||
}
|
||||
download := func() (string, http.Header) {
|
||||
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, signed)
|
||||
resp, err := client.Get(signed["signedUrl"].(string))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return string(body), resp.Header
|
||||
}
|
||||
|
||||
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
upload := created["signedUploadUrl"].(string)
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||
for attempt := range 2 {
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||
}
|
||||
|
||||
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusBadRequest, status)
|
||||
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||
require.Equal(t, http.StatusOK, status, finalized)
|
||||
id := finalized["artifactId"].(string)
|
||||
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, id, refinalized["artifactId"])
|
||||
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusConflict, status)
|
||||
require.Equal(t, "already_exists", conflict["code"])
|
||||
|
||||
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
artifacts := listed["artifacts"].([]any)
|
||||
require.Len(t, artifacts, 1)
|
||||
artifact := artifacts[0].(map[string]any)
|
||||
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||
delete(artifact, "createdAt")
|
||||
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||
require.Empty(t, other["artifacts"])
|
||||
|
||||
body, header := download()
|
||||
require.Equal(t, "hello world", body)
|
||||
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||
|
||||
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusNotFound, status)
|
||||
require.Equal(t, "not_found", missing["code"])
|
||||
|
||||
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||
require.Equal(t, http.StatusCreated, status)
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
body, header = download()
|
||||
require.Equal(t, "gz", body)
|
||||
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||
|
||||
rerun := http.NewServeMux()
|
||||
registerV4(rerun, artifactPath)
|
||||
recorder := httptest.NewRecorder()
|
||||
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestSafeResolve(t *testing.T) {
|
||||
@@ -176,7 +281,7 @@ func TestSafeResolve(t *testing.T) {
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
||||
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
lastID int64
|
||||
created map[string]bool
|
||||
}
|
||||
|
||||
const (
|
||||
zipMimeType = "application/zip"
|
||||
maxTwirpBytes = 1 << 20
|
||||
)
|
||||
|
||||
func registerV4(router *http.ServeMux, baseDir string) {
|
||||
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||
}
|
||||
|
||||
type twirpError struct {
|
||||
status int
|
||||
code string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *twirpError) Error() string { return e.msg }
|
||||
|
||||
func notFound(name string) error {
|
||||
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||
}
|
||||
|
||||
func badRequest(format string, args ...any) error {
|
||||
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var req Req
|
||||
var resp Resp
|
||||
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||
if err != nil {
|
||||
err = badRequest("malformed request: %v", err)
|
||||
} else {
|
||||
resp, err = handle(r, req)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||
errors.As(err, &terr)
|
||||
w.WriteHeader(terr.status)
|
||||
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||
return
|
||||
}
|
||||
_ = json.MarshalWrite(w, resp)
|
||||
}
|
||||
}
|
||||
|
||||
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||
sum := sha256.Sum256([]byte(name))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (s *serverV4) runDir(runID string) string {
|
||||
return filepath.Join(s.dir, fileName(runID))
|
||||
}
|
||||
|
||||
func (s *serverV4) artifactDir(runID, name string) string {
|
||||
return filepath.Join(s.runDir(runID), fileName(name))
|
||||
}
|
||||
|
||||
func (s *serverV4) urlDir(r *http.Request) string {
|
||||
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||
}
|
||||
|
||||
func readMeta(dir string) (*monolithArtifact, error) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var meta monolithArtifact
|
||||
return &meta, json.Unmarshal(data, &meta)
|
||||
}
|
||||
|
||||
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||
data, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||
}
|
||||
|
||||
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||
}
|
||||
|
||||
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.created[dir] {
|
||||
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.lastID++
|
||||
if err := writeMeta(dir, &monolithArtifact{
|
||||
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||
DatabaseID: s.lastID,
|
||||
Name: req.Name,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.created[dir] = true
|
||||
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
type blockList struct {
|
||||
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||
}
|
||||
|
||||
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||
var err error
|
||||
switch comp := r.URL.Query().Get("comp"); comp {
|
||||
case "block":
|
||||
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||
case "blocklist":
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||
case "": // Put Blob, which docker/build-push-action uses
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||
_, err := io.Copy(out, r.Body)
|
||||
return err
|
||||
})
|
||||
default:
|
||||
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
func writeFile(name string, body io.Reader) error {
|
||||
file, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, body)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||
out, err := os.CreateTemp(dir, "blob")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Join(err, os.Remove(out.Name()))
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||
}
|
||||
|
||||
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||
var list blockList
|
||||
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range list.Blocks {
|
||||
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendBlock(out io.Writer, name string) error {
|
||||
in, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
} else if meta.Digest != "" {
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||
}
|
||||
defer blob.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, blob)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
if size != req.Size {
|
||||
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||
}
|
||||
if req.Hash != "" && req.Hash != digest {
|
||||
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||
}
|
||||
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||
defer s.mu.Unlock()
|
||||
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
meta.Size, meta.Digest = size, digest
|
||||
if err := writeMeta(dir, meta); err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||
}
|
||||
|
||||
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||
run := s.runDir(req.WorkflowRunBackendID)
|
||||
entries, err := os.ReadDir(run)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return listArtifactsResponse{}, err
|
||||
}
|
||||
var resp listArtifactsResponse
|
||||
for _, entry := range entries {
|
||||
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||
if err != nil || meta.Digest == "" ||
|
||||
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||
continue
|
||||
}
|
||||
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||
}
|
||||
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||
}
|
||||
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil || meta.Digest == "" {
|
||||
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer blob.Close()
|
||||
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||
contentType, filename := string(data), meta.Name
|
||||
if !strings.Contains(contentType, "/") {
|
||||
contentType = zipMimeType
|
||||
}
|
||||
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||
filename += ".zip"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||
}
|
||||
|
||||
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return deleteArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
delete(s.created, dir)
|
||||
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||
now := time.Now()
|
||||
claims, _ := json.Marshal(map[string]any{
|
||||
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||
"ac": `[{"Scope":"","Permission":3}]`,
|
||||
"nbf": now.Unix(),
|
||||
"exp": now.Add(lifetime).Unix(),
|
||||
"jti": rand.Text(),
|
||||
})
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import "time"
|
||||
|
||||
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||
|
||||
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||
createArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
SignedUploadURL string `json:"signedUploadUrl"`
|
||||
}
|
||||
|
||||
finalizeArtifactRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
Size int64 `json:",string"`
|
||||
Hash string
|
||||
}
|
||||
finalizeArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
ArtifactID int64 `json:"artifactId,string"`
|
||||
}
|
||||
|
||||
listArtifactsRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
NameFilter string
|
||||
IDFilter int64 `json:",string"`
|
||||
}
|
||||
listArtifactsResponse struct {
|
||||
Artifacts []monolithArtifact `json:"artifacts"`
|
||||
}
|
||||
monolithArtifact struct {
|
||||
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||
DatabaseID int64 `json:"databaseId,string"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size,string"`
|
||||
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||
Digest string `json:"digest"` // empty until the upload is finalized
|
||||
}
|
||||
|
||||
getSignedArtifactURLRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
}
|
||||
getSignedArtifactURLResponse struct {
|
||||
SignedURL string `json:"signedUrl"`
|
||||
}
|
||||
|
||||
createArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactResponse = finalizeArtifactResponse
|
||||
)
|
||||
@@ -14,13 +14,8 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
|
||||
|
||||
// Dryrun returns true if the current context is dryrun
|
||||
func Dryrun(ctx context.Context) bool {
|
||||
val := ctx.Value(dryrunContextKeyVal)
|
||||
if val != nil {
|
||||
if dryrun, ok := val.(bool); ok {
|
||||
return dryrun
|
||||
}
|
||||
}
|
||||
return false
|
||||
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
|
||||
return dryrun
|
||||
}
|
||||
|
||||
// WithDryrun adds a value to the context for dryrun
|
||||
|
||||
+43
-23
@@ -7,7 +7,9 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
@@ -21,8 +23,7 @@ type Conditional func(ctx context.Context) bool
|
||||
// NewInfoExecutor is an executor that logs messages
|
||||
func NewInfoExecutor(format string, args ...any) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := Logger(ctx)
|
||||
logger.Infof(format, args...)
|
||||
Logger(ctx).Infof(format, args...)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -30,8 +31,7 @@ func NewInfoExecutor(format string, args ...any) Executor {
|
||||
// NewDebugExecutor is an executor that logs messages
|
||||
func NewDebugExecutor(format string, args ...any) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := Logger(ctx)
|
||||
logger.Debugf(format, args...)
|
||||
Logger(ctx).Debugf(format, args...)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -39,9 +39,7 @@ func NewDebugExecutor(format string, args ...any) Executor {
|
||||
// NewPipelineExecutor creates a new executor from a series of other executors
|
||||
func NewPipelineExecutor(executors ...Executor) Executor {
|
||||
if len(executors) == 0 {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return NewErrorExecutor(nil)
|
||||
}
|
||||
var rtn Executor
|
||||
for _, executor := range executors {
|
||||
@@ -61,6 +59,13 @@ func NewErrorExecutor(err error) Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
|
||||
if recovered := recover(); recovered != nil {
|
||||
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
|
||||
*err = fmt.Errorf("panic: %v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
// NewParallelExecutor creates a new executor from a parallel of other executors
|
||||
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||
if len(executors) == 0 {
|
||||
@@ -80,8 +85,8 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||
|
||||
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||
|
||||
for i := 0; i < parallel; i++ {
|
||||
go func(workerID int, work <-chan Executor, errs chan<- error) {
|
||||
for workerID := range parallel {
|
||||
go func() {
|
||||
logging.Debugf("Worker %d started", workerID)
|
||||
taskCount := 0
|
||||
for executor := range work {
|
||||
@@ -91,17 +96,12 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||
// goroutine which would leave the runner process hung.
|
||||
// https://gitea.com/gitea/runner/issues/371
|
||||
errs <- func() (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logging.Errorf("panic in executor: %v\n%s", r, debug.Stack())
|
||||
err = fmt.Errorf("panic: %v", r)
|
||||
}
|
||||
}()
|
||||
defer RecoverPanic(&err)
|
||||
return executor(ctx)
|
||||
}()
|
||||
}
|
||||
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||
}(i, work, errs)
|
||||
}()
|
||||
}
|
||||
|
||||
for i := range executors {
|
||||
@@ -150,12 +150,7 @@ func (e Executor) If(conditional Conditional) Executor {
|
||||
|
||||
// IfNot only runs this executor if conditional is true
|
||||
func (e Executor) IfNot(conditional Conditional) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if !conditional(ctx) {
|
||||
return e(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
|
||||
}
|
||||
|
||||
// IfBool only runs this executor if conditional is true
|
||||
@@ -165,13 +160,38 @@ func (e Executor) IfBool(conditional bool) Executor {
|
||||
})
|
||||
}
|
||||
|
||||
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||
|
||||
// Retry runs the executor up to attempts times until it succeeds.
|
||||
func (e Executor) Retry(attempts int) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := e(ctx)
|
||||
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
backoff := RetryBackoff()
|
||||
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finally adds an executor to run after other executor
|
||||
func (e Executor) Finally(finally Executor) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
err := e(ctx)
|
||||
err2 := finally(ctx)
|
||||
if err2 != nil {
|
||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||
}
|
||||
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
+86
-101
@@ -7,53 +7,36 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewWorkflow(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := t.Context()
|
||||
require.NoError(t, NewPipelineExecutor()(ctx))
|
||||
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
|
||||
|
||||
// empty
|
||||
emptyWorkflow := NewPipelineExecutor()
|
||||
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// error case
|
||||
errorWorkflow := NewErrorExecutor(errors.New("test error"))
|
||||
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// multiple success case
|
||||
runcount := 0
|
||||
successWorkflow := NewPipelineExecutor(
|
||||
func(ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
})
|
||||
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(2, runcount)
|
||||
increment := func(_ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
}
|
||||
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
|
||||
assert.Equal(t, 2, runcount)
|
||||
}
|
||||
|
||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
||||
// find the gate already open so the last one still finishes with no partner left.
|
||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||
var counted, active, peak atomic.Int32
|
||||
var once sync.Once
|
||||
reached := make(chan struct{})
|
||||
|
||||
return func(ctx context.Context) error {
|
||||
return func(_ctx context.Context) error {
|
||||
counted.Add(1)
|
||||
running := active.Add(1)
|
||||
for {
|
||||
@@ -72,18 +55,14 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
|
||||
}
|
||||
|
||||
func TestNewParallelExecutor(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
exec, count, maxActive := concurrencyProbe(2)
|
||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
||||
|
||||
// parallelism below 1 falls back to a single worker
|
||||
exec, count, maxActive = concurrencyProbe(1)
|
||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
||||
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
|
||||
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
|
||||
assert.Equal(t, int32(3), count.Load())
|
||||
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||
@@ -99,66 +78,49 @@ func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||
assert.ErrorIs(err, context.Canceled)
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorFailed(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
count := 0
|
||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count++
|
||||
return errors.New("fake error")
|
||||
})
|
||||
err := NewParallelExecutor(1, errorWorkflow)(ctx)
|
||||
assert.Equal(1, count)
|
||||
assert.ErrorIs(context.Canceled, err)
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorCanceled(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
errExpected := errors.New("fake error")
|
||||
|
||||
var count atomic.Int32
|
||||
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count.Add(1)
|
||||
return nil
|
||||
})
|
||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count.Add(1)
|
||||
return errExpected
|
||||
})
|
||||
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
|
||||
assert.Equal(int32(3), count.Load())
|
||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
||||
var successCount atomic.Int32
|
||||
executors := make([]Executor, 5)
|
||||
for i := range executors {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
if i == 2 {
|
||||
return errors.New("fake error")
|
||||
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
canceled bool
|
||||
parallel, tasks int
|
||||
}{
|
||||
"failure": {false, 2, 5},
|
||||
"canceled": {true, 2, 5},
|
||||
"canceled single task": {true, 1, 1},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
if tc.canceled {
|
||||
cancel()
|
||||
}
|
||||
var calls atomic.Int32
|
||||
executors := make([]Executor, tc.tasks)
|
||||
for i := range executors {
|
||||
executors[i] = func(_ctx context.Context) error {
|
||||
calls.Add(1)
|
||||
if i == tc.tasks/2 {
|
||||
return errors.New("fake error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
successCount.Add(1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
require.Error(t, NewParallelExecutor(2, executors...)(t.Context()))
|
||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
||||
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
|
||||
if tc.canceled {
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
} else {
|
||||
require.EqualError(t, err, "fake error")
|
||||
}
|
||||
assert.Equal(t, int32(tc.tasks), calls.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
var calls []string
|
||||
record := func(name string) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return func(_ctx context.Context) error {
|
||||
calls = append(calls, name)
|
||||
return nil
|
||||
}
|
||||
@@ -170,19 +132,42 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
||||
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
||||
|
||||
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("calls = %v, want %v", calls, want)
|
||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||
}
|
||||
|
||||
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||
backoff := RetryBackoff
|
||||
RetryBackoff = func() time.Duration { return 0 }
|
||||
t.Cleanup(func() { RetryBackoff = backoff })
|
||||
transient := errors.New("transient")
|
||||
for _, tc := range []struct {
|
||||
failures, calls int
|
||||
err error
|
||||
}{
|
||||
{failures: 2, calls: 3},
|
||||
{failures: 5, calls: 3, err: transient},
|
||||
} {
|
||||
var calls int
|
||||
err := Executor(func(context.Context) error {
|
||||
calls++
|
||||
if calls <= tc.failures {
|
||||
return transient
|
||||
}
|
||||
return nil
|
||||
}).Retry(3)(t.Context())
|
||||
require.Equal(t, tc.err, err)
|
||||
require.Equal(t, tc.calls, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||
mainErr := errors.New("main failed")
|
||||
finalErr := errors.New("cleanup failed")
|
||||
|
||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||
require.Error(t, err)
|
||||
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
|
||||
t.Fatalf("finally error = %q, want both cleanup and original error", err)
|
||||
for _, mainErr := range []error{context.Canceled, nil} {
|
||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||
require.ErrorIs(t, err, finalErr)
|
||||
if mainErr != nil {
|
||||
require.ErrorIs(t, err, mainErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
+287
-16
@@ -8,7 +8,9 @@ import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
@@ -16,14 +18,36 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
)
|
||||
|
||||
var actionFetches sync.Map
|
||||
|
||||
var symlink = (*os.Root).Symlink
|
||||
|
||||
var symlinkFallback = runtime.GOOS == "windows"
|
||||
|
||||
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||
|
||||
type treeFile struct {
|
||||
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||
Size int64
|
||||
}
|
||||
|
||||
type actionFetch struct {
|
||||
done chan struct{}
|
||||
dir, sha string
|
||||
err error
|
||||
retry bool // the leader was cancelled or panicked, so followers fetch themselves
|
||||
}
|
||||
|
||||
var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/runner backs off
|
||||
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
||||
return min(max(time.Duration(seconds)*time.Second, 10*time.Second), 10*time.Minute)
|
||||
@@ -32,6 +56,36 @@ var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/ru
|
||||
}
|
||||
|
||||
func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) { // input.Dir is one clone URL and ref's, holding a directory per commit
|
||||
if !reflect.ValueOf(input).Comparable() {
|
||||
return fetchAction(ctx, input)
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
call := &actionFetch{done: make(chan struct{}), retry: true}
|
||||
previous, loaded := actionFetches.LoadOrStore(input, call)
|
||||
if !loaded {
|
||||
func() {
|
||||
defer func() { actionFetches.Delete(input); close(call.done) }()
|
||||
call.dir, call.sha, call.err = fetchAction(ctx, input)
|
||||
call.retry = ctx.Err() != nil
|
||||
}()
|
||||
if call.retry {
|
||||
continue
|
||||
}
|
||||
return call.dir, call.sha, call.err
|
||||
}
|
||||
call, _ = previous.(*actionFetch)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-call.done:
|
||||
if !call.retry && ctx.Err() == nil {
|
||||
return call.dir, call.sha, call.err
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", ctx.Err()
|
||||
}
|
||||
|
||||
func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) {
|
||||
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
@@ -45,9 +99,14 @@ func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, s
|
||||
record := filepath.Join(input.Dir, "ref")
|
||||
data, _ := os.ReadFile(record)
|
||||
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||
if !isFullObjectID(sha) || !exists(tree(sha)) {
|
||||
switch {
|
||||
case !isFullObjectID(sha):
|
||||
sha, etag = "", ""
|
||||
} else if input.OfflineMode || isFullObjectID(input.Ref) {
|
||||
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||
case !treeIntact(input.Dir, sha):
|
||||
sha, etag = "", ""
|
||||
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||
return tree(sha), sha, nil
|
||||
}
|
||||
|
||||
@@ -132,7 +191,7 @@ func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, toke
|
||||
newEtag := response.Header.Get("ETag")
|
||||
gz, err := gzip.NewReader(response.Body)
|
||||
if err == nil {
|
||||
sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||
@@ -150,18 +209,18 @@ func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, e
|
||||
clone := input
|
||||
clone.Dir = filepath.Join(input.Dir, "git")
|
||||
sha, _, err := resolveClone(ctx, clone)
|
||||
if err != nil || exists(filepath.Join(input.Dir, sha)) {
|
||||
if err != nil || treeIntact(input.Dir, sha) {
|
||||
return sha, err
|
||||
}
|
||||
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
|
||||
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||
return sha, err
|
||||
}
|
||||
|
||||
func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -174,6 +233,9 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
defer root.Close()
|
||||
|
||||
sha := ""
|
||||
sizes := map[string]int64{}
|
||||
links := map[string]string{}
|
||||
var deferred []string
|
||||
reader := tar.NewReader(archive)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
@@ -184,7 +246,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
return "", err
|
||||
}
|
||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
|
||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||
return sha, nil
|
||||
}
|
||||
continue
|
||||
@@ -200,11 +262,10 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
case tar.TypeDir:
|
||||
err = root.MkdirAll(name, 0o755)
|
||||
case tar.TypeReg:
|
||||
err = writeTreeFile(root, name, header.Mode, reader)
|
||||
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||
case tar.TypeSymlink:
|
||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||
err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
}
|
||||
links[filepath.Clean(name)] = header.Linkname
|
||||
deferred = append(deferred, filepath.Clean(name))
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -213,23 +274,233 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
if !isFullObjectID(sha) {
|
||||
return "", errors.New("the archive names no commit")
|
||||
}
|
||||
linksByParent := map[string][]string{}
|
||||
if symlinkFallback {
|
||||
for name := range links {
|
||||
linksByParent[filepath.Dir(name)] = append(linksByParent[filepath.Dir(name)], filepath.Base(name))
|
||||
}
|
||||
}
|
||||
for requireTarget := symlinkFallback; len(deferred) > 0; {
|
||||
unresolved := deferred[:0]
|
||||
for _, name := range deferred {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if requireTarget {
|
||||
if _, err := root.Stat(filepath.Dir(name) + string(filepath.Separator) + links[name]); err != nil {
|
||||
unresolved = append(unresolved, name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := symlink(root, links[name], name); err == nil || !symlinkFallback {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := copyTreeLink(ctx, root, name, links, linksByParent, sizes); err != nil {
|
||||
return "", fmt.Errorf("materialize symlink %q: %w", filepath.ToSlash(name), err)
|
||||
}
|
||||
}
|
||||
if len(unresolved) == len(deferred) {
|
||||
requireTarget = false
|
||||
}
|
||||
deferred = unresolved
|
||||
}
|
||||
if err := root.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sha, os.Rename(temp, filepath.Join(dir, sha))
|
||||
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||
}
|
||||
|
||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
|
||||
func resolveTreeLink(name string, links map[string]string) (string, bool) {
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(name, string(filepath.Separator))
|
||||
var resolved []string
|
||||
for count := 0; len(parts) > 0; {
|
||||
part := parts[0]
|
||||
parts = parts[1:]
|
||||
switch part {
|
||||
case "", ".":
|
||||
continue
|
||||
case "..":
|
||||
if len(resolved) == 0 {
|
||||
return "", false
|
||||
}
|
||||
resolved = resolved[:len(resolved)-1]
|
||||
continue
|
||||
}
|
||||
target, ok := links[filepath.Join(append(resolved, part)...)]
|
||||
if !ok {
|
||||
resolved = append(resolved, part)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if target == "" || os.IsPathSeparator(target[0]) || filepath.VolumeName(target) != "" || count > 255 {
|
||||
return "", false
|
||||
}
|
||||
parts = append(strings.Split(filepath.FromSlash(target), string(filepath.Separator)), parts...)
|
||||
}
|
||||
return filepath.Join(append([]string{"."}, resolved...)...), true
|
||||
}
|
||||
|
||||
var (
|
||||
errTreeLinkTarget = errors.New("unmaterializable symlink target")
|
||||
errTreeLinkCycle = fmt.Errorf("cyclic %w", errTreeLinkTarget)
|
||||
)
|
||||
|
||||
func copyTreeLink(ctx context.Context, root *os.Root, name string, links map[string]string, linksByParent map[string][]string, sizes map[string]int64) error {
|
||||
writeLinkText := func(destination, target string) error { // as git writes links it cannot create
|
||||
if err := root.RemoveAll(destination); err != nil {
|
||||
return err
|
||||
}
|
||||
for written := range sizes {
|
||||
if written == destination || strings.HasPrefix(written, destination+string(filepath.Separator)) {
|
||||
delete(sizes, written)
|
||||
}
|
||||
}
|
||||
size, err := writeTreeFile(root, destination, 0o644, strings.NewReader(target))
|
||||
sizes[destination] = size
|
||||
return err
|
||||
}
|
||||
active := map[string]bool{}
|
||||
var copyTarget func(string, string) error
|
||||
copyTarget = func(source, destination string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source, ok := resolveTreeLink(source, links)
|
||||
if !ok {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
file, err := root.Open(source)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode().IsRegular() {
|
||||
sizes[destination], err = writeTreeFile(root, destination, int64(info.Mode().Perm()), file)
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
if active[source] {
|
||||
return errTreeLinkCycle
|
||||
}
|
||||
active[source] = true
|
||||
defer delete(active, source)
|
||||
entries, err := file.ReadDir(-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children := map[string]bool{}
|
||||
for _, entry := range entries {
|
||||
children[entry.Name()] = true
|
||||
}
|
||||
for _, child := range linksByParent[source] {
|
||||
children[child] = true
|
||||
}
|
||||
if err := root.MkdirAll(destination, info.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
for child := range children {
|
||||
childSource, childDestination := filepath.Join(source, child), filepath.Join(destination, child)
|
||||
err := copyTarget(childSource, childDestination)
|
||||
if target, isLink := links[childSource]; isLink && errors.Is(err, errTreeLinkTarget) && !errors.Is(err, errTreeLinkCycle) {
|
||||
err = writeLinkText(childDestination, target)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := copyTarget(name, name); !errors.Is(err, errTreeLinkTarget) {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeLinkText(name, links[name])
|
||||
}
|
||||
|
||||
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||
|
||||
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||
files := make([]treeFile, 0, len(sizes))
|
||||
for name, size := range sizes {
|
||||
files = append(files, treeFile{[]byte(name), size})
|
||||
}
|
||||
data, err := json.Marshal(files)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||
return err
|
||||
}
|
||||
tree := filepath.Join(dir, sha)
|
||||
if !exists(tree) {
|
||||
return os.Rename(temp, tree)
|
||||
}
|
||||
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temp, tree)
|
||||
}
|
||||
|
||||
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||
func treeIntact(dir, sha string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||
return false
|
||||
}
|
||||
var files []treeFile
|
||||
if json.Unmarshal(data, &files) != nil {
|
||||
return false
|
||||
}
|
||||
for _, file := range files {
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||
perm := os.FileMode(0o644)
|
||||
if mode&0o111 != 0 {
|
||||
perm = 0o755
|
||||
}
|
||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
_, err = io.Copy(file, content)
|
||||
return errors.Join(err, file.Close())
|
||||
n, err := io.Copy(file, content)
|
||||
return n, errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func exists(name string) bool {
|
||||
|
||||
@@ -6,7 +6,9 @@ package git
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
@@ -14,11 +16,15 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -30,13 +36,15 @@ func TestFetchAction(t *testing.T) {
|
||||
|
||||
root := t.TempDir()
|
||||
remote := filepath.Join(root, "org", "action")
|
||||
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
||||
require.NoError(t, os.CopyFS(remote, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||
require.NoError(t, gitCmd("-C", remote, "update-index", "--chmod=+x", "run.sh"))
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||
expectedSha := gitRevParse(t, remote, "main")
|
||||
|
||||
gitPath, err := exec.LookPath("git")
|
||||
require.NoError(t, err)
|
||||
@@ -44,6 +52,8 @@ func TestFetchAction(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var archiveRequests []http.Header
|
||||
var mirror *httptest.Server
|
||||
archiveCache := map[string][]byte{}
|
||||
revisions := map[string]string{}
|
||||
requests, archives, failures := 0, "", 0
|
||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
mu.Lock()
|
||||
@@ -70,7 +80,14 @@ func TestFetchAction(t *testing.T) {
|
||||
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||
sha, ok := revisions[ref]
|
||||
if !ok {
|
||||
var err error
|
||||
sha, err = gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||
if err == nil {
|
||||
revisions[ref] = sha
|
||||
}
|
||||
}
|
||||
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||
if request.Header.Get("If-None-Match") == etag {
|
||||
writer.WriteHeader(http.StatusNotModified)
|
||||
@@ -80,7 +97,12 @@ func TestFetchAction(t *testing.T) {
|
||||
if archives != "unprefixed" {
|
||||
args = append(args, "--prefix=action/")
|
||||
}
|
||||
archive, _ := runGit(request.Context(), remote, nil, args...)
|
||||
archiveKey := strings.Join(args, "\x00")
|
||||
archive, ok := archiveCache[archiveKey]
|
||||
if !ok {
|
||||
archive, _ = runGit(request.Context(), remote, nil, args...)
|
||||
archiveCache[archiveKey] = archive
|
||||
}
|
||||
writer.Header().Set("ETag", etag)
|
||||
_, _ = writer.Write(archive)
|
||||
})
|
||||
@@ -102,17 +124,87 @@ func TestFetchAction(t *testing.T) {
|
||||
return NewGitCloneExecutorInput{URL: server.URL + "/org/action", Ref: ref, Dir: t.TempDir(), Token: "secret"}
|
||||
}
|
||||
|
||||
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
input := newInput("main")
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
releaseFetches := sync.OnceFunc(func() { close(release) })
|
||||
t.Cleanup(releaseFetches)
|
||||
var once sync.Once
|
||||
var panicked atomic.Bool
|
||||
transport := actionFetchTransport(func(request *http.Request) (*http.Response, error) {
|
||||
once.Do(func() { close(started) })
|
||||
select {
|
||||
case <-request.Context().Done():
|
||||
return nil, request.Context().Err()
|
||||
case <-release:
|
||||
}
|
||||
if scenario == "leader panics" && panicked.CompareAndSwap(false, true) {
|
||||
panic("broken archive")
|
||||
}
|
||||
return http.DefaultTransport.RoundTrip(request)
|
||||
})
|
||||
input.Transport = &transport
|
||||
leaderCtx, cancelLeader := context.WithCancel(t.Context())
|
||||
defer cancelLeader()
|
||||
waiting := &actionFetchContext{Context: t.Context(), waiting: make(chan struct{})}
|
||||
followerInput := input
|
||||
expectedRequests := 1
|
||||
if scenario == "different token" {
|
||||
followerInput.Token, expectedRequests = "other", 2
|
||||
}
|
||||
leaderDone, followerDone := make(chan error, 1), make(chan error, 1)
|
||||
fetch := func(ctx context.Context, input NewGitCloneExecutorInput, done chan<- error) {
|
||||
var err error
|
||||
defer func() { done <- err }()
|
||||
defer common.RecoverPanic(&err)
|
||||
var tree, sha string
|
||||
if tree, sha, err = FetchAction(ctx, input); err == nil {
|
||||
assert.Equal(t, expectedSha, sha)
|
||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||
}
|
||||
}
|
||||
go fetch(leaderCtx, input, leaderDone)
|
||||
<-started
|
||||
go fetch(waiting, followerInput, followerDone)
|
||||
<-waiting.waiting
|
||||
if scenario == "leader cancelled" {
|
||||
cancelLeader()
|
||||
require.ErrorIs(t, <-leaderDone, context.Canceled)
|
||||
}
|
||||
releaseFetches()
|
||||
switch scenario {
|
||||
case "leader cancelled":
|
||||
case "leader panics":
|
||||
require.ErrorContains(t, <-leaderDone, "panic: broken archive")
|
||||
default:
|
||||
require.NoError(t, <-leaderDone)
|
||||
}
|
||||
require.NoError(t, <-followerDone)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, expectedRequests)
|
||||
if scenario == "different token" {
|
||||
assert.NotEqual(t, headers[0].Get("Authorization"), headers[1].Get("Authorization"))
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("downloads, revalidates and follows a moved tag", func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
input := newInput("v1")
|
||||
tree, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "v1"), sha)
|
||||
assert.Equal(t, expectedSha, sha)
|
||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.EqualValues(t, 0o755, info.Mode().Perm())
|
||||
}
|
||||
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", link)
|
||||
@@ -123,9 +215,13 @@ func TestFetchAction(t *testing.T) {
|
||||
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||
expectedSha = gitRevParse(t, remote, "v1")
|
||||
mu.Lock()
|
||||
clear(revisions)
|
||||
mu.Unlock()
|
||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "v1"), movedSha)
|
||||
assert.Equal(t, expectedSha, movedSha)
|
||||
assert.NotEqual(t, tree, moved)
|
||||
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||
|
||||
@@ -133,10 +229,17 @@ func TestFetchAction(t *testing.T) {
|
||||
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, movedSha, offlineSha)
|
||||
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||
_, _, err = FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||
reused, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, moved, reused)
|
||||
|
||||
requests, headers := served()
|
||||
assert.Equal(t, 3, requests)
|
||||
require.Len(t, headers, 3)
|
||||
assert.Equal(t, 4, requests)
|
||||
require.Len(t, headers, 4)
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||
@@ -145,7 +248,7 @@ func TestFetchAction(t *testing.T) {
|
||||
|
||||
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
sha := gitRevParse(t, remote, "main")
|
||||
sha := expectedSha
|
||||
input := newInput(sha)
|
||||
for range 2 {
|
||||
tree, got, err := FetchAction(t.Context(), input)
|
||||
@@ -160,12 +263,74 @@ func TestFetchAction(t *testing.T) {
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
})
|
||||
|
||||
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||
sha := expectedSha
|
||||
for _, mode := range []string{"prefixed", "none"} {
|
||||
serve(mode, 0)
|
||||
input := newInput(sha)
|
||||
tree, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
yml := filepath.Join(tree, "action.yml")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||
_, _, err = FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
for _, damage := range []func() error{
|
||||
func() error { return os.Truncate(yml, 4) },
|
||||
func() error { return os.Remove(tree + treeSizes) },
|
||||
} {
|
||||
require.NoError(t, damage(), mode)
|
||||
_, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
body, err := os.ReadFile(yml)
|
||||
require.NoError(t, err, mode)
|
||||
assert.Equal(t, "name: action\n", string(body), mode)
|
||||
}
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 3, mode)
|
||||
}
|
||||
|
||||
serve("prefixed", 0)
|
||||
input := newInput(sha)
|
||||
tree, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
yml := filepath.Join(tree, "action.yml")
|
||||
require.NoError(t, os.Truncate(yml, 4))
|
||||
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||
require.NoError(t, err)
|
||||
release := sync.OnceFunc(unlock)
|
||||
defer release()
|
||||
waiting := make(chan struct{})
|
||||
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||
close(waiting)
|
||||
return AcquireCloneLock(ctx, dir)
|
||||
}
|
||||
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := FetchAction(t.Context(), input)
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||
case <-waiting:
|
||||
}
|
||||
info, err := os.Stat(yml)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||
release()
|
||||
require.NoError(t, <-done)
|
||||
body, err := os.ReadFile(yml)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "name: action\n", string(body))
|
||||
})
|
||||
|
||||
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||
serve("redirect", 2)
|
||||
input := newInput("main")
|
||||
_, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
||||
assert.Equal(t, expectedSha, sha)
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
_, headers := served()
|
||||
require.Len(t, headers, 4)
|
||||
@@ -178,7 +343,7 @@ func TestFetchAction(t *testing.T) {
|
||||
serve(mode, 0)
|
||||
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||
require.NoError(t, err, mode)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha, mode)
|
||||
assert.Equal(t, expectedSha, sha, mode)
|
||||
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||
_, headers := served()
|
||||
@@ -193,13 +358,30 @@ func TestFetchAction(t *testing.T) {
|
||||
input.URL = "https://example.invalid/org/action"
|
||||
_, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
||||
assert.Equal(t, expectedSha, sha)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 1)
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
})
|
||||
}
|
||||
|
||||
type actionFetchTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (transport actionFetchTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return transport(request)
|
||||
}
|
||||
|
||||
type actionFetchContext struct {
|
||||
context.Context
|
||||
waiting chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (ctx *actionFetchContext) Done() <-chan struct{} {
|
||||
ctx.once.Do(func() { close(ctx.waiting) })
|
||||
return ctx.Context.Done()
|
||||
}
|
||||
|
||||
func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
outside := t.TempDir()
|
||||
for name, entries := range map[string][]*tar.Header{
|
||||
@@ -216,10 +398,176 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(&archive, dir, false)
|
||||
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||
require.Error(t, err)
|
||||
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTreeCopiesDeferredLinks(t *testing.T) {
|
||||
originalSymlink, originalFallback := symlink, symlinkFallback
|
||||
symlink = func(_root *os.Root, _target, _name string) error { return errors.New("symlink privilege unavailable") }
|
||||
symlinkFallback = true
|
||||
t.Cleanup(func() { symlink, symlinkFallback = originalSymlink, originalFallback })
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
links [][2]string
|
||||
files []string
|
||||
text bool
|
||||
}{
|
||||
{name: "file chain", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "dist/index.js"}}, files: []string{"index.js", "entry.js"}},
|
||||
{name: "directory chain", links: [][2]string{{"lib", "bundle"}, {"bundle", "dist"}, {"dist/entry.js", "index.js"}}, files: []string{"lib/index.js", "lib/entry.js", "lib/nested/index.js", "bundle/index.js", "bundle/entry.js", "bundle/nested/index.js"}},
|
||||
{name: "directory in target path", links: [][2]string{{"index.js", "bundle/nested/index.js"}, {"bundle", "dist"}}, files: []string{"index.js", "bundle/nested/index.js"}},
|
||||
{name: "parent after directory link", links: [][2]string{{"index.js", "bundle/../index.js"}, {"bundle", "dist/nested"}}, files: []string{"index.js", "bundle/index.js"}},
|
||||
{name: "outside", links: [][2]string{{"index.js", "../outside.js"}}, text: true},
|
||||
{name: "absolute", links: [][2]string{{"index.js", t.TempDir()}}, text: true},
|
||||
{name: "dangling", links: [][2]string{{"index.js", "missing.js"}}, text: true},
|
||||
{name: "dangling child of directory link", links: [][2]string{{"lib", "dist"}, {"dist/optional", "missing.js"}}, files: []string{"lib/index.js", "lib/nested/index.js"}},
|
||||
{name: "cycle", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "index.js"}}, text: true},
|
||||
{name: "directory cycle", links: [][2]string{{"dist/loop", "."}}, text: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
sha := strings.Repeat("a", 40)
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist/nested", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
for _, link := range test.links {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: link[0], Typeflag: tar.TypeSymlink, Linkname: link[1]}))
|
||||
}
|
||||
content := "console.log('action');\n"
|
||||
for _, name := range []string{"dist/index.js", "dist/nested/index.js"} {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(content))}))
|
||||
_, err := writer.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
if test.text {
|
||||
for _, link := range test.links {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, link[1], string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
}
|
||||
}
|
||||
for _, name := range test.files {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), nil, 0o644))
|
||||
assert.False(t, treeIntact(dir, sha), name)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), []byte(content), 0o755))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||
}
|
||||
sha := strings.Repeat("a", 40)
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||
_, err := writer.Write([]byte("x"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||
require.NoError(t, err)
|
||||
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
}
|
||||
|
||||
func TestExtractTreeCreatesLinksInPlatformOrder(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires symlink privileges")
|
||||
}
|
||||
original, originalFallback := symlink, symlinkFallback
|
||||
var created []string
|
||||
symlink = func(root *os.Root, target, name string) error {
|
||||
created = append(created, name)
|
||||
return root.Symlink(target, name)
|
||||
}
|
||||
t.Cleanup(func() { symlink, symlinkFallback = original, originalFallback })
|
||||
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||
for _, entry := range []*tar.Header{
|
||||
{Name: "a", Typeflag: tar.TypeSymlink, Linkname: "b"},
|
||||
{Name: "b", Typeflag: tar.TypeSymlink, Linkname: "dist"},
|
||||
{Name: "dangling", Typeflag: tar.TypeSymlink, Linkname: "missing"},
|
||||
{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||
} {
|
||||
require.NoError(t, writer.WriteHeader(entry))
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
fallback bool
|
||||
want []string
|
||||
}{
|
||||
{name: "native", want: []string{"a", "b", "dangling"}},
|
||||
{name: "fallback", fallback: true, want: []string{"b", "a", "dangling"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
symlinkFallback = test.fallback
|
||||
created = nil
|
||||
_, err := extractTree(t.Context(), bytes.NewReader(archive.Bytes()), t.TempDir(), false)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, created)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyTreeLinkCancellationBetweenEntries(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
defer root.Close()
|
||||
require.NoError(t, root.Mkdir("source", 0o755))
|
||||
for _, name := range []string{"first", "second"} {
|
||||
_, err := writeTreeFile(root, filepath.Join("source", name), 0o644, strings.NewReader("content"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
sizes := map[string]int64{}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
observed := &treeCopyContext{Context: ctx, check: func() {
|
||||
if len(sizes) == 1 {
|
||||
cancel()
|
||||
}
|
||||
}}
|
||||
require.ErrorIs(t, copyTreeLink(observed, root, "copy", map[string]string{"copy": "source"}, nil, sizes), context.Canceled)
|
||||
assert.Len(t, sizes, 1)
|
||||
}
|
||||
|
||||
type treeCopyContext struct {
|
||||
context.Context
|
||||
check func()
|
||||
}
|
||||
|
||||
func (ctx *treeCopyContext) Err() error {
|
||||
ctx.check()
|
||||
return ctx.Context.Err()
|
||||
}
|
||||
+61
-17
@@ -5,11 +5,13 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -26,6 +28,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
"gitea.com/gitea/runner/internal/pkg/process"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -79,25 +82,25 @@ func (e *Error) Unwrap() error { return e.err }
|
||||
func (e *Error) Commit() string { return e.commit }
|
||||
|
||||
// FindGitRevision get the current git revision
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
||||
func FindGitRevision(ctx context.Context, file string) (string, error) {
|
||||
logger := common.Logger(ctx)
|
||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||
return "", "", err
|
||||
return "", err
|
||||
}
|
||||
if len(sha) < 7 {
|
||||
return "", "", errors.New("HEAD could not be resolved")
|
||||
return "", errors.New("HEAD could not be resolved")
|
||||
}
|
||||
logger.Debugf("Found revision: %s", sha)
|
||||
return sha[:7], sha, nil
|
||||
return sha, nil
|
||||
}
|
||||
|
||||
// FindGitRef get the current git ref
|
||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||
logger := common.Logger(ctx)
|
||||
logger.Debugf("Loading revision from git directory")
|
||||
_, revision, err := FindGitRevision(ctx, file)
|
||||
revision, err := FindGitRevision(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -182,6 +185,7 @@ type NewGitCloneExecutorInput struct {
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
@@ -320,9 +324,18 @@ func cachedRef(ref string) string {
|
||||
|
||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||
if !isFullObjectID(ref) {
|
||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
||||
return sha, nil
|
||||
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||
if strings.ContainsAny(ref, "\x00\n") {
|
||||
for _, revision := range revisions {
|
||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||
return sha, nil
|
||||
}
|
||||
}
|
||||
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||
if isFullObjectID(sha) {
|
||||
return sha, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -377,6 +390,21 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
return config
|
||||
}
|
||||
|
||||
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||
var config []string
|
||||
if certFile != "" {
|
||||
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||
}
|
||||
if caFile != "" {
|
||||
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||
}
|
||||
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||
config = append(config, prefix+"sslBackend=openssl")
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
@@ -427,17 +455,20 @@ func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
||||
}
|
||||
|
||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||
}
|
||||
|
||||
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Stdin = stdin
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||
name, _, _ := strings.Cut(entry, "=")
|
||||
return slices.Contains(LocalEnvVars, name)
|
||||
@@ -446,15 +477,28 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
var treeKill *process.TreeKill
|
||||
if runtime.GOOS == "windows" {
|
||||
treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
|
||||
} else {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stdout bytes.Buffer
|
||||
var stderr strings.Builder
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
err := cmd.Start()
|
||||
if err == nil {
|
||||
if treeKill != nil {
|
||||
if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
|
||||
defer killer.Close()
|
||||
}
|
||||
}
|
||||
err = cmd.Wait()
|
||||
}
|
||||
return output, nil
|
||||
if err != nil {
|
||||
return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
+162
-203
@@ -29,6 +29,47 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var gitFixtureDir string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
var err error
|
||||
gitFixtureDir, err = os.MkdirTemp("", "runner-git-fixture-")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
code := 1
|
||||
if err = gitCmd("init", "--quiet", "--template=", "--bare", "--initial-branch=main", filepath.Join(gitFixtureDir, "remote")); err == nil {
|
||||
err = gitCmd("init", "--quiet", "--template=", "--initial-branch=main", filepath.Join(gitFixtureDir, "work"))
|
||||
}
|
||||
if err == nil {
|
||||
code = m.Run()
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
if err := os.RemoveAll(gitFixtureDir); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
code = 1
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestSSLConfig(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
prefix, certFile, caFile, executionOS string
|
||||
expected []string
|
||||
}{
|
||||
{"http.https://example.com/.", "cert.pem", "ca.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslCAInfo=ca.pem", "http.https://example.com/.sslBackend=openssl"}},
|
||||
{"http.", "", "ca.pem", "windows", []string{"http.sslCAInfo=ca.pem", "http.sslBackend=openssl"}},
|
||||
{"http.", "cert.pem", "ca.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslCAInfo=ca.pem"}},
|
||||
{"http.", "", "", "Windows", nil},
|
||||
} {
|
||||
t.Run(testcase.prefix+testcase.certFile+testcase.caFile+testcase.executionOS, func(t *testing.T) {
|
||||
assert.Equal(t, testcase.expected, SSLConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.caFile, testcase.executionOS))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindGitSlug(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
remoteURL string
|
||||
@@ -67,42 +108,19 @@ func TestErrorWrapsCommitAndCause(t *testing.T) {
|
||||
require.Equal(t, "abc123", err.Commit())
|
||||
}
|
||||
|
||||
func cleanGitHooks(dir string) error {
|
||||
hooksDir := filepath.Join(dir, ".git", "hooks")
|
||||
files, err := os.ReadDir(hooksDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
relName := filepath.Join(hooksDir, f.Name())
|
||||
if err := os.Remove(relName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
const remoteURL = "https://github.com/owner/repo.git"
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, cleanGitHooks(basedir))
|
||||
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||
|
||||
shortSHA, sha, err := FindGitRevision(t.Context(), basedir)
|
||||
sha, err := FindGitRevision(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
||||
require.Len(t, sha, 64)
|
||||
require.Equal(t, sha[:7], shortSHA)
|
||||
|
||||
ref, err := FindGitRef(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
@@ -122,112 +140,84 @@ func TestGitFindRef(t *testing.T) {
|
||||
basedir := t.TempDir()
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Prepare func(t *testing.T, dir string)
|
||||
Assert func(t *testing.T, ref string, err error)
|
||||
commands [][]string
|
||||
ref string
|
||||
}{
|
||||
"new_repo": {
|
||||
Prepare: func(t *testing.T, dir string) {},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.Error(t, err)
|
||||
},
|
||||
},
|
||||
"new_repo": {},
|
||||
"new_repo_with_commit": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/master", ref)
|
||||
},
|
||||
commands: [][]string{{"commit", "--allow-empty", "-m", "msg"}},
|
||||
ref: "refs/heads/master",
|
||||
},
|
||||
"current_head_is_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "commit msg"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.2.3"))
|
||||
require.NoError(t, gitCmd("-C", dir, "checkout", "v1.2.3"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/tags/v1.2.3", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "commit msg"},
|
||||
{"tag", "v1.2.3"},
|
||||
{"checkout", "v1.2.3"},
|
||||
},
|
||||
ref: "refs/tags/v1.2.3",
|
||||
},
|
||||
"current_head_is_same_as_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "1.4.2 release"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/tags/v1.4.2", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "1.4.2 release"},
|
||||
{"tag", "v1.4.2"},
|
||||
},
|
||||
ref: "refs/tags/v1.4.2",
|
||||
},
|
||||
"current_head_is_not_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg2"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/master", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "msg"},
|
||||
{"tag", "v1.4.2"},
|
||||
{"commit", "--allow-empty", "-m", "msg2"},
|
||||
},
|
||||
ref: "refs/heads/master",
|
||||
},
|
||||
"current_head_is_another_branch": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "checkout", "-b", "mybranch"))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/mybranch", ref)
|
||||
commands: [][]string{
|
||||
{"checkout", "-b", "mybranch"},
|
||||
{"commit", "--allow-empty", "-m", "msg"},
|
||||
},
|
||||
ref: "refs/heads/mybranch",
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := filepath.Join(basedir, name)
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
|
||||
require.NoError(t, cleanGitHooks(dir))
|
||||
tt.Prepare(t, dir)
|
||||
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/master\n"), 0o644))
|
||||
for _, command := range tt.commands {
|
||||
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
||||
}
|
||||
ref, err := FindGitRef(t.Context(), dir)
|
||||
tt.Assert(t, ref, err)
|
||||
if tt.ref == "" {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.ref, ref)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutor(t *testing.T) {
|
||||
// Build a local bare "remote" so this runs offline and fast. The cases below mirror
|
||||
// the tag/branch/sha/short-sha ref paths the executor handles, formerly exercised by
|
||||
// cloning actions/checkout and anchore/scan-action over the network.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
remoteDir, workDir := newGitRemote(t)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".gitattributes"), []byte("*.sh text\n"), 0o644))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||
|
||||
// A branch with a dash in the name (mirrors the historical scan-action@act-fails case).
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
||||
|
||||
out, err := exec.Command("git", "-C", workDir, "rev-parse", "main").Output()
|
||||
require.NoError(t, err)
|
||||
fullSha := strings.TrimSpace(string(out))
|
||||
fullSha := gitRevParse(t, workDir, "main")
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v2", "act-fails", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
var clones sync.WaitGroup
|
||||
for name, tt := range map[string]struct {
|
||||
Err error
|
||||
Ref string
|
||||
@@ -255,46 +245,43 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
Ref: fullSha[:4],
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: "https://example.invalid/action",
|
||||
Ref: tt.Ref,
|
||||
Dir: dir,
|
||||
})
|
||||
clones.Go(func() {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: "https://example.invalid/action",
|
||||
Ref: tt.Ref,
|
||||
Dir: dir,
|
||||
})
|
||||
|
||||
err := clone(t.Context())
|
||||
if tt.Err != nil {
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, err)
|
||||
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
||||
return
|
||||
}
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
marker := filepath.Join(dir, "marker")
|
||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||
err := clone(t.Context())
|
||||
if tt.Err != nil {
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tt.Err, err)
|
||||
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
marker := filepath.Join(dir, "marker")
|
||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||
})
|
||||
})
|
||||
}
|
||||
clones.Wait()
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
createRemote := func(message string) string {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
return remoteDir
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", message))
|
||||
return dir
|
||||
}
|
||||
|
||||
oldRemoteDir := createRemote("old-action")
|
||||
@@ -341,13 +328,7 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
|
||||
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
remoteDir, workDir := newGitRemote(t, "initial", "feature-1")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
dir := t.TempDir()
|
||||
@@ -380,16 +361,9 @@ func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||
|
||||
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with a single commit on main.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
remoteDir, workDir := newGitRemote(t, "initial")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Prime the cache with an online clone of main.
|
||||
cacheDir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
@@ -398,8 +372,6 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
})(t.Context()))
|
||||
|
||||
t.Run("cached branch resolves without fetching", func(t *testing.T) {
|
||||
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
|
||||
// finds no local refs/heads/<ref>.
|
||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
@@ -414,7 +386,6 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("unresolvable cached ref returns error", func(t *testing.T) {
|
||||
// The ref was never cached; offline mode cannot resolve it and must return an error.
|
||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "never-fetched",
|
||||
@@ -427,50 +398,33 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
|
||||
func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
for _, m := range []string{"c1", "c2", "c3"} {
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
|
||||
}
|
||||
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v1"))
|
||||
|
||||
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
||||
|
||||
t.Run("branch is cloned shallowly", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
})
|
||||
|
||||
t.Run("tag is cloned shallowly", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "v1", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
})
|
||||
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, []string{"c3"}, gitSubjects(t, dir), "only the tip commit should be present")
|
||||
})
|
||||
}
|
||||
|
||||
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
|
||||
require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
|
||||
require.NoError(t, gitCmd("clone", "--bare", "--template=", "--config", "uploadpack.allowAnySHA1InWant="+strconv.FormatBool(allowUnadvertised), remoteDir, remote))
|
||||
return remote
|
||||
}
|
||||
|
||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||
@@ -479,6 +433,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remote := unadvertisedRemote(t, true)
|
||||
dir := t.TempDir()
|
||||
cloneAt := func(ref string) error {
|
||||
@@ -489,7 +444,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
|
||||
require.NoError(t, cloneAt(unadvertisedSHA))
|
||||
assert.FileExists(t, shallowMarker(dir))
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
assert.Len(t, gitSubjects(t, dir), 1)
|
||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||
|
||||
olderSHA := gitRevParse(t, workDir, "HEAD~2")
|
||||
@@ -502,6 +457,8 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "c2", "c3")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
@@ -514,19 +471,17 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.Equal(t, []string{"c4"}, gitSubjects(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
})
|
||||
|
||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
||||
work := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
||||
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
||||
require.NoError(t, gitCmd("clone", "--bare", "--template=", work, remote))
|
||||
sha := gitRevParse(t, work, "HEAD")
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||
@@ -541,20 +496,15 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
|
||||
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "pinned"))
|
||||
remoteDir, workDir := newGitRemote(t, "pinned")
|
||||
pinned := gitRevParse(t, workDir, "HEAD")
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", pinned))
|
||||
|
||||
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
|
||||
@@ -564,6 +514,21 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
||||
t.Helper()
|
||||
remoteDir, workDir := t.TempDir(), t.TempDir()
|
||||
require.NoError(t, os.CopyFS(remoteDir, os.DirFS(filepath.Join(gitFixtureDir, "remote"))))
|
||||
require.NoError(t, os.CopyFS(workDir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
config, err := os.ReadFile(filepath.Join(workDir, ".git", "config"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".git", "config"),
|
||||
fmt.Appendf(config, "\n[remote \"origin\"]\n\turl = %s\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", strconv.Quote(remoteDir)), 0o644))
|
||||
for _, message := range messages {
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||
}
|
||||
return remoteDir, workDir
|
||||
}
|
||||
|
||||
func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()
|
||||
@@ -571,13 +536,11 @@ func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func gitRevCount(t *testing.T, dir string) int {
|
||||
func gitSubjects(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", "-C", dir, "rev-list", "--count", "HEAD").Output()
|
||||
out, err := exec.Command("git", "-C", dir, "log", "--format=%s").Output()
|
||||
require.NoError(t, err)
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
return strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
}
|
||||
|
||||
func gitHeadSubject(t *testing.T, dir string) string {
|
||||
@@ -591,9 +554,6 @@ func gitCmd(args ...string) error {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
// Inject a deterministic identity and ignore the host's global/system config so commits
|
||||
// succeed regardless of the host having no user.name/user.email (e.g. CI, GITHUB_ACTIONS
|
||||
// unset) or a global commit.gpgsign, and without mutating the developer's ~/.gitconfig.
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GIT_AUTHOR_NAME=Unit Test",
|
||||
"[email protected]",
|
||||
@@ -695,7 +655,6 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
|
||||
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
block := make(chan struct{})
|
||||
@@ -715,7 +674,7 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
})
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||
require.NoError(t, gitCmd("init", dir))
|
||||
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
@@ -728,8 +687,8 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
|
||||
select {
|
||||
case <-reached:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
case err := <-done:
|
||||
t.Fatalf("the executor finished before reaching the remote: %v", err)
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
@@ -14,13 +14,8 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
|
||||
|
||||
// JobError returns the job error for current context if any
|
||||
func JobError(ctx context.Context) error {
|
||||
val := ctx.Value(jobErrorContextKeyVal)
|
||||
if val != nil {
|
||||
if container, ok := val.(map[string]error); ok {
|
||||
return container["error"]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
|
||||
return container["error"]
|
||||
}
|
||||
|
||||
func SetJobError(ctx context.Context, err error) {
|
||||
@@ -31,6 +26,5 @@ func SetJobError(ctx context.Context, err error) {
|
||||
|
||||
// WithJobErrorContainer adds a value to the context as a container for an error
|
||||
func WithJobErrorContainer(ctx context.Context) context.Context {
|
||||
container := map[string]error{}
|
||||
return context.WithValue(ctx, jobErrorContextKeyVal, container)
|
||||
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
|
||||
}
|
||||
@@ -26,9 +26,7 @@ type lineWriter struct {
|
||||
|
||||
// NewLineWriter creates a new instance of a line writer
|
||||
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
||||
w := new(lineWriter)
|
||||
w.handlers = handlers
|
||||
return w
|
||||
return &lineWriter{handlers: handlers}
|
||||
}
|
||||
|
||||
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
||||
@@ -39,24 +37,15 @@ func FlushWriter(w io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *lineWriter) Write(p []byte) (n int, err error) {
|
||||
pBuf := bytes.NewBuffer(p)
|
||||
written := 0
|
||||
for {
|
||||
line, err := pBuf.ReadString('\n')
|
||||
w, _ := lw.buffer.WriteString(line)
|
||||
written += w
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return written, err
|
||||
func (lw *lineWriter) Write(p []byte) (int, error) {
|
||||
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
|
||||
_, _ = lw.buffer.Write(line)
|
||||
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||
lw.handleLine(lw.buffer.String())
|
||||
lw.buffer.Reset()
|
||||
}
|
||||
lw.handleLine(lw.buffer.String())
|
||||
lw.buffer.Reset()
|
||||
}
|
||||
|
||||
return written, nil
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
||||
@@ -72,8 +61,7 @@ func (lw *lineWriter) Flush() {
|
||||
|
||||
func (lw *lineWriter) handleLine(line string) {
|
||||
for _, h := range lw.handlers {
|
||||
ok := h(line)
|
||||
if !ok {
|
||||
if !h(line) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,24 +9,24 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLineWriter(t *testing.T) {
|
||||
lines := make([]string, 0)
|
||||
lineHandler := func(s string) bool {
|
||||
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
|
||||
var lines []string
|
||||
lineWriter := NewLineWriter(func(s string) bool {
|
||||
lines = append(lines, s)
|
||||
return true
|
||||
}
|
||||
|
||||
lineWriter := NewLineWriter(lineHandler)
|
||||
})
|
||||
|
||||
assert := assert.New(t)
|
||||
write := func(s string) {
|
||||
n, err := lineWriter.Write([]byte(s))
|
||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(len(s), n, s)
|
||||
}
|
||||
|
||||
write("")
|
||||
write("hello")
|
||||
write(" ")
|
||||
write("world!!\nextra")
|
||||
@@ -34,39 +34,17 @@ func TestLineWriter(t *testing.T) {
|
||||
write(" line\n")
|
||||
write("no newline here...")
|
||||
|
||||
assert.Len(lines, 4)
|
||||
assert.Equal("hello world!!\n", lines[0])
|
||||
assert.Equal("extra line\n", lines[1])
|
||||
assert.Equal(" and another\n", lines[2])
|
||||
assert.Equal("last line\n", lines[3])
|
||||
}
|
||||
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
|
||||
assert.Equal(expected, lines)
|
||||
|
||||
func TestLineWriterFlush(t *testing.T) {
|
||||
lines := make([]string, 0)
|
||||
lineHandler := func(s string) bool {
|
||||
lines = append(lines, s)
|
||||
return true
|
||||
}
|
||||
|
||||
lineWriter := NewLineWriter(lineHandler)
|
||||
|
||||
assert := assert.New(t)
|
||||
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
|
||||
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
|
||||
|
||||
// Only the newline-terminated line is emitted before flushing.
|
||||
assert.Equal([]string{"complete line\n"}, lines)
|
||||
|
||||
// Flushing emits the buffered, not-yet-terminated trailing line.
|
||||
FlushWriter(lineWriter)
|
||||
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
|
||||
expected = append(expected, "no newline here...")
|
||||
assert.Equal(expected, lines)
|
||||
|
||||
// Flushing again is a no-op: nothing is buffered.
|
||||
FlushWriter(lineWriter)
|
||||
assert.Len(lines, 2)
|
||||
assert.Equal(expected, lines)
|
||||
}
|
||||
|
||||
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
||||
// FlushWriter must be a safe no-op for writers that do not buffer lines.
|
||||
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
||||
}
|
||||
@@ -34,10 +34,8 @@ const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
|
||||
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
|
||||
// It affects the job logger, not the global logger.
|
||||
func LoggerHandler(ctx context.Context) logging.Handler {
|
||||
if h, ok := ctx.Value(loggerHandlerKeyVal).(logging.Handler); ok {
|
||||
return h
|
||||
}
|
||||
return nil
|
||||
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
|
||||
return h
|
||||
}
|
||||
|
||||
// WithLoggerHandler adds the extra handler for the job logger to the context
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
@@ -94,11 +95,7 @@ func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
}
|
||||
prefixFieldClashes(data, caller != nil)
|
||||
|
||||
keys := make([]string, 0, len(data))
|
||||
for k := range data {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
keys := slices.Sorted(maps.Keys(data))
|
||||
|
||||
var b bytes.Buffer
|
||||
if terminal && !opts.DisableColors {
|
||||
|
||||
@@ -73,12 +73,10 @@ func Enabled(level Level) bool { return Default().Enabled(level) }
|
||||
|
||||
func With(args ...any) *Logger { return Default().With(args...) }
|
||||
func WithError(err error) *Logger { return Default().WithError(err) }
|
||||
func Tracef(format string, args ...any) { Default().logf(LevelTrace, format, args) }
|
||||
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
|
||||
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
|
||||
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
|
||||
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
|
||||
func Trace(args ...any) { Default().log(LevelTrace, args) }
|
||||
func Debug(args ...any) { Default().log(LevelDebug, args) }
|
||||
func Info(args ...any) { Default().log(LevelInfo, args) }
|
||||
func Warn(args ...any) { Default().log(LevelWarn, args) }
|
||||
|
||||
@@ -43,7 +43,6 @@ func (l *Logger) Enabled(level Level) bool {
|
||||
return l.h.Enabled(l.ctx, level)
|
||||
}
|
||||
|
||||
func (l *Logger) Tracef(format string, args ...any) { l.logf(LevelTrace, format, args) }
|
||||
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
|
||||
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
|
||||
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
|
||||
|
||||
@@ -38,6 +38,7 @@ func (c countingStringer) String() string { *c.n++; return "x" }
|
||||
|
||||
func TestLogger(t *testing.T) {
|
||||
h := logtest.NewWithLevel(logging.LevelTrace)
|
||||
assert.Nil(t, h.LastRecord())
|
||||
l := logging.New(h)
|
||||
|
||||
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
|
||||
@@ -53,12 +54,9 @@ func TestLogger(t *testing.T) {
|
||||
assert.Equal(t, "j1", job)
|
||||
errv, _ := recs[0].Attr("error")
|
||||
require.EqualError(t, errv.(error), "boom")
|
||||
assert.Equal(t, "a1 2b", recs[1].Message) // fmt.Sprint spacing
|
||||
assert.Equal(t, "a1 2b", recs[1].Message)
|
||||
bad, _ := recs[2].Attr("!BADKEY")
|
||||
assert.Equal(t, 7, bad)
|
||||
|
||||
h.Reset()
|
||||
assert.Nil(t, h.LastRecord())
|
||||
}
|
||||
|
||||
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
|
||||
@@ -117,10 +115,8 @@ func TestDefaultLevel(t *testing.T) {
|
||||
|
||||
func TestSlogHandler(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
slogger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||
l := logging.New(logging.NewSlogHandler(slogger.Handler())).With("job", "j1")
|
||||
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
|
||||
|
||||
l.Tracef("dropped")
|
||||
assert.False(t, l.Enabled(logging.LevelTrace))
|
||||
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
|
||||
|
||||
|
||||
@@ -72,12 +72,6 @@ func (h *Handler) LastRecord() *logging.Record {
|
||||
return &r
|
||||
}
|
||||
|
||||
func (h *Handler) Reset() {
|
||||
h.store.mu.Lock()
|
||||
defer h.store.mu.Unlock()
|
||||
h.store.records = nil
|
||||
}
|
||||
|
||||
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
|
||||
func SetDefault(t testing.TB) *Handler {
|
||||
t.Helper()
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import "strings"
|
||||
|
||||
// LookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||
func LookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||
if value, ok := values[name]; ok || !caseInsensitive {
|
||||
return value, ok
|
||||
}
|
||||
for key, value := range values {
|
||||
if strings.EqualFold(key, name) {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
var zero V
|
||||
return zero, false
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
type interfaceIP struct {
|
||||
net.IP
|
||||
net.Interface
|
||||
Name string
|
||||
}
|
||||
|
||||
// GetOutboundIP returns an outbound IP address of this machine.
|
||||
@@ -43,7 +43,7 @@ func GetOutboundIP() net.IP {
|
||||
ip = v.IP
|
||||
}
|
||||
if ip.IsGlobalUnicast() {
|
||||
ips = append(ips, interfaceIP{ip, i})
|
||||
ips = append(ips, interfaceIP{ip, i.Name})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,29 +59,25 @@ func preferredIP(ips []interfaceIP) net.IP {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
ifi := ips[i].Interface
|
||||
ifj := ips[j].Interface
|
||||
first, second := ips[i], ips[j]
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
|
||||
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
ipi := ips[i].IP
|
||||
ipj := ips[j].IP
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
|
||||
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// en0 is preferred to en1
|
||||
if ifi.Name != ifj.Name {
|
||||
return ifi.Name < ifj.Name
|
||||
if first.Name != second.Name {
|
||||
return first.Name < second.Name
|
||||
}
|
||||
|
||||
// fallback
|
||||
return ipi.String() < ipj.String()
|
||||
return first.String() < second.String()
|
||||
})
|
||||
return ips[0].IP
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
"github.com/moby/moby/api/types/container"
|
||||
)
|
||||
|
||||
var hostOS = runtime.GOOS // replaced in tests
|
||||
|
||||
// ExitCodeError reports a non-zero process exit code from a container command.
|
||||
type ExitCodeError int
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -46,9 +46,9 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
|
||||
logger.Debugf("Building image from '%v'", input.ContextDir)
|
||||
|
||||
tags := []string{input.ImageTag}
|
||||
options := client.ImageBuildOptions{
|
||||
Tags: tags,
|
||||
Tags: []string{input.ImageTag},
|
||||
Labels: map[string]string{"com.gitea.runner.image": input.ImageTag}, // own image ID per tag, so removing one never deletes another job's identical build
|
||||
Remove: true,
|
||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||
Dockerfile: input.Dockerfile,
|
||||
@@ -75,12 +75,10 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
|
||||
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
||||
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
||||
|
||||
err = logDockerResponse(logger, resp.Body, err != nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return logDockerResponse(logger, resp.Body, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,6 +126,10 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
||||
}
|
||||
|
||||
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||
contextDir, err := filepath.EvalSymlinks(contextDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
seen := map[string]bool{}
|
||||
hardlinks := map[uint64]string{}
|
||||
@@ -135,8 +137,7 @@ func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, incl
|
||||
for _, include := range includes {
|
||||
include = filepath.FromSlash(include)
|
||||
var parents []parentMatch
|
||||
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
||||
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
||||
err := filepath.WalkDir(filepath.Join(contextDir, include), func(file string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
@@ -213,7 +214,7 @@ func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string,
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
}
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, filepath.ToSlash(link))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,34 +1,59 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||
var labels string
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||
labels = request.URL.Query().Get("labels")
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
t.Setenv("DOCKER_CONFIG", t.TempDir())
|
||||
t.Setenv("DOCKER_AUTH_CONFIG", "")
|
||||
|
||||
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||
})(t.Context()), "build rejected")
|
||||
require.JSONEq(t, `{"com.gitea.runner.image":"action:test"}`, labels)
|
||||
}
|
||||
|
||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "build"), 0o755))
|
||||
for name, content := range map[string]string{
|
||||
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
||||
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
||||
|
||||
contexts := []string{dir}
|
||||
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||
contexts = append(contexts, link+string(filepath.Separator))
|
||||
contexts = append(contexts, link, link+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, contextDir := range contexts {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
+25
-59
@@ -2,14 +2,14 @@
|
||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||
// * appended with license information
|
||||
// * regexp and ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
|
||||
// * cdiDeviceName in place of the container-device-interface parser's IsQualifiedName
|
||||
// * jsontext in place of json.Compact for the seccomp profile
|
||||
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
|
||||
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for nat port types
|
||||
//
|
||||
// docker/cli is licensed under the Apache License, Version 2.0.
|
||||
// See DOCKER_LICENSE for the full license text.
|
||||
@@ -154,9 +154,6 @@ type containerOptions struct {
|
||||
init bool
|
||||
umask opts.UmaskOpt
|
||||
annotations *opts.MapOpts
|
||||
|
||||
Image string
|
||||
Args []string
|
||||
}
|
||||
|
||||
// addFlags adds all command line flags that will be used by parse to the FlagSet
|
||||
@@ -354,7 +351,7 @@ type containerConfig struct {
|
||||
// parse parses the args for the specified command and generates a Config,
|
||||
// a HostConfig and returns them with the specified command.
|
||||
// If the specified args are not valid, it will return an error.
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo,unparam // verbatim copy from docker/cli
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo // verbatim copy from docker/cli
|
||||
var (
|
||||
attachStdin = copts.attach.Get("stdin")
|
||||
attachStdout = copts.attach.Get("stdout")
|
||||
@@ -422,12 +419,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
tmpfs[k] = v
|
||||
}
|
||||
|
||||
var runCmd, entrypoint []string
|
||||
|
||||
if len(copts.Args) > 0 {
|
||||
runCmd = copts.Args
|
||||
}
|
||||
|
||||
var entrypoint []string
|
||||
if copts.entrypoint != "" {
|
||||
entrypoint = []string{copts.entrypoint}
|
||||
} else if flags.Changed("entrypoint") {
|
||||
@@ -448,36 +440,15 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
portBindings := network.PortMap{}
|
||||
for port, bindings := range natPortBindings {
|
||||
p, err := network.ParsePort(string(port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
portBindings[p] = []network.PortBinding{}
|
||||
for _, b := range bindings {
|
||||
var hostIP netip.Addr
|
||||
if b.HostIP != "" {
|
||||
hostIP, err = netip.ParseAddr(b.HostIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
portBindings[p] = append(portBindings[p], network.PortBinding{
|
||||
HostIP: hostIP,
|
||||
HostPort: b.HostPort,
|
||||
})
|
||||
}
|
||||
portBindings, err := convertPortMap(natPortBindings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add published ports as exposed ports.
|
||||
exposedPorts := network.PortSet{}
|
||||
for port := range ports {
|
||||
p, err := network.ParsePort(string(port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exposedPorts[p] = struct{}{}
|
||||
exposedPorts, err := convertPortSet(ports)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Merge in exposed ports to the map of published ports
|
||||
@@ -618,11 +589,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
|
||||
deviceRequests := copts.gpus.Value()
|
||||
if len(cdiDeviceNames) > 0 {
|
||||
cdiDeviceRequest := container.DeviceRequest{
|
||||
deviceRequests = append(deviceRequests, container.DeviceRequest{
|
||||
Driver: "cdi",
|
||||
DeviceIDs: cdiDeviceNames,
|
||||
}
|
||||
deviceRequests = append(deviceRequests, cdiDeviceRequest)
|
||||
})
|
||||
}
|
||||
|
||||
resources := container.Resources{
|
||||
@@ -668,8 +638,6 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
AttachStdout: attachStdout,
|
||||
AttachStderr: attachStderr,
|
||||
Env: envVariables,
|
||||
Cmd: runCmd,
|
||||
Image: copts.Image,
|
||||
Volumes: volumes,
|
||||
Entrypoint: entrypoint,
|
||||
WorkingDir: copts.workingDir,
|
||||
@@ -756,6 +724,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isUserDefinedNetwork(mode container.NetworkMode) bool { // Docker's helper uses the host OS instead of the daemon's.
|
||||
return !mode.IsDefault() && mode != network.NetworkBridge && mode != network.NetworkHost && !mode.IsNone() && !mode.IsContainer()
|
||||
}
|
||||
|
||||
// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
|
||||
// them with the old --network-alias and --links. If returns an error if conflicting options
|
||||
// are found.
|
||||
@@ -784,7 +756,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
||||
}
|
||||
|
||||
for i, n := range copts.netMode.Value() {
|
||||
if container.NetworkMode(n.Target).IsUserDefined() {
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) {
|
||||
hasUserDefined = true
|
||||
} else {
|
||||
hasNonUserDefined = true
|
||||
@@ -810,10 +782,8 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
||||
// For backward compatibility: if no custom options are provided for the network,
|
||||
// and only a single network is specified, omit the endpoint-configuration
|
||||
// on the client (the daemon will still create it when creating the container)
|
||||
if i == 0 && len(copts.netMode.Value()) == 1 {
|
||||
if ep == nil || reflect.ValueOf(*ep).IsZero() {
|
||||
continue
|
||||
}
|
||||
if i == 0 && len(copts.netMode.Value()) == 1 && reflect.ValueOf(*ep).IsZero() {
|
||||
continue
|
||||
}
|
||||
endpoints[n.Target] = ep
|
||||
}
|
||||
@@ -844,14 +814,12 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
||||
return invalidParameter(errors.New("conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses"))
|
||||
}
|
||||
if copts.aliases.Len() > 0 {
|
||||
n.Aliases = make([]string, copts.aliases.Len())
|
||||
copy(n.Aliases, copts.aliases.GetSlice())
|
||||
n.Aliases = slices.Clone(copts.aliases.GetSlice())
|
||||
}
|
||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||
// for the default bridge it defines a legacy-link.
|
||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
||||
n.Links = make([]string, copts.links.Len())
|
||||
copy(n.Links, copts.links.GetSlice())
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) && copts.links.Len() > 0 {
|
||||
n.Links = slices.Clone(copts.links.GetSlice())
|
||||
}
|
||||
if copts.ipv4Address != nil {
|
||||
if ipv4, ok := netip.AddrFromSlice(copts.ipv4Address.To4()); ok {
|
||||
@@ -876,7 +844,7 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
||||
if strings.TrimSpace(ep.Target) == "" {
|
||||
return nil, errors.New("no name set for network")
|
||||
}
|
||||
if !container.NetworkMode(ep.Target).IsUserDefined() {
|
||||
if !isUserDefinedNetwork(container.NetworkMode(ep.Target)) {
|
||||
if len(ep.Aliases) > 0 {
|
||||
return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
|
||||
}
|
||||
@@ -890,7 +858,6 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
||||
}
|
||||
epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
|
||||
if len(ep.DriverOpts) > 0 {
|
||||
epConfig.DriverOpts = make(map[string]string)
|
||||
epConfig.DriverOpts = ep.DriverOpts
|
||||
}
|
||||
if len(ep.Links) > 0 {
|
||||
@@ -934,11 +901,10 @@ func convertToStandardNotation(ports []string) ([]string, error) {
|
||||
}
|
||||
|
||||
func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
|
||||
loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts)
|
||||
if loggingDriver == "none" && len(loggingOpts) > 0 {
|
||||
return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver)
|
||||
}
|
||||
return loggingOptsMap, nil
|
||||
return opts.ConvertKVStringsToMap(loggingOpts), nil
|
||||
}
|
||||
|
||||
// takes a local seccomp daemon, reads the file contents for sending to the daemon
|
||||
|
||||
+149
-292
@@ -2,7 +2,7 @@
|
||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
|
||||
// * appended with license information
|
||||
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
|
||||
//
|
||||
@@ -69,7 +69,6 @@ func parseRun(args []string) (*container.Config, *container.HostConfig, *network
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// TODO(dnephin): fix tests to accept ContainerConfig; see https://github.com/moby/moby/pull/31621
|
||||
containerCfg, err := parse(flags, copts, runtime.GOOS)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
@@ -81,13 +80,12 @@ func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
|
||||
flags := pflag.NewFlagSet("run", pflag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
flags.Usage = nil
|
||||
copts := addFlags(flags)
|
||||
return flags, copts
|
||||
return flags, addFlags(flags)
|
||||
}
|
||||
|
||||
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
|
||||
t.Helper()
|
||||
config, hostConfig, nwConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash"))
|
||||
config, hostConfig, nwConfig, err := parseRun(strings.Split(args, " "))
|
||||
require.NoError(t, err)
|
||||
return config, hostConfig, nwConfig
|
||||
}
|
||||
@@ -179,53 +177,21 @@ func TestParseRunUmask(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseRunAttach(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected container.Config
|
||||
for _, tc := range []struct {
|
||||
input string
|
||||
stdin, stdout, stderr bool
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
expected: container.Config{
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-i",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin -a stdout",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin -a stdout -a stderr",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
{"", false, true, true},
|
||||
{"-i", true, true, true},
|
||||
{"-a stdin", true, false, false},
|
||||
{"-a stdin -a stdout", true, true, false},
|
||||
{"-a stdin -a stdout -a stderr", true, true, true},
|
||||
} {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
config, _, _ := mustParse(t, tc.input)
|
||||
require.Equal(t, tc.expected.AttachStdin, config.AttachStdin)
|
||||
require.Equal(t, tc.expected.AttachStdout, config.AttachStdout)
|
||||
require.Equal(t, tc.expected.AttachStderr, config.AttachStderr)
|
||||
require.Equal(t, tc.stdin, config.AttachStdin)
|
||||
require.Equal(t, tc.stdout, config.AttachStdout)
|
||||
require.Equal(t, tc.stderr, config.AttachStderr)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -276,104 +242,47 @@ func TestParseRunWithInvalidArgs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithVolumes(t *testing.T) { //nolint:gocyclo // verbatim copy from docker/cli tests
|
||||
// A single volume
|
||||
arr, tryit := setupPlatformVolume([]string{`/tmp`}, []string{`c:\tmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %q is missing from volumes. Received %v", tryit, config.Volumes)
|
||||
}
|
||||
|
||||
// Two volumes
|
||||
arr, tryit = setupPlatformVolume([]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[0], config.Volumes)
|
||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[1], config.Volumes)
|
||||
}
|
||||
|
||||
// A single bind mount
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || hostConfig.Binds[0] != arr[0] {
|
||||
t.Fatalf("Error parsing volume flags, %q should mount-bind the path before the colon into the path after the colon. Received %v %v", arr[0], hostConfig.Binds, config.Volumes)
|
||||
}
|
||||
|
||||
// Two bind mounts.
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
}
|
||||
|
||||
// Two bind mounts, first read-only, second read-write.
|
||||
// TODO Windows: The Windows version uses read-write as that's the only mode it supports. Can change this post TP4
|
||||
arr, tryit = setupPlatformVolume(
|
||||
[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`},
|
||||
[]string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
}
|
||||
|
||||
// Similar to previous test but with alternate modes which are only supported by Linux
|
||||
if runtime.GOOS != "windows" {
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, []string{})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
func TestParseWithVolumes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
unix, windows []string
|
||||
binds int
|
||||
}{
|
||||
{[]string{`/tmp`}, []string{`c:\tmp`}, 0},
|
||||
{[]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`}, 0},
|
||||
{[]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`}, 1},
|
||||
{[]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`}, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`}, []string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`}, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, nil, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, nil, 2},
|
||||
{[]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`}, 1},
|
||||
{nil, []string{os.Getenv("SystemDrive") + `\:d:`}, 1},
|
||||
} {
|
||||
volumes := tc.unix
|
||||
if runtime.GOOS == "windows" {
|
||||
volumes = tc.windows
|
||||
}
|
||||
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, []string{})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
if len(volumes) == 0 {
|
||||
continue
|
||||
}
|
||||
t.Run(strings.Join(volumes, ","), func(t *testing.T) {
|
||||
var args []string
|
||||
for _, volume := range volumes {
|
||||
args = append(args, "-v", volume)
|
||||
}
|
||||
config, hostConfig, _, err := parseRun(args)
|
||||
require.NoError(t, err)
|
||||
if tc.binds == 0 {
|
||||
assert.Nil(t, hostConfig.Binds)
|
||||
} else {
|
||||
assert.ElementsMatch(t, volumes[:tc.binds], hostConfig.Binds)
|
||||
}
|
||||
expectedVolumes := map[string]struct{}{}
|
||||
for _, volume := range volumes[tc.binds:] {
|
||||
expectedVolumes[volume] = struct{}{}
|
||||
}
|
||||
assert.Equal(t, expectedVolumes, config.Volumes)
|
||||
})
|
||||
}
|
||||
|
||||
// One bind mount and one volume
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] {
|
||||
t.Fatalf("Error parsing volume flags, %s and %s should only one and only one bind mount %s. Received %s", arr[0], arr[1], arr[0], hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags %s and %s. %s is missing from volumes. Received %v", arr[0], arr[1], arr[1], config.Volumes)
|
||||
}
|
||||
|
||||
// Root to non-c: drive letter (Windows specific)
|
||||
if runtime.GOOS == "windows" {
|
||||
arr, tryit = setupPlatformVolume([]string{}, []string{os.Getenv("SystemDrive") + `\:d:`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] || len(config.Volumes) != 0 {
|
||||
t.Fatalf("Error parsing %s. Should have a single bind mount and no volumes", arr[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setupPlatformVolume takes two arrays of volume specs - a Unix style
|
||||
// spec and a Windows style spec. Depending on the platform being unit tested,
|
||||
// it returns one of them, along with a volume string that would be passed
|
||||
// on the docker CLI (e.g. -v /bar -v /foo).
|
||||
func setupPlatformVolume(u, w []string) ([]string, string) {
|
||||
var a []string
|
||||
if runtime.GOOS == "windows" {
|
||||
a = w
|
||||
} else {
|
||||
a = u
|
||||
}
|
||||
s := ""
|
||||
for _, v := range a {
|
||||
s = s + "-v " + v + " "
|
||||
}
|
||||
return a, s
|
||||
}
|
||||
|
||||
// check if (a == c && b == d) || (a == d && b == c)
|
||||
// because maps are randomized
|
||||
func compareRandomizedStrings(a, b, c, d string) error {
|
||||
if a == c && b == d {
|
||||
return nil
|
||||
}
|
||||
if a == d && b == c {
|
||||
return nil
|
||||
}
|
||||
return errors.New("strings don't match")
|
||||
}
|
||||
|
||||
// Simple parse with MacAddress validation
|
||||
@@ -414,46 +323,29 @@ func TestParseWithMemorySwap(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseHostname(t *testing.T) {
|
||||
validHostnames := map[string]string{
|
||||
"hostname": "hostname",
|
||||
"host-name": "host-name",
|
||||
"hostname123": "hostname123",
|
||||
"123hostname": "123hostname",
|
||||
"hostname-of-63-bytes-long-should-be-valid-and-without-any-error": "hostname-of-63-bytes-long-should-be-valid-and-without-any-error",
|
||||
}
|
||||
hostnameWithDomain := "--hostname=hostname.domainname"
|
||||
hostnameWithDomainTld := "--hostname=hostname.domainname.tld"
|
||||
for hostname, expectedHostname := range validHostnames {
|
||||
if config, _, _ := mustParse(t, "--hostname="+hostname); config.Hostname != expectedHostname {
|
||||
t.Fatalf("Expected the config to have 'hostname' as %q, got %q", expectedHostname, config.Hostname)
|
||||
}
|
||||
}
|
||||
if config, _, _ := mustParse(t, hostnameWithDomain); config.Hostname != "hostname.domainname" || config.Domainname != "" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname, got %q", config.Hostname)
|
||||
}
|
||||
if config, _, _ := mustParse(t, hostnameWithDomainTld); config.Hostname != "hostname.domainname.tld" || config.Domainname != "" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname.tld, got %q", config.Hostname)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHostnameDomainname(t *testing.T) {
|
||||
validDomainnames := map[string]string{
|
||||
"domainname": "domainname",
|
||||
"domain-name": "domain-name",
|
||||
"domainname123": "domainname123",
|
||||
"123domainname": "123domainname",
|
||||
"domainname-63-bytes-long-should-be-valid-and-without-any-errors": "domainname-63-bytes-long-should-be-valid-and-without-any-errors",
|
||||
}
|
||||
for domainname, expectedDomainname := range validDomainnames {
|
||||
if config, _, _ := mustParse(t, "--domainname="+domainname); config.Domainname != expectedDomainname {
|
||||
t.Fatalf("Expected the config to have 'domainname' as %q, got %q", expectedDomainname, config.Domainname)
|
||||
}
|
||||
}
|
||||
if config, _, _ := mustParse(t, "--hostname=some.prefix --domainname=domainname"); config.Hostname != "some.prefix" || config.Domainname != "domainname" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as 'some.prefix' and 'domainname' as 'domainname', got %q and %q", config.Hostname, config.Domainname)
|
||||
}
|
||||
if config, _, _ := mustParse(t, "--hostname=another-prefix --domainname=domainname.tld"); config.Hostname != "another-prefix" || config.Domainname != "domainname.tld" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as 'another-prefix' and 'domainname' as 'domainname.tld', got %q and %q", config.Hostname, config.Domainname)
|
||||
for _, tc := range []struct {
|
||||
input, hostname, domainname string
|
||||
}{
|
||||
{"--hostname=hostname", "hostname", ""},
|
||||
{"--hostname=host-name", "host-name", ""},
|
||||
{"--hostname=hostname123", "hostname123", ""},
|
||||
{"--hostname=123hostname", "123hostname", ""},
|
||||
{"--hostname=hostname-of-63-bytes-long-should-be-valid-and-without-any-error", "hostname-of-63-bytes-long-should-be-valid-and-without-any-error", ""},
|
||||
{"--hostname=hostname.domainname", "hostname.domainname", ""},
|
||||
{"--hostname=hostname.domainname.tld", "hostname.domainname.tld", ""},
|
||||
{"--domainname=domainname", "", "domainname"},
|
||||
{"--domainname=domain-name", "", "domain-name"},
|
||||
{"--domainname=domainname123", "", "domainname123"},
|
||||
{"--domainname=123domainname", "", "123domainname"},
|
||||
{"--domainname=domainname-63-bytes-long-should-be-valid-and-without-any-errors", "", "domainname-63-bytes-long-should-be-valid-and-without-any-errors"},
|
||||
{"--hostname=some.prefix --domainname=domainname", "some.prefix", "domainname"},
|
||||
{"--hostname=another-prefix --domainname=domainname.tld", "another-prefix", "domainname.tld"},
|
||||
} {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
config, _, _ := mustParse(t, tc.input)
|
||||
require.Equal(t, tc.hostname, config.Hostname)
|
||||
require.Equal(t, tc.domainname, config.Domainname)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,9 +389,15 @@ func TestParseWithExpose(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("merge with published", func(t *testing.T) {
|
||||
// Merge with actual published port
|
||||
config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"})
|
||||
config, hostConfig, _, err := parseRun([]string{"--publish=80", "--publish=127.0.0.1:8080:80", "--publish=[::1]:8081:80", "--expose=80-81/tcp"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, networktypes.PortMap{
|
||||
networktypes.MustParsePort("80/tcp"): {
|
||||
{},
|
||||
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "8080"},
|
||||
{HostIP: netip.MustParseAddr("::1"), HostPort: "8081"},
|
||||
},
|
||||
}, hostConfig.PortBindings)
|
||||
assert.Len(t, config.ExposedPorts, 2)
|
||||
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")}
|
||||
for _, port := range ports {
|
||||
@@ -984,41 +882,29 @@ func TestParseRestartPolicyAutoRemove(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseHealth(t *testing.T) {
|
||||
checkOk := func(args ...string) *container.HealthConfig {
|
||||
config, _, _, err := parseRun(args)
|
||||
if err != nil {
|
||||
t.Fatalf("%#v: %v", args, err)
|
||||
}
|
||||
return config.Healthcheck
|
||||
}
|
||||
checkError := func(expected string, args ...string) {
|
||||
config, _, _, err := parseRun(args)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error, but got %#v", config)
|
||||
}
|
||||
if err.Error() != expected {
|
||||
t.Fatalf("Expected %#v, got %#v", expected, err)
|
||||
}
|
||||
}
|
||||
health := checkOk("--no-healthcheck", "img", "cmd")
|
||||
if health == nil || len(health.Test) != 1 || health.Test[0] != "NONE" {
|
||||
t.Fatalf("--no-healthcheck failed: %#v", health)
|
||||
}
|
||||
|
||||
health = checkOk("--health-cmd=/check.sh -q", "img", "cmd")
|
||||
if len(health.Test) != 2 || health.Test[0] != "CMD-SHELL" || health.Test[1] != "/check.sh -q" {
|
||||
t.Fatalf("--health-cmd: got %#v", health.Test)
|
||||
}
|
||||
if health.Timeout != 0 {
|
||||
t.Fatalf("--health-cmd: timeout = %s", health.Timeout)
|
||||
}
|
||||
|
||||
checkError("--no-healthcheck conflicts with --health-* options",
|
||||
"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
|
||||
|
||||
health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd")
|
||||
if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second || health.StartInterval != 1*time.Second {
|
||||
t.Fatalf("--health-*: got %#v", health)
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
expected *container.HealthConfig
|
||||
wantErr string
|
||||
}{
|
||||
{[]string{"--no-healthcheck", "img", "cmd"}, &container.HealthConfig{Test: []string{"NONE"}}, ""},
|
||||
{[]string{"--health-cmd=/check.sh -q", "img", "cmd"}, &container.HealthConfig{Test: []string{"CMD-SHELL", "/check.sh -q"}}, ""},
|
||||
{[]string{"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd"}, nil, "--no-healthcheck conflicts with --health-* options"},
|
||||
{
|
||||
[]string{"--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd"},
|
||||
&container.HealthConfig{Timeout: 2 * time.Second, Retries: 3, Interval: 4500 * time.Millisecond, StartPeriod: 5 * time.Second, StartInterval: time.Second},
|
||||
"",
|
||||
},
|
||||
} {
|
||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||
config, _, _, err := parseRun(tc.args)
|
||||
if tc.wantErr != "" {
|
||||
require.EqualError(t, err, tc.wantErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, config.Healthcheck)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1037,82 +923,53 @@ func TestParseLoggingOpts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
||||
expErr := "--env-file: open nonexistent: no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expErr = "--env-file: open nonexistent: The system cannot find the file specified."
|
||||
}
|
||||
// env ko
|
||||
if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
||||
}
|
||||
// env ok
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Env) != 1 || config.Env[0] != "ENV1=value1" {
|
||||
t.Fatalf("Expected a config with [ENV1=value1], got %v", config.Env)
|
||||
}
|
||||
config, _, _, err = parseRun([]string{"--env-file=testdata/valid.env", "--env=ENV2=value2", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Env) != 2 || config.Env[0] != "ENV1=value1" || config.Env[1] != "ENV2=value2" {
|
||||
t.Fatalf("Expected a config with [ENV1=value1 ENV2=value2], got %v", config.Env)
|
||||
func TestParseFileVariables(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
flag, file, key string
|
||||
}{
|
||||
{"env", "valid.env", "ENV"},
|
||||
{"label", "valid.label", "LABEL"},
|
||||
} {
|
||||
t.Run(tc.flag, func(t *testing.T) {
|
||||
expectedError := "no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expectedError = "The system cannot find the file specified."
|
||||
}
|
||||
_, _, _, err := parseRun([]string{"--" + tc.flag + "-file=nonexistent"})
|
||||
require.EqualError(t, err, "--"+tc.flag+"-file: open nonexistent: "+expectedError)
|
||||
for _, extra := range []bool{false, true} {
|
||||
args := []string{"--" + tc.flag + "-file=testdata/" + tc.file}
|
||||
expected := []string{tc.key + "1=value1"}
|
||||
if extra {
|
||||
args = append(args, "--"+tc.flag+"="+tc.key+"2=value2")
|
||||
expected = append(expected, tc.key+"2=value2")
|
||||
}
|
||||
config, _, _, err := parseRun(args)
|
||||
require.NoError(t, err)
|
||||
if tc.flag == "env" {
|
||||
assert.Equal(t, expected, config.Env)
|
||||
} else {
|
||||
assert.Len(t, config.Labels, len(expected))
|
||||
for _, label := range expected {
|
||||
key, value, _ := strings.Cut(label, "=")
|
||||
assert.Equal(t, value, config.Labels[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
|
||||
// UTF8 with BOM
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := []string{"FOO=BAR", "HELLO=" + string([]byte{0xe6, 0x82, 0xa8, 0xe5, 0xa5, 0xbd}), "BAR=FOO"}
|
||||
if len(config.Env) != len(env) {
|
||||
t.Fatalf("Expected a config with %d env variables, got %v: %v", len(env), len(config.Env), config.Env)
|
||||
}
|
||||
for i, v := range env {
|
||||
if config.Env[i] != v {
|
||||
t.Fatalf("Expected a config with [%s], got %v", v, []byte(config.Env[i]))
|
||||
}
|
||||
}
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"FOO=BAR", "HELLO=您好", "BAR=FOO"}, config.Env)
|
||||
|
||||
// UTF16 with BOM
|
||||
e := "invalid utf8 bytes at line"
|
||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
||||
}
|
||||
// UTF16BE with BOM
|
||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16be.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
||||
expErr := "--label-file: open nonexistent: no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expErr = "--label-file: open nonexistent: The system cannot find the file specified."
|
||||
}
|
||||
// label ko
|
||||
if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
||||
}
|
||||
// label ok
|
||||
config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Labels) != 1 || config.Labels["LABEL1"] != "value1" {
|
||||
t.Fatalf("Expected a config with [LABEL1:value1], got %v", config.Labels)
|
||||
}
|
||||
config, _, _, err = parseRun([]string{"--label-file=testdata/valid.label", "--label=LABEL2=value2", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Labels) != 2 || config.Labels["LABEL1"] != "value1" || config.Labels["LABEL2"] != "value2" {
|
||||
t.Fatalf("Expected a config with [LABEL1:value1 LABEL2:value2], got %v", config.Labels)
|
||||
for _, file := range []string{"utf16.env", "utf16be.env"} {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
_, _, _, err := parseRun([]string{"--env-file=testdata/" + file})
|
||||
require.ErrorContains(t, err, "invalid utf8 bytes at line")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -58,18 +58,14 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
if _, err := cli.ImageInspect(ctx, imageName); cerrdefs.IsNotFound(err) {
|
||||
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
||||
Force: force,
|
||||
PruneChildren: pruneChildren,
|
||||
}); cerrdefs.IsNotFound(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
||||
Force: force,
|
||||
PruneChildren: pruneChildren,
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -5,14 +5,17 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -21,49 +24,55 @@ func init() {
|
||||
logging.SetLevel(logging.LevelDebug)
|
||||
}
|
||||
|
||||
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
|
||||
// network or emulation since there is nothing to run) and returns its tag, removing it after
|
||||
// the test.
|
||||
func buildScratchImage(t *testing.T, platform string) string {
|
||||
func loadScratchImage(t *testing.T, platform string) string {
|
||||
t.Helper()
|
||||
tag := fmt.Sprintf("act-test-exists-%s:latest", strings.TrimPrefix(platform, "linux/"))
|
||||
cmd := exec.Command("docker", "build", "--platform", platform, "-t", tag, "-")
|
||||
cmd.Stdin = strings.NewReader("FROM scratch\nLABEL act-test=1\n")
|
||||
// Force BuildKit: it records the requested architecture in the image config for a
|
||||
// FROM-scratch build, whereas the classic builder ignores --platform and tags it with the
|
||||
// host arch, which would break the per-platform existence assertions below.
|
||||
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, string(out))
|
||||
t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() })
|
||||
tag := fmt.Sprintf("act-test-exists-%s-%d:latest", strings.TrimPrefix(platform, "linux/"), time.Now().UnixNano())
|
||||
var archive bytes.Buffer
|
||||
require.NoError(t, writeFilesTar(t.Context(), &archive, 0, 0,
|
||||
&FileEntry{Name: "config.json", Mode: 0o600, Body: fmt.Sprintf(`{"architecture":%q,"os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`, strings.TrimPrefix(platform, "linux/"))},
|
||||
&FileEntry{Name: "manifest.json", Mode: 0o600, Body: fmt.Sprintf(`[{"Config":"config.json","RepoTags":[%q],"Layers":[]}]`, tag)},
|
||||
))
|
||||
cli, err := GetDockerClient(t.Context())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = cli.Close() })
|
||||
response, err := cli.ImageLoad(t.Context(), &archive)
|
||||
require.NoError(t, err)
|
||||
defer response.Close()
|
||||
require.NoError(t, logDockerResponse(common.Logger(t.Context()), response, false))
|
||||
t.Cleanup(func() {
|
||||
_, _ = cli.ImageRemove(context.WithoutCancel(t.Context()), tag, mobyclient.ImageRemoveOptions{Force: true})
|
||||
})
|
||||
return tag
|
||||
}
|
||||
|
||||
func TestImageExistsLocally(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireDocker(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// a non-existent image is reported absent
|
||||
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.False(t, missing)
|
||||
|
||||
// Build tiny images for two architectures locally so per-platform existence can be checked
|
||||
// offline (formerly pulled node:24-bookworm-slim for amd64 and arm64 over the network).
|
||||
amd64Ref := buildScratchImage(t, "linux/amd64")
|
||||
arm64Ref := buildScratchImage(t, "linux/arm64")
|
||||
amd64Ref := loadScratchImage(t, "linux/amd64")
|
||||
arm64Ref := loadScratchImage(t, "linux/arm64")
|
||||
|
||||
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.True(t, amd64Exists)
|
||||
|
||||
// a non-host architecture image is detected for its own architecture
|
||||
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.True(t, arm64Exists)
|
||||
|
||||
// a present image is reported absent for a different platform
|
||||
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.False(t, wrongPlatform)
|
||||
|
||||
removed, err := RemoveImage(ctx, amd64Ref, false, false)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, removed)
|
||||
removed, err = RemoveImage(ctx, amd64Ref, false, false)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, removed)
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -33,18 +33,10 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
||||
defer dockerResponse.Close()
|
||||
|
||||
scanner := bufio.NewScanner(dockerResponse)
|
||||
msg := dockerMessage{}
|
||||
|
||||
scanner.Buffer(nil, 64*1024*1024) // a build echoes long instructions in one record
|
||||
for scanner.Scan() {
|
||||
line := scanner.Bytes()
|
||||
|
||||
msg.ID = ""
|
||||
msg.Stream = ""
|
||||
msg.Error = ""
|
||||
msg.ErrorDetail.Message = ""
|
||||
msg.Status = ""
|
||||
msg.Progress = ""
|
||||
|
||||
var msg dockerMessage
|
||||
if err := json.Unmarshal(line, &msg); err != nil {
|
||||
writeLog(logger, false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
|
||||
continue
|
||||
@@ -74,7 +66,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -85,14 +85,14 @@ networks:
|
||||
}
|
||||
|
||||
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return createDockerNetwork(ctx, cli, name, opts)
|
||||
}
|
||||
}).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||
@@ -157,7 +157,7 @@ func isAddressPoolExhausted(err error) bool {
|
||||
}
|
||||
|
||||
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -167,7 +167,7 @@ func NewDockerNetworkConnectExecutor(networkName, container string) common.Execu
|
||||
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||
@@ -182,14 +182,14 @@ func connectNetwork(ctx context.Context, cli client.APIClient, networkName, cont
|
||||
}
|
||||
|
||||
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return disconnectNetwork(ctx, cli, networkName, container)
|
||||
}
|
||||
}).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||
@@ -201,14 +201,14 @@ func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, c
|
||||
}
|
||||
|
||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return removeNetwork(ctx, cli, name)
|
||||
}
|
||||
}).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2025 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ package container
|
||||
import (
|
||||
"testing"
|
||||
|
||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -17,32 +18,17 @@ func TestParsePlatform(t *testing.T) {
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
|
||||
t.Run("os/arch", func(t *testing.T) {
|
||||
got, err := parsePlatform("linux/amd64")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "amd64", got.Architecture)
|
||||
assert.Empty(t, got.Variant)
|
||||
})
|
||||
|
||||
t.Run("os/arch/variant", func(t *testing.T) {
|
||||
got, err := parsePlatform("linux/arm/v7")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "arm", got.Architecture)
|
||||
assert.Equal(t, "v7", got.Variant)
|
||||
})
|
||||
|
||||
t.Run("input is lowercased", func(t *testing.T) {
|
||||
got, err := parsePlatform("Linux/AMD64/V8")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "amd64", got.Architecture)
|
||||
assert.Equal(t, "v8", got.Variant)
|
||||
})
|
||||
for input, want := range map[string]specs.Platform{
|
||||
"linux/amd64": {OS: "linux", Architecture: "amd64"},
|
||||
"linux/arm/v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
|
||||
"Linux/AMD64/V8": {OS: "linux", Architecture: "amd64", Variant: "v8"},
|
||||
} {
|
||||
t.Run(input, func(t *testing.T) {
|
||||
got, err := parsePlatform(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &want, got)
|
||||
})
|
||||
}
|
||||
|
||||
for _, bad := range []string{
|
||||
"amd64",
|
||||
|
||||
+115
-22
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -40,7 +40,9 @@ const (
|
||||
jobLabel = "com.gitea.runner.job."
|
||||
maxCreateBody = 8 << 20
|
||||
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
dockerProxyProbeWorkerTimeout = time.Minute
|
||||
dockerProxyProbeTarget = "/gitea-runner-probe"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -48,13 +50,27 @@ var (
|
||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||
)
|
||||
|
||||
var dockerProxyProbes sync.Map
|
||||
|
||||
type dockerProxyDirectory struct {
|
||||
dir, daemonDir string
|
||||
socket os.FileInfo
|
||||
}
|
||||
|
||||
type dockerProxyProbe struct {
|
||||
directory dockerProxyDirectory
|
||||
done chan struct{}
|
||||
seen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
||||
return nil
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
cli, err := GetDockerClient(probeCtx)
|
||||
cli, err := client.New(client.FromEnv) // negotiates on first request, so a cached probe costs no round trip
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -63,37 +79,78 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if info, err := os.Stat(daemonSocket); err != nil || info.Mode()&os.ModeSocket == 0 {
|
||||
socket, err := os.Stat(daemonSocket)
|
||||
if err != nil || socket.Mode()&os.ModeSocket == 0 {
|
||||
return nil
|
||||
}
|
||||
dir, daemonDir := runnerContainerWorkdir(probeCtx, cli)
|
||||
seen := daemonDir != ""
|
||||
if !seen {
|
||||
if dir, err = filepath.Abs(os.TempDir()); err == nil {
|
||||
daemonDir = dir
|
||||
seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
|
||||
}
|
||||
}
|
||||
workdir, _ := os.Getwd()
|
||||
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||
probe, err := probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
if !seen {
|
||||
if !probe.seen {
|
||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
proxy, err := StartDockerProxy(daemonSocket, dir, job)
|
||||
proxy, err := StartDockerProxy(daemonSocket, probe.directory.dir, job)
|
||||
if err != nil {
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
|
||||
proxy.Socket = probe.directory.daemonDir + strings.TrimPrefix(proxy.Socket, probe.directory.dir)
|
||||
return proxy
|
||||
}
|
||||
|
||||
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (*dockerProxyProbe, error) {
|
||||
for {
|
||||
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
|
||||
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
|
||||
if loaded {
|
||||
probe, _ = actual.(*dockerProxyProbe)
|
||||
if !os.SameFile(probe.directory.socket, socket) { // daemon replacement invalidates cached paths
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
go func() { // outlives callers' deadlines so a slow daemon still gets its result cached
|
||||
defer cli.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeWorkerTimeout)
|
||||
defer cancel()
|
||||
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||
probe.seen = probe.directory.daemonDir != ""
|
||||
if !probe.seen {
|
||||
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
|
||||
probe.directory.daemonDir = probe.directory.dir
|
||||
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||
}
|
||||
}
|
||||
if probe.seen {
|
||||
dockerProxyProbes.LoadOrStore(key, probe)
|
||||
} else {
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
}
|
||||
close(probe.done)
|
||||
removeStaleProbeContainers(cli)
|
||||
}()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return &dockerProxyProbe{}, ctx.Err()
|
||||
case <-probe.done:
|
||||
}
|
||||
if ctx.Err() == nil && errors.Is(probe.err, context.DeadlineExceeded) {
|
||||
continue
|
||||
}
|
||||
return probe, probe.err
|
||||
}
|
||||
}
|
||||
|
||||
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
||||
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
||||
workdir, err := os.Getwd()
|
||||
@@ -144,19 +201,25 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
if len(images.Items) == 0 {
|
||||
return false, errors.New("no image available for the docker proxy probe")
|
||||
}
|
||||
created, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
}},
|
||||
})
|
||||
var created client.ContainerCreateResult
|
||||
for _, image := range images.Items {
|
||||
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: dockerProxyProbeTarget, ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
}},
|
||||
})
|
||||
if !cerrdefs.IsNotFound(err) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if cerrdefs.IsInvalidArgument(err) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, dockerProxyProbeTarget)
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||
@@ -165,6 +228,36 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
return seen, err
|
||||
}
|
||||
|
||||
// removeStaleProbeContainers removes probe containers left by abandoned creates once any probe in flight when listed has ended.
|
||||
func removeStaleProbeContainers(cli client.APIClient) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
listed, err := cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
Filters: make(client.Filters).Add("status", string(container.StateCreated)).Add("volume", dockerProxyProbeTarget),
|
||||
})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("listing stale docker proxy probe containers failed: %v", err)
|
||||
return
|
||||
}
|
||||
listed.Items = slices.DeleteFunc(listed.Items, func(stale container.Summary) bool {
|
||||
return stale.State != container.StateCreated || !slices.ContainsFunc(stale.Mounts, func(mountPoint container.MountPoint) bool {
|
||||
return mountPoint.Type == mount.TypeBind && !mountPoint.RW && mountPoint.Destination == dockerProxyProbeTarget
|
||||
})
|
||||
})
|
||||
if len(listed.Items) == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(dockerProxyProbeWorkerTimeout + dockerProxyProbeTimeout)
|
||||
for _, stale := range listed.Items {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
if err := (&containerReference{cli: cli, id: stale.ID}).remove()(ctx); err != nil {
|
||||
common.Logger(ctx).Warnf("removing stale docker proxy probe container failed: %v", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
info, err := os.Stat(daemonSocket)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
@@ -68,6 +70,7 @@ func TestDockerProxy(t *testing.T) {
|
||||
bodies := make(chan []byte, 8)
|
||||
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
|
||||
streamsDone := make(chan struct{}, 2)
|
||||
var probes atomic.Int32
|
||||
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
|
||||
listener, err := net.Listen("unix", daemonSocket)
|
||||
require.NoError(t, err)
|
||||
@@ -75,12 +78,30 @@ func TestDockerProxy(t *testing.T) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/create"):
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if bytes.Contains(body, []byte(`"Image":"probe-image"`)) {
|
||||
_, _ = w.Write([]byte(`{"Id":"probe"}`))
|
||||
return
|
||||
}
|
||||
if r.URL.RawQuery == "wait" {
|
||||
close(createStarted)
|
||||
<-releaseCreate
|
||||
}
|
||||
bodies <- body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.HasSuffix(r.URL.Path, "/images/json"):
|
||||
if probes.Add(1) == 1 {
|
||||
http.Error(w, "probe unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||
http.NotFound(w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||
w.Header().Set("X-Docker-Container-Path-Stat", "e30=")
|
||||
case r.Method == http.MethodDelete:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
|
||||
if r.URL.Path == "/exec/error/start" {
|
||||
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
|
||||
@@ -211,6 +232,30 @@ func TestDockerProxy(t *testing.T) {
|
||||
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
|
||||
})
|
||||
|
||||
t.Run("discovery caches only successes", func(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("automatic proxy discovery requires Linux")
|
||||
}
|
||||
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||
tempDir := shortTempDir(t)
|
||||
t.Setenv("TMPDIR", tempDir)
|
||||
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||
for _, job := range []string{"first", "second"} {
|
||||
proxy := NewDockerProxy(t.Context(), job)
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
}
|
||||
assert.EqualValues(t, 2, probes.Load())
|
||||
workdir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
|
||||
cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
|
||||
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
assert.EqualValues(t, 3, probes.Load())
|
||||
})
|
||||
|
||||
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
|
||||
for _, stream := range []struct{ path, upgrade string }{
|
||||
{path: "/v1.47/exec/abc/start"},
|
||||
@@ -328,6 +373,9 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
||||
|
||||
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
||||
if os.Geteuid() != 0 && errors.Is(err, os.ErrPermission) {
|
||||
t.Skipf("non-root runners that cannot take the daemon socket's ownership get it directly: %v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
||||
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
||||
@@ -429,10 +477,27 @@ func BenchmarkDockerProxy(b *testing.B) {
|
||||
|
||||
type probeClient struct {
|
||||
mobyclient.APIClient
|
||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
create func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
containers []container.Summary
|
||||
}
|
||||
|
||||
func (c *probeClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
return mobyclient.SystemInfoResult{}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
|
||||
return mobyclient.ContainerListResult{Items: c.containers}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||
return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||
@@ -440,11 +505,11 @@ func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.Conta
|
||||
}
|
||||
|
||||
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "probe-image"}}}, nil
|
||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(opts)
|
||||
func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(ctx, opts)
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
@@ -455,9 +520,193 @@ func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobycl
|
||||
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
||||
}
|
||||
|
||||
func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
type result struct {
|
||||
seen bool
|
||||
err error
|
||||
}
|
||||
for name, createErr := range map[string]error{"unavailable directory": nil, "daemon error": errors.New("probe failed")} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
key := [3]string{"daemon", dir, dir}
|
||||
release := make(chan struct{})
|
||||
var calls, removals atomic.Int32
|
||||
visible := false
|
||||
cli := &probeClient{
|
||||
create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
|
||||
},
|
||||
stat: func(_path string) error {
|
||||
if visible {
|
||||
return nil
|
||||
} else {
|
||||
return cerrdefs.ErrNotFound
|
||||
}
|
||||
},
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removals.Add(1)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
results := make(chan result, 8)
|
||||
for range cap(results) {
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
results <- result{probe.seen, err}
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
replacement, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
replaced := make(chan result, 1)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||
replaced <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
close(release)
|
||||
for range cap(results) {
|
||||
result := <-results
|
||||
require.ErrorIs(t, result.err, createErr)
|
||||
assert.False(t, result.seen)
|
||||
}
|
||||
replacementResult := <-replaced
|
||||
require.ErrorIs(t, replacementResult.err, createErr)
|
||||
assert.False(t, replacementResult.seen)
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
if createErr == nil {
|
||||
assert.EqualValues(t, 2, removals.Load())
|
||||
} else {
|
||||
assert.Zero(t, removals.Load())
|
||||
}
|
||||
createErr, visible = nil, true
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, probe.seen)
|
||||
assert.Equal(t, dir, probe.directory.dir)
|
||||
assert.Equal(t, dir, probe.directory.daemonDir)
|
||||
assert.EqualValues(t, 3, calls.Load())
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
})
|
||||
})
|
||||
}
|
||||
t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
key := [3]string{"daemon", dir, dir}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
var calls, removals atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removals.Add(1)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
leader <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
follower <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
leaderResult := <-leader
|
||||
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||
assert.False(t, leaderResult.seen)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
late <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
close(release)
|
||||
for _, results := range []chan result{follower, late} {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
assert.True(t, result.seen)
|
||||
}
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
assert.EqualValues(t, 1, removals.Load())
|
||||
_, cached := dockerProxyProbes.Load(key)
|
||||
assert.True(t, cached)
|
||||
})
|
||||
})
|
||||
t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
var calls atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
<-ctx.Done()
|
||||
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, probe.seen)
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveStaleProbeContainers(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
|
||||
writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
|
||||
var removed []string
|
||||
cli := &probeClient{
|
||||
containers: []container.Summary{
|
||||
{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
|
||||
{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
|
||||
{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
|
||||
},
|
||||
remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removed = append(removed, id)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
start := time.Now()
|
||||
removeStaleProbeContainers(cli)
|
||||
assert.Equal(t, []string{"abandoned"}, removed)
|
||||
assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDaemonSeesDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
markers := make(map[string]bool)
|
||||
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
daemonDir string
|
||||
@@ -474,9 +723,14 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
removed := false
|
||||
var attemptedImages []string
|
||||
cli := &probeClient{
|
||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||
attemptedImages = append(attemptedImages, opts.Config.Image)
|
||||
if opts.Config.Image == "removed-image" {
|
||||
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrNotFound
|
||||
}
|
||||
cancel()
|
||||
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
||||
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
||||
@@ -504,8 +758,10 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
}
|
||||
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
||||
require.ErrorIs(t, err, testCase.removeErr)
|
||||
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
|
||||
assert.Equal(t, testCase.seen, seen)
|
||||
assert.Equal(t, testCase.removed, removed)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
@@ -514,6 +770,9 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerContainerWorkdir(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("a runner container is Linux")
|
||||
}
|
||||
workdir := t.TempDir()
|
||||
t.Chdir(workdir)
|
||||
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
// NewDockerPullExecutor function to create a run executor for the container
|
||||
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
logger.Debugf("docker pull %v", input.Image)
|
||||
|
||||
@@ -83,7 +83,7 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||
}
|
||||
}).Retry(3)
|
||||
}
|
||||
|
||||
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
|
||||
@@ -114,7 +114,7 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
|
||||
if err != nil {
|
||||
return imagePullOptions, err
|
||||
}
|
||||
if authConfig.Username == "" && authConfig.Password == "" {
|
||||
if authConfig.Username == "" && authConfig.Password == "" && authConfig.IdentityToken == "" && authConfig.RegistryToken == "" {
|
||||
return imagePullOptions, nil
|
||||
}
|
||||
logger.Info("using DockerAuthConfig authentication for docker pull")
|
||||
|
||||
@@ -6,12 +6,17 @@ package container
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
"github.com/moby/moby/api/pkg/authconfig"
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -64,6 +69,21 @@ func TestGetImagePullOptions(t *testing.T) {
|
||||
})
|
||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths":{"identity.example.com":{"identitytoken":"identity-token"},"registry.example.com":{"registrytoken":"registry-token"}}}`), 0o600))
|
||||
t.Setenv("DOCKER_CONFIG", dir)
|
||||
for host, want := range map[string]registry.AuthConfig{
|
||||
"identity.example.com": {IdentityToken: "identity-token", ServerAddress: "identity.example.com"},
|
||||
"registry.example.com": {RegistryToken: "registry-token", ServerAddress: "registry.example.com"},
|
||||
} {
|
||||
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{Image: host + "/app"})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, options.RegistryAuth, host)
|
||||
got, err := authconfig.Decode(options.RegistryAuth)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, *got)
|
||||
}
|
||||
}
|
||||
|
||||
// A digest-pinned image is immutable, so its local copy is always current.
|
||||
@@ -82,4 +102,8 @@ func TestLogDockerResponseError(t *testing.T) {
|
||||
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
||||
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
||||
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
||||
require.NoError(t, streamErr(`{"stream":"`+strings.Repeat("x", 70000)+`"}`))
|
||||
require.ErrorIs(t, logDockerResponse(logger, io.NopCloser(io.MultiReader(
|
||||
strings.NewReader(`{"status":"Downloading"}`+"\n"), iotest.ErrReader(io.ErrUnexpectedEOF),
|
||||
)), false), io.ErrUnexpectedEOF)
|
||||
}
|
||||
+118
-93
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -50,15 +50,12 @@ const drainGracePeriod = 2 * time.Second
|
||||
|
||||
// NewContainer creates a reference to a container
|
||||
func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||
cr := new(containerReference)
|
||||
cr.input = input
|
||||
// Resolved up front because the image pull runs before the container is created.
|
||||
cf := createFlagsFromOptions(input.allOptions())
|
||||
if cf.platform != "" {
|
||||
cr.input.Platform = cf.platform
|
||||
input.Platform = cf.platform
|
||||
}
|
||||
cr.pullPolicy = cf.pull
|
||||
return cr
|
||||
return &containerReference{input: input, pullPolicy: cf.pull}
|
||||
}
|
||||
|
||||
// supportsContainerImagePlatform reports whether the Docker server API version
|
||||
@@ -208,18 +205,30 @@ func (cr *containerReference) DumpLogs(ctx context.Context) error {
|
||||
if cr.id == "" {
|
||||
return cr.missingContainerError("read its logs")
|
||||
}
|
||||
tty, err := cr.containerTTY(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer logs.Close()
|
||||
return cr.copyOutput(logs)
|
||||
return cr.copyOutput(logs, tty)
|
||||
}
|
||||
|
||||
func (cr *containerReference) containerTTY(ctx context.Context) (bool, error) {
|
||||
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to inspect container: %w", err)
|
||||
}
|
||||
return inspect.Container.Config != nil && inspect.Container.Config.Tty, nil
|
||||
}
|
||||
|
||||
// copyOutput writes a container stream to the writers the container was created with,
|
||||
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
|
||||
func (cr *containerReference) copyOutput(stream io.Reader) error {
|
||||
func (cr *containerReference) copyOutput(stream io.Reader, tty bool) error {
|
||||
outWriter := cr.input.Stdout
|
||||
if outWriter == nil {
|
||||
outWriter = os.Stdout
|
||||
@@ -230,7 +239,7 @@ func (cr *containerReference) copyOutput(stream io.Reader) error {
|
||||
}
|
||||
|
||||
var err error
|
||||
if !cr.input.AllocatePTY || os.Getenv("NORAW") != "" {
|
||||
if !tty || os.Getenv("NORAW") != "" {
|
||||
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
|
||||
} else {
|
||||
_, err = io.Copy(outWriter, stream)
|
||||
@@ -299,7 +308,7 @@ func containerInfoFromInspect(inspect container.InspectResponse) *Info {
|
||||
}
|
||||
|
||||
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun)
|
||||
return parseEnvFile(cr, srcPath, env, false).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
@@ -357,6 +366,7 @@ type containerReference struct {
|
||||
id string
|
||||
input *NewContainerInput
|
||||
pullPolicy string
|
||||
daemonOS string
|
||||
UID int
|
||||
GID int
|
||||
// attachDone is closed by the attach() streaming goroutine once it has
|
||||
@@ -366,13 +376,18 @@ type containerReference struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
}
|
||||
|
||||
func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
func GetDockerClient(ctx context.Context) (client.APIClient, error) {
|
||||
cli, _, err := newDockerClient(ctx)
|
||||
return cli, err
|
||||
}
|
||||
|
||||
func newDockerClient(ctx context.Context) (cli client.APIClient, daemonOS string, err error) {
|
||||
dockerHost := os.Getenv("DOCKER_HOST")
|
||||
|
||||
if strings.HasPrefix(dockerHost, "ssh://") {
|
||||
var args []string
|
||||
if args, err = sshArgs(dockerHost); err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
cli, err = client.New(
|
||||
client.WithHost("http://docker.example.com"), // the dialer below ignores the address
|
||||
@@ -382,29 +397,29 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
cli, err = client.New(client.FromEnv)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
return nil, "", fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
}
|
||||
// Best-effort API version negotiation, matching the old client.NegotiateAPIVersion
|
||||
// behaviour. Ping failures here are non-fatal: the connection is exercised again on
|
||||
// the first real call, and the client falls back to the daemon's default API version.
|
||||
if _, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true}); err != nil {
|
||||
ping, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("docker daemon ping during version negotiation failed, continuing: %v", err)
|
||||
}
|
||||
|
||||
return cli, nil
|
||||
return cli, ping.OSType, nil
|
||||
}
|
||||
|
||||
func GetHostInfo(ctx context.Context) (info system.Info, err error) {
|
||||
var cli client.APIClient
|
||||
cli, err = GetDockerClient(ctx)
|
||||
func GetHostInfo(ctx context.Context) (system.Info, error) {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return info, err
|
||||
return system.Info{}, err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
result, err := cli.Info(ctx, client.InfoOptions{})
|
||||
if err != nil {
|
||||
return info, err
|
||||
return system.Info{}, err
|
||||
}
|
||||
|
||||
return result.Info, nil
|
||||
@@ -432,16 +447,32 @@ func RunnerArch(ctx context.Context) string {
|
||||
return info.Architecture
|
||||
}
|
||||
|
||||
func (cr *containerReference) targetOS(ctx context.Context) string {
|
||||
if cr.daemonOS == "" {
|
||||
cr.daemonOS = "linux"
|
||||
if cr.cli != nil {
|
||||
if result, err := cr.cli.Info(ctx, client.InfoOptions{}); err == nil && result.Info.OSType != "" {
|
||||
cr.daemonOS = result.Info.OSType
|
||||
}
|
||||
}
|
||||
}
|
||||
return cr.daemonOS
|
||||
}
|
||||
|
||||
func (cr *containerReference) markExecutable(ctx context.Context) bool {
|
||||
return hostOS == "windows" && cr.targetOS(ctx) == "linux"
|
||||
}
|
||||
|
||||
func (cr *containerReference) connect() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.cli != nil {
|
||||
return nil
|
||||
}
|
||||
cli, err := GetDockerClient(ctx)
|
||||
cli, daemonOS, err := newDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cr.cli = cli
|
||||
cr.cli, cr.daemonOS = cli, daemonOS
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -451,6 +482,7 @@ func (cr *containerReference) Close() common.Executor {
|
||||
if cr.cli != nil {
|
||||
err := cr.cli.Close()
|
||||
cr.cli = nil
|
||||
cr.daemonOS = ""
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close client: %w", err)
|
||||
}
|
||||
@@ -476,22 +508,23 @@ func (cr *containerReference) find() common.Executor {
|
||||
}
|
||||
cr.id = ""
|
||||
}
|
||||
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
})
|
||||
result, err := cr.cli.ContainerInspect(ctx, cr.input.Name, client.ContainerInspectOptions{})
|
||||
if err == nil && result.Container.Name == "/"+cr.input.Name {
|
||||
cr.id = result.Container.ID
|
||||
return nil
|
||||
} else if cerrdefs.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{All: true}) // inspect also matches ids, and podman fails it with a 500 for a container removed mid-request
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list containers: %w", err)
|
||||
}
|
||||
|
||||
for _, c := range containers.Items {
|
||||
for _, name := range c.Names {
|
||||
if name[1:] == cr.input.Name {
|
||||
cr.id = c.ID
|
||||
return nil
|
||||
}
|
||||
for _, candidate := range containers.Items {
|
||||
if slices.Contains(candidate.Names, "/"+cr.input.Name) {
|
||||
cr.id = candidate.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -502,7 +535,7 @@ func (cr *containerReference) remove() common.Executor {
|
||||
if idOrName == "" && cr.input != nil {
|
||||
idOrName = cr.input.Name
|
||||
}
|
||||
if idOrName == "" {
|
||||
if idOrName == "" || cr.cli == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -558,32 +591,32 @@ func (input *NewContainerInput) allOptions() string {
|
||||
return strings.TrimSpace(input.RunnerOptions + " " + input.WorkflowOptions)
|
||||
}
|
||||
|
||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
|
||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) error {
|
||||
logger := common.Logger(ctx)
|
||||
options := cr.input.allOptions()
|
||||
|
||||
if options == "" {
|
||||
return config, hostConfig, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// For Gitea, checked here because the parse below is what would read those files
|
||||
if err := rejectHostReadingOptions(cr.input.WorkflowOptions); err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
// parse configuration from CLI container.options
|
||||
flags, copts, cf, err := parseContainerOptions(options)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
if err := cf.validate(); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
|
||||
if len(copts.netMode.Value()) == 0 {
|
||||
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
||||
return fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -594,9 +627,10 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
copts.privileged = false
|
||||
}
|
||||
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
serverOS := cr.targetOS(ctx)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
// workflow aliases join the runner's own, deduped so a re-create cannot grow the input
|
||||
for _, endpoint := range containerConfig.NetworkingConfig.EndpointsConfig {
|
||||
@@ -609,9 +643,9 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
|
||||
// For Gitea, forcing --privileged off is not enough, other options reach the host too
|
||||
if !hostConfig.Privileged {
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions, serverOS)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig, trusted)
|
||||
}
|
||||
@@ -623,7 +657,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
|
||||
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
|
||||
|
||||
cr.sanitizeConfig(ctx, nil, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
|
||||
cr.sanitizeConfig(ctx, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
|
||||
overlayVolumes(hostConfig, containerConfig.HostConfig)
|
||||
binds := hostConfig.Binds
|
||||
mounts := hostConfig.Mounts
|
||||
@@ -644,7 +678,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
hostConfig.NetworkMode = networkMode
|
||||
logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
|
||||
|
||||
return config, hostConfig, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
@@ -710,8 +744,7 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
AutoRemove: input.AutoRemove,
|
||||
}
|
||||
|
||||
config, hostConfig, err = cr.mergeContainerConfigs(ctx, config, hostConfig)
|
||||
if err != nil {
|
||||
if err := cr.mergeContainerConfigs(ctx, config, hostConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -720,18 +753,13 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
config, hostConfig = cr.sanitizeConfig(ctx, config, hostConfig)
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
|
||||
var networkingConfig *network.NetworkingConfig
|
||||
n := hostConfig.NetworkMode
|
||||
// IsUserDefined and IsHost are broken on windows
|
||||
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
||||
endpointConfig := &network.EndpointSettings{
|
||||
Aliases: input.NetworkAliases,
|
||||
}
|
||||
if isUserDefinedNetwork(hostConfig.NetworkMode) && len(input.NetworkAliases) > 0 {
|
||||
networkingConfig = &network.NetworkingConfig{
|
||||
EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
input.NetworkMode: endpointConfig,
|
||||
input.NetworkMode: {Aliases: input.NetworkAliases},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -772,15 +800,11 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
||||
|
||||
logger.Debugf("Exec command '%s'", cmd)
|
||||
isTerminal := cr.input.AllocatePTY
|
||||
var wd string
|
||||
if workdir != "" {
|
||||
if strings.HasPrefix(workdir, "/") {
|
||||
wd = workdir
|
||||
} else {
|
||||
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
||||
}
|
||||
} else {
|
||||
wd = cr.input.WorkingDir
|
||||
wd := cr.input.WorkingDir
|
||||
if strings.HasPrefix(workdir, "/") {
|
||||
wd = workdir
|
||||
} else if workdir != "" {
|
||||
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
||||
}
|
||||
logger.Debugf("Working directory '%s'", wd)
|
||||
|
||||
@@ -805,8 +829,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
err = cr.waitForCommand(ctx, resp.HijackedResponse)
|
||||
if err != nil {
|
||||
if err := cr.waitForCommand(ctx, resp.HijackedResponse); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -863,7 +886,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
||||
cmdResponse := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
cmdResponse <- cr.copyOutput(resp.Reader)
|
||||
cmdResponse <- cr.copyOutput(resp.Reader, cr.input.AllocatePTY)
|
||||
}()
|
||||
|
||||
select {
|
||||
@@ -879,7 +902,10 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
||||
// truncate the tail of the log. The goroutine exits once the hijacked
|
||||
// stream is closed by resp.Close() in the caller's defer.
|
||||
select {
|
||||
case <-cmdResponse:
|
||||
case err := <-cmdResponse:
|
||||
if err != nil {
|
||||
logger.Debugf("Failed to drain command output after cancellation: %v", err)
|
||||
}
|
||||
case <-time.After(drainGracePeriod):
|
||||
logger.Warn("Timed out draining command output after cancellation")
|
||||
}
|
||||
@@ -888,11 +914,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
||||
// from executing
|
||||
return ctx.Err()
|
||||
case err := <-cmdResponse:
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -908,28 +930,24 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
return err
|
||||
}
|
||||
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
||||
defer func(tarFile *os.File) {
|
||||
name := tarFile.Name()
|
||||
err := tarFile.Close()
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
defer func() {
|
||||
if err := tarFile.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||
logger.Error(err)
|
||||
}
|
||||
err = os.Remove(name)
|
||||
if err != nil {
|
||||
if err := os.Remove(tarFile.Name()); err != nil {
|
||||
logger.Error(err)
|
||||
}
|
||||
}(tarFile)
|
||||
}()
|
||||
extractionRoot := "/"
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID, cr.markExecutable(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath)
|
||||
_, err = tarFile.Seek(0, 0)
|
||||
if err != nil {
|
||||
if _, err := tarFile.Seek(0, 0); err != nil {
|
||||
return fmt.Errorf("failed to seek tar archive: %w", err)
|
||||
}
|
||||
return cr.upload(ctx, extractionRoot, tarFile)
|
||||
@@ -960,6 +978,10 @@ func (cr *containerReference) upload(ctx context.Context, dstPath string, conten
|
||||
|
||||
func (cr *containerReference) attach() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
tty, err := cr.containerTTY(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
||||
Stream: true,
|
||||
Stdout: true,
|
||||
@@ -970,9 +992,12 @@ func (cr *containerReference) attach() common.Executor {
|
||||
}
|
||||
done := make(chan struct{})
|
||||
cr.attachDone = done
|
||||
stopClose := context.AfterFunc(ctx, out.Close)
|
||||
go func() {
|
||||
defer close(done)
|
||||
if copyErr := cr.copyOutput(out.Reader); copyErr != nil {
|
||||
defer out.Close()
|
||||
defer stopClose()
|
||||
if copyErr := cr.copyOutput(out.Reader, tty); copyErr != nil && ctx.Err() == nil {
|
||||
common.Logger(ctx).Error(copyErr)
|
||||
}
|
||||
}()
|
||||
@@ -1007,6 +1032,9 @@ func (cr *containerReference) wait() common.Executor {
|
||||
return fmt.Errorf("failed to wait for container: %w", err)
|
||||
}
|
||||
case status := <-waitResult.Result:
|
||||
if status.Error != nil {
|
||||
return fmt.Errorf("failed to wait for container: %s", status.Error.Message)
|
||||
}
|
||||
statusCode = status.StatusCode
|
||||
}
|
||||
|
||||
@@ -1085,21 +1113,19 @@ func resetOption[T any](logger *logging.Logger, option string, field *T, trusted
|
||||
|
||||
// parseOptionsHostConfig parses one options string on its own, to see what it alone asks for.
|
||||
// Even "" goes through the parser, or its empty slices and maps would differ from a real parse.
|
||||
func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
func parseOptionsHostConfig(options, serverOS string) (*container.HostConfig, error) {
|
||||
flags, copts, _, err := parseContainerOptions(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
return containerConfig.HostConfig, nil
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, hostConfig *container.HostConfig) {
|
||||
logger := common.Logger(ctx)
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
|
||||
@@ -1148,13 +1174,12 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
||||
}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
|
||||
return config, hostConfig
|
||||
}
|
||||
|
||||
func SanitizeBinds(ctx context.Context, validVolumes, binds []string) []string {
|
||||
cr := &containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}
|
||||
_, hostConfig := cr.sanitizeConfig(ctx, nil, &container.HostConfig{Binds: binds})
|
||||
hostConfig := &container.HostConfig{Binds: binds}
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
return hostConfig.Binds
|
||||
}
|
||||
|
||||
|
||||
+340
-166
@@ -13,9 +13,13 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
@@ -23,10 +27,12 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/api/types/system"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
@@ -61,6 +67,14 @@ func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
|
||||
type mockDockerClient struct {
|
||||
mobyclient.APIClient
|
||||
mock.Mock
|
||||
info system.Info
|
||||
infoErr error
|
||||
infoCalls int
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
m.infoCalls++
|
||||
return mobyclient.SystemInfoResult{Info: m.info}, m.infoErr
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ServerVersion(ctx context.Context, opts mobyclient.ServerVersionOptions) (mobyclient.ServerVersionResult, error) {
|
||||
@@ -88,6 +102,16 @@ func (m *mockDockerClient) ContainerAttach(ctx context.Context, containerID stri
|
||||
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerLogs(ctx context.Context, containerID string, opts mobyclient.ContainerLogsOptions) (mobyclient.ContainerLogsResult, error) {
|
||||
args := m.Called(ctx, containerID, opts)
|
||||
return args.Get(0).(mobyclient.ContainerLogsResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerStart(ctx context.Context, containerID string, opts mobyclient.ContainerStartOptions) (mobyclient.ContainerStartResult, error) {
|
||||
args := m.Called(ctx, containerID, opts)
|
||||
return args.Get(0).(mobyclient.ContainerStartResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
|
||||
args := m.Called(ctx, containerID, opts)
|
||||
return args.Get(0).(mobyclient.ContainerWaitResult)
|
||||
@@ -154,6 +178,7 @@ func (r *interruptReader) Read(_ []byte) (int, error) {
|
||||
type mockConn struct {
|
||||
net.Conn
|
||||
mock.Mock
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (m *mockConn) Write(b []byte) (n int, err error) {
|
||||
@@ -162,6 +187,7 @@ func (m *mockConn) Write(b []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
func (m *mockConn) Close() (err error) {
|
||||
m.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -182,11 +208,9 @@ func TestDockerExecAbort(t *testing.T) {
|
||||
}, nil)
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{},
|
||||
}
|
||||
|
||||
channel := make(chan error)
|
||||
@@ -209,12 +233,10 @@ func TestDockerExecAbort(t *testing.T) {
|
||||
func TestDockerExecFailure(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
conn := &mockConn{}
|
||||
|
||||
client := &mockDockerClient{}
|
||||
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
|
||||
Conn: conn,
|
||||
Conn: &mockConn{},
|
||||
Reader: bufio.NewReader(strings.NewReader("output")),
|
||||
}, nil)
|
||||
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
|
||||
@@ -222,11 +244,9 @@ func TestDockerExecFailure(t *testing.T) {
|
||||
}, nil)
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{},
|
||||
}
|
||||
|
||||
err := cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
|
||||
@@ -235,7 +255,21 @@ func TestDockerExecFailure(t *testing.T) {
|
||||
assert.Equal(t, ExitCodeError(1), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 1.", err.Error())
|
||||
|
||||
conn.AssertExpectations(t)
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerExecStreamFailure(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).
|
||||
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).
|
||||
Return(mobyclient.ExecAttachResult{
|
||||
Conn: &mockConn{},
|
||||
Reader: bufio.NewReader(bytes.NewReader(stdcopyFrame(stdcopy.Systemerr, "unable to start exec"))),
|
||||
}, nil)
|
||||
require.ErrorContains(t, (&containerReference{id: "123", cli: client, input: &NewContainerInput{}}).
|
||||
exec([]string{"sh"}, nil, "", "")(ctx), "unable to start exec")
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -250,86 +284,89 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
// TestDockerAttachFlushesTrailingLine verifies that wait() blocks until the
|
||||
// attach() streaming goroutine has drained and flushed the container's output,
|
||||
// so a final line without a trailing newline is not lost.
|
||||
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
tty bool
|
||||
}{
|
||||
{name: "multiplexed"},
|
||||
{name: "tty from container options", tty: true},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
output := []byte("line one\nlast line without newline")
|
||||
if !testcase.tty {
|
||||
output = stdcopyFrame(stdcopy.Stdout, string(output))
|
||||
}
|
||||
var lines []string
|
||||
logWriter := common.NewLineWriter(func(line string) bool {
|
||||
lines = append(lines, line)
|
||||
return true
|
||||
})
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Tty: testcase.tty}}}, nil)
|
||||
conn := &mockConn{}
|
||||
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||
Return(mobyclient.ContainerAttachResult{
|
||||
Conn: conn,
|
||||
Reader: bufio.NewReader(bytes.NewReader(output)),
|
||||
}, nil)
|
||||
client.On("ContainerLogs", ctx, "123", mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}).
|
||||
Return(io.NopCloser(bytes.NewReader(output)), nil)
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
statusCh <- container.WaitResponse{StatusCode: 0}
|
||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{Stdout: logWriter, Stderr: logWriter}}
|
||||
|
||||
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
|
||||
|
||||
var lines []string
|
||||
logWriter := common.NewLineWriter(func(s string) bool {
|
||||
lines = append(lines, s)
|
||||
return true
|
||||
})
|
||||
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||
Return(mobyclient.ContainerAttachResult{
|
||||
Conn: &mockConn{},
|
||||
Reader: bufio.NewReader(framed),
|
||||
}, nil)
|
||||
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
statusCh <- container.WaitResponse{StatusCode: 0}
|
||||
errCh := make(chan error, 1)
|
||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||
Return(mobyclient.ContainerWaitResult{
|
||||
Result: (<-chan container.WaitResponse)(statusCh),
|
||||
Error: (<-chan error)(errCh),
|
||||
require.NoError(t, cr.attach()(ctx))
|
||||
require.NoError(t, cr.wait()(ctx))
|
||||
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
||||
assert.True(t, conn.closed.Load())
|
||||
require.NoError(t, cr.DumpLogs(ctx))
|
||||
assert.Equal(t, []string{"line one\n", "last line without newline", "line one\n", "last line without newline"}, lines)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
Stdout: logWriter,
|
||||
Stderr: logWriter,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
require.NoError(t, cr.attach()(ctx))
|
||||
require.NoError(t, cr.wait()(ctx))
|
||||
|
||||
// wait() must have blocked until the goroutine drained AND flushed; the
|
||||
// trailing, non-newline-terminated line must therefore be present. Reading
|
||||
// lines here is race-free because wait() synchronizes on attachDone, which
|
||||
// the goroutine closes after the final append.
|
||||
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
||||
func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
|
||||
conn, peer := net.Pipe()
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
t.Cleanup(func() { _ = peer.Close() })
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", mock.Anything, "123", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{}}}, nil)
|
||||
client.On("ContainerAttach", mock.Anything, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||
Return(mobyclient.ContainerAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
|
||||
client.On("ContainerStart", mock.Anything, "123", mobyclient.ContainerStartOptions{}).
|
||||
Return(mobyclient.ContainerStartResult{}, errors.New("container start failed"))
|
||||
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{}}
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
require.ErrorContains(t, cr.Start(true)(ctx), "container start failed")
|
||||
cancel()
|
||||
<-cr.attachDone
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerWaitFailure(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
statusCh <- container.WaitResponse{StatusCode: 2}
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||
Return(mobyclient.ContainerWaitResult{
|
||||
Result: (<-chan container.WaitResponse)(statusCh),
|
||||
Error: (<-chan error)(errCh),
|
||||
})
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
}
|
||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||
cr := &containerReference{id: "123", cli: client}
|
||||
|
||||
err := cr.wait()(ctx)
|
||||
var exitErr ExitCodeError
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
assert.Equal(t, ExitCodeError(2), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 2.", err.Error())
|
||||
|
||||
statusCh <- container.WaitResponse{Error: &container.WaitExitError{Message: "container wait failed"}}
|
||||
require.EqualError(t, cr.wait()(ctx), "failed to wait for container: container wait failed")
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -398,62 +435,79 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// find() must drop a stale cached id so later Copy/Exec don't hit the
|
||||
// daemon with a torn-down container.
|
||||
func TestRemovePreservesClientCreationError(t *testing.T) {
|
||||
t.Setenv("DOCKER_HOST", "://")
|
||||
|
||||
require.ErrorContains(t, (&containerReference{input: &NewContainerInput{Name: "job-1"}}).
|
||||
Remove()(t.Context()), "failed to connect to docker daemon")
|
||||
}
|
||||
|
||||
func TestFindRevalidatesStaleID(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
|
||||
boom := errors.New("daemon unreachable")
|
||||
newCR := func(id string) (*containerReference, *mockDockerClient) {
|
||||
client := &mockDockerClient{}
|
||||
return &containerReference{id: id, cli: client, input: &NewContainerInput{Name: "job-1"}}, client
|
||||
for _, tc := range []struct {
|
||||
name, cachedID string
|
||||
cachedErr error
|
||||
lookup container.InspectResponse
|
||||
lookupErr error
|
||||
listed []container.Summary
|
||||
wantID string
|
||||
retryList bool
|
||||
}{
|
||||
{name: "stale id cleared, name lookup empty", cachedID: "stale", cachedErr: notFound, lookupErr: notFound},
|
||||
{name: "stale id cleared, name lookup repopulates", cachedID: "stale", cachedErr: notFound, lookup: container.InspectResponse{ID: "fresh", Name: "/job-1"}, wantID: "fresh"},
|
||||
{name: "id match falls back to listing", lookup: container.InspectResponse{ID: "job-1abc", Name: "/somebody-else"}, listed: []container.Summary{{ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh"},
|
||||
{name: "live id kept", cachedID: "live", wantID: "live"},
|
||||
{name: "transient inspect error trusts cache", cachedID: "live", cachedErr: boom, wantID: "live"},
|
||||
{name: "name lookup error falls back to listing", lookupErr: boom, listed: []container.Summary{{ID: "other", Names: []string{"/somebody-else"}}, {ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh", retryList: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &mockDockerClient{}
|
||||
cr := &containerReference{id: tc.cachedID, cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
if tc.cachedID != "" {
|
||||
client.On("ContainerInspect", ctx, tc.cachedID, inspectOpts).Return(mobyclient.ContainerInspectResult{}, tc.cachedErr)
|
||||
}
|
||||
if tc.cachedID == "" || cerrdefs.IsNotFound(tc.cachedErr) {
|
||||
client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{Container: tc.lookup}, tc.lookupErr)
|
||||
}
|
||||
if tc.listed != nil {
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: tc.listed}, nil).Once()
|
||||
}
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, tc.wantID, cr.id)
|
||||
if tc.retryList {
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, boom)
|
||||
cr.id = ""
|
||||
require.ErrorIs(t, cr.find()(ctx), boom)
|
||||
}
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
listOpts := mobyclient.ContainerListOptions{All: true}
|
||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||
}
|
||||
|
||||
t.Run("stale id cleared, name lookup empty", func(t *testing.T) {
|
||||
cr, client := newCR("stale")
|
||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Empty(t, cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("stale id cleared, name lookup repopulates", func(t *testing.T) {
|
||||
cr, client := newCR("stale")
|
||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
||||
{ID: "other", Names: []string{"/somebody-else"}},
|
||||
{ID: "fresh", Names: []string{"/job-1"}},
|
||||
}}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "fresh", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("live id kept", func(t *testing.T) {
|
||||
cr, client := newCR("live")
|
||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "live", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("transient inspect error trusts cache", func(t *testing.T) {
|
||||
cr, client := newCR("live")
|
||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, boom)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "live", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("list error propagates", func(t *testing.T) {
|
||||
cr, client := newCR("")
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, boom)
|
||||
require.ErrorIs(t, cr.find()(ctx), boom)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
func TestRunnerArchCachedPerRun(t *testing.T) {
|
||||
var infos atomic.Int32
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||
infos.Add(1)
|
||||
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
extensions := &LinuxContainerEnvironmentExtensions{}
|
||||
run := WithRunnerArch(t.Context())
|
||||
var lookups sync.WaitGroup
|
||||
for range 4 {
|
||||
lookups.Go(func() { assert.Equal(t, "ARM64", extensions.GetRunnerContext(run)["arch"]) })
|
||||
}
|
||||
lookups.Wait()
|
||||
assert.Equal(t, "ARM64", extensions.GetRunnerContext(t.Context())["arch"])
|
||||
assert.EqualValues(t, 2, infos.Load())
|
||||
}
|
||||
|
||||
// Every daemon entry point fails fast with a clear, container-named
|
||||
@@ -461,7 +515,8 @@ func TestFindRevalidatesStaleID(t *testing.T) {
|
||||
func TestRejectsMissingContainer(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
|
||||
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
|
||||
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
check := func(op string, err error) {
|
||||
t.Helper()
|
||||
@@ -485,6 +540,9 @@ func TestRejectsMissingContainer(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||
|
||||
for _, testcase := range []struct {
|
||||
name, destination, extractionRoot, entry string
|
||||
directory bool
|
||||
@@ -494,14 +552,13 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
hook := logtest.NewWithLevel(logging.LevelError)
|
||||
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
|
||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
||||
{ID: "fresh", Names: []string{"/job-1"}},
|
||||
}}, nil)
|
||||
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
|
||||
client.On("CopyToContainer", ctx, "fresh", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
|
||||
@@ -513,21 +570,17 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
|
||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
if testcase.directory {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
||||
} else {
|
||||
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||
}
|
||||
assert.Equal(t, "fresh", cr.id)
|
||||
assert.Empty(t, hook.AllRecords())
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Type assert containerReference implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &containerReference{}
|
||||
|
||||
func TestCheckVolumes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
@@ -622,10 +675,11 @@ func TestCheckVolumes(t *testing.T) {
|
||||
ValidVolumes: tc.validVolumes,
|
||||
},
|
||||
}
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts})
|
||||
hostConf := &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
||||
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
||||
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts)) // every drop is warned about
|
||||
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -645,27 +699,21 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
|
||||
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
|
||||
}
|
||||
|
||||
// Both of these are read here, on the runner, so a workflow could read the runner's files
|
||||
// and environment with them.
|
||||
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||
hostFile := filepath.Join(t.TempDir(), "host.env")
|
||||
hostFile := filepath.Join(t.TempDir(), "host file.env")
|
||||
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
||||
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
||||
|
||||
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
||||
for _, option := range []string{shellquote.Join("--env-file", hostFile), shellquote.Join("--label-file", hostFile)} {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
||||
|
||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.ErrorContains(t, err, "not allowed in a workflow")
|
||||
require.ErrorContains(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}), "not allowed in a workflow")
|
||||
|
||||
// the runner reading its own files is what those options are for
|
||||
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
|
||||
_, _, err = cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||
}
|
||||
|
||||
// a bare name is no longer resolved from the runner's environment, for either source
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
NetworkMode: "bridge",
|
||||
@@ -673,8 +721,8 @@ func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
|
||||
}}
|
||||
|
||||
config, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
config := &container.Config{}
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), config, &container.HostConfig{}))
|
||||
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
|
||||
}
|
||||
|
||||
@@ -711,7 +759,6 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
assert.Equal(t, &container.HostConfig{}, hostConfig)
|
||||
}
|
||||
|
||||
// mergeOptions merges both option sources into a bare container, returning the result and its log.
|
||||
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *logtest.Handler) {
|
||||
t.Helper()
|
||||
logger, hook := logtest.NewLogger()
|
||||
@@ -723,12 +770,12 @@ func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privilege
|
||||
ValidVolumes: []string{"**"},
|
||||
}}
|
||||
|
||||
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{
|
||||
hostConfig := &container.HostConfig{
|
||||
Privileged: privileged,
|
||||
UsernsMode: container.UsernsMode("private"),
|
||||
NetworkMode: container.NetworkMode("bridge"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, hostConfig))
|
||||
return hostConfig, hook
|
||||
}
|
||||
|
||||
@@ -782,23 +829,26 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
escapingPath := allowed + string(filepath.Separator) + ".." + string(filepath.Separator) + "denied"
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf := &container.HostConfig{
|
||||
Binds: []string{escapingPath + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
|
||||
linkPath := filepath.Join(allowed, "link")
|
||||
if err := os.Symlink(denied, linkPath); err != nil {
|
||||
t.Skipf("cannot create symlink: %v", err)
|
||||
}
|
||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf = &container.HostConfig{
|
||||
Binds: []string{linkPath + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
|
||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf = &container.HostConfig{
|
||||
Binds: []string{filepath.Join(linkPath, "missing") + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
}
|
||||
|
||||
@@ -808,11 +858,12 @@ func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
|
||||
|
||||
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
||||
logger, hook := logtest.NewLogger()
|
||||
_, sanitized := (&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{Mounts: []mount.Mount{
|
||||
sanitized := &container.HostConfig{Mounts: []mount.Mount{
|
||||
hostConfig.Mounts[0],
|
||||
{Type: mount.TypeVolume, Target: "/anonymous"},
|
||||
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
||||
}})
|
||||
}}
|
||||
(&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), sanitized)
|
||||
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
||||
assert.Len(t, hook.AllRecords(), 2)
|
||||
}
|
||||
@@ -893,6 +944,7 @@ func TestContainerInfoFromInspect(t *testing.T) {
|
||||
"/etc/hosts": "",
|
||||
"/etc/hostname": "",
|
||||
}, info.Mounts)
|
||||
assert.Equal(t, "/var/lib/docker/volumes/job/_data/subdir", info.DaemonPath("/workspace/owner/repo/subdir"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -908,11 +960,11 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf := &container.HostConfig{
|
||||
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
|
||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, cr.mergeContainerConfigs(ctx, &container.Config{}, hostConf))
|
||||
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
|
||||
assert.Empty(t, hostConf.Mounts)
|
||||
}
|
||||
@@ -940,7 +992,7 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||
},
|
||||
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if len(entrypoint) > 0 {
|
||||
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||
} else {
|
||||
@@ -958,14 +1010,46 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCreateNetworkAliases(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
networkMode string
|
||||
wantAliases bool
|
||||
}{
|
||||
{"job-network", true},
|
||||
{"host", false},
|
||||
{"bridge", false},
|
||||
{"default", false},
|
||||
{"none", false},
|
||||
{"container:x", false},
|
||||
} {
|
||||
t.Run(test.networkMode, func(t *testing.T) {
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{NetworkMode: test.networkMode, NetworkAliases: []string{"service"}},
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, container.NetworkMode(test.networkMode), opts.HostConfig.NetworkMode)
|
||||
if test.wantAliases {
|
||||
assert.Equal(t, &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
"job-network": {Aliases: []string{"service"}},
|
||||
}}, opts.NetworkingConfig)
|
||||
} else {
|
||||
assert.Nil(t, opts.NetworkingConfig)
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "created"}, nil
|
||||
}},
|
||||
}
|
||||
require.NoError(t, cr.create(nil, nil)(t.Context()))
|
||||
assert.Equal(t, "created", cr.id)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
|
||||
}}
|
||||
|
||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||
assert.Equal(t, []string{"redis", "redis-primary"}, cr.input.NetworkAliases)
|
||||
}
|
||||
|
||||
@@ -980,3 +1064,93 @@ func TestSupportsContainerImagePlatformDaemonError(t *testing.T) {
|
||||
require.ErrorContains(t, err, "cannot connect to the Docker daemon")
|
||||
assert.False(t, supported)
|
||||
}
|
||||
|
||||
func TestContainerOptionsUseDaemonOS(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
serverOS string
|
||||
infoErr error
|
||||
device string
|
||||
expected container.DeviceMapping
|
||||
}{
|
||||
{name: "Linux daemon", serverOS: "linux", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unknown daemon", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unavailable daemon info", infoErr: errors.New("unavailable"), device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "Windows daemon", serverOS: "windows", device: "class/01234567-89ab-cdef-0123-456789abcdef", expected: container.DeviceMapping{PathOnHost: "class/01234567-89ab-cdef-0123-456789abcdef"}},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}, infoErr: testcase.infoErr}
|
||||
cr := &containerReference{cli: cli, input: &NewContainerInput{RunnerOptions: "--device=" + testcase.device, WorkflowOptions: "--device=" + testcase.device, NetworkMode: "bridge"}}
|
||||
for range 2 {
|
||||
hostConfig := &container.HostConfig{}
|
||||
require.NoError(t, cr.mergeContainerConfigs(t.Context(), &container.Config{}, hostConfig))
|
||||
assert.Equal(t, []container.DeviceMapping{testcase.expected}, hostConfig.Devices)
|
||||
}
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTakesDaemonOSFromPing(t *testing.T) {
|
||||
var infos atomic.Int32
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
writer.Header().Set("OSType", "windows")
|
||||
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||
infos.Add(1)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
cr := &containerReference{input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.connect()(t.Context()))
|
||||
assert.Equal(t, "windows", cr.targetOS(t.Context()))
|
||||
assert.Zero(t, infos.Load())
|
||||
}
|
||||
|
||||
func TestContainerCopyModes(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||
|
||||
for _, testcase := range []struct {
|
||||
hostOS, serverOS string
|
||||
expected int64
|
||||
}{
|
||||
{"windows", "linux", 0o755},
|
||||
{"windows", "windows", 0o666},
|
||||
{"linux", "linux", 0o666},
|
||||
} {
|
||||
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
||||
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
||||
original := hostOS
|
||||
hostOS = testcase.hostOS
|
||||
t.Cleanup(func() { hostOS = original })
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}}
|
||||
var modes []int64
|
||||
cli.On("CopyToContainer", mock.Anything, "job", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
reader := tar.NewReader(args.Get(2).(mobyclient.CopyToContainerOptions).Content)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
modes = append(modes, header.Mode)
|
||||
assert.Equal(t, "run.sh", filepath.Base(header.Name))
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, file.Body, string(content))
|
||||
_, err = reader.Next()
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
}).Return(mobyclient.CopyToContainerResult{}, nil).Twice()
|
||||
cr := &containerReference{cli: cli, id: "job", input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.copyContent("/workspace", file)(t.Context()))
|
||||
require.NoError(t, cr.copyDir("/workspace", source+string(filepath.Separator), false, true)(t.Context()))
|
||||
assert.Equal(t, []int64{0o600, testcase.expected}, modes)
|
||||
if testcase.hostOS == "windows" {
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
} else {
|
||||
assert.Zero(t, cli.infoCalls)
|
||||
}
|
||||
cli.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -30,6 +30,7 @@ func TestSSHArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, err := exec.LookPath("sh"); err != nil {
|
||||
t.Skip("needs a POSIX shell")
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -4,7 +4,18 @@
|
||||
|
||||
package container
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func JoinPath(elem ...string) string { // path.Join would collapse the leading // of a UNC act path on a Windows host
|
||||
if len(elem) > 0 && strings.HasPrefix(elem[0], "//") {
|
||||
return "/" + path.Join(elem...)
|
||||
}
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
type ExecutionsEnvironment interface {
|
||||
Container
|
||||
|
||||
@@ -7,6 +7,7 @@ package container
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -39,7 +40,8 @@ type HostEnvironment struct {
|
||||
// when Workdir points at a caller-owned checkout (e.g. `act` local mode).
|
||||
CleanWorkdir bool
|
||||
ActPath string
|
||||
CleanUp func()
|
||||
Root string
|
||||
TrackingID string // RUNNER_TRACKING_ID of the job, a step with another value keeps its leftovers
|
||||
StdOut io.Writer
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
|
||||
@@ -55,7 +57,7 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
||||
e.procGroupOnce.Do(func() {
|
||||
group, err := process.NewGroup()
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("could not create the job's process group; processes a step leaves behind can only be reclaimed by the workspace scan: %v", err)
|
||||
common.Logger(ctx).Warnf("could not create the job's process group, processes a step leaves behind may outlive the job: %v", err)
|
||||
return
|
||||
}
|
||||
e.procGroup.Store(group)
|
||||
@@ -64,15 +66,11 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Close() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
@@ -168,15 +166,11 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Pull(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Start(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
type ptyWriter struct {
|
||||
@@ -193,7 +187,6 @@ func (w *ptyWriter) Write(buf []byte) (int, error) {
|
||||
}
|
||||
if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
|
||||
_, _ = w.Out.Write([]byte("\n"))
|
||||
return n, io.EOF
|
||||
}
|
||||
return n, io.EOF
|
||||
}
|
||||
@@ -233,53 +226,43 @@ func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
|
||||
return ppty, tty, nil
|
||||
}
|
||||
|
||||
func writeKeepAlive(ppty io.Writer) {
|
||||
c := 1
|
||||
var err error
|
||||
for c == 1 && err == nil {
|
||||
c, err = ppty.Write([]byte{4})
|
||||
<-time.After(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) {
|
||||
defer func() {
|
||||
finishLog()
|
||||
}()
|
||||
if _, err := io.Copy(writer, ppty); err != nil {
|
||||
return
|
||||
func writeKeepAlive(ctx context.Context, ppty io.Writer) {
|
||||
for ctx.Err() == nil {
|
||||
if count, err := ppty.Write([]byte{4}); count != 1 || err != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) UpdateFromImageEnv(_ *map[string]string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func getEnvListFromMap(env map[string]string) []string {
|
||||
envList := make([]string, 0)
|
||||
for k, v := range env {
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
envList := make([]string, 0, len(env))
|
||||
for key, value := range env {
|
||||
envList = append(envList, key+"="+value)
|
||||
}
|
||||
return envList
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error {
|
||||
envList := getEnvListFromMap(env)
|
||||
var wd string
|
||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, workdir string) error {
|
||||
wd := e.Path
|
||||
if workdir != "" {
|
||||
if filepath.IsAbs(workdir) {
|
||||
wd = workdir
|
||||
} else {
|
||||
wd = workdir
|
||||
if !filepath.IsAbs(workdir) {
|
||||
wd = filepath.Join(e.Path, workdir)
|
||||
}
|
||||
} else {
|
||||
wd = e.Path
|
||||
}
|
||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||
defer common.FlushWriter(e.StdOut)
|
||||
process.UTF8Console()
|
||||
|
||||
f, err := lookupPathHost(command[0], env, e.StdOut)
|
||||
if err != nil {
|
||||
@@ -288,9 +271,13 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
cmd := exec.CommandContext(ctx, f)
|
||||
cmd.Path = f
|
||||
cmd.Args = command
|
||||
cmd.Stdin = nil
|
||||
cmd.Stdout = e.StdOut
|
||||
cmd.Env = envList
|
||||
trackingID := e.TrackingID
|
||||
if value, ok := common.LookupFold(env, "RUNNER_TRACKING_ID", e.IsEnvironmentCaseInsensitive()); ok {
|
||||
trackingID = value
|
||||
}
|
||||
tracked := trackingID == e.TrackingID
|
||||
cmd.Env = append(getEnvListFromMap(env), "RUNNER_TRACKING_ID="+trackingID)
|
||||
cmd.Stderr = e.StdOut
|
||||
cmd.Dir = wd
|
||||
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
||||
@@ -298,6 +285,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
// Kills the step's whole tree on cancellation and bounds the post-exit I/O
|
||||
// wait, so an orphan holding cmd's stdout pipe cannot hang cmd.Wait().
|
||||
treeKill := process.NewTreeKill(cmd)
|
||||
treeKill.KeepLeftovers = !tracked
|
||||
|
||||
var ppty *os.File
|
||||
var tty *os.File
|
||||
@@ -322,16 +310,19 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
writer = &ptyWriter{Out: e.StdOut}
|
||||
var finishLog context.CancelFunc
|
||||
logctx, finishLog = context.WithCancel(context.Background())
|
||||
go copyPtyOutput(writer, ppty, finishLog)
|
||||
go writeKeepAlive(ppty)
|
||||
go func() {
|
||||
defer finishLog()
|
||||
_, _ = io.Copy(writer, ppty)
|
||||
}()
|
||||
go writeKeepAlive(logctx, ppty)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Assign before the step's Killer so the step's job nests inside the group's;
|
||||
// cancellation still scopes to this step's tree.
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group; a process it leaves behind may outlive the job: %v", err)
|
||||
if tracked {
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group, a process it leaves behind may outlive the job: %v", err)
|
||||
}
|
||||
}
|
||||
if k, kerr := treeKill.Capture(cmd.Process); kerr != nil {
|
||||
common.Logger(ctx).Warnf("process tree kill setup failed, falling back to single-process kill: %v", kerr)
|
||||
@@ -339,31 +330,31 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
defer k.Close()
|
||||
}
|
||||
err = treeKill.Wait(ctx, cmd)
|
||||
if err != nil {
|
||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||
return ExitCodeError(exitErr.ExitCode())
|
||||
}
|
||||
return err
|
||||
}
|
||||
if tty != nil {
|
||||
writer.AutoStop.Store(true)
|
||||
if _, err := tty.WriteString("\x04"); err != nil {
|
||||
common.Logger(ctx).Debug("Failed to write EOT")
|
||||
}
|
||||
<-logctx.Done()
|
||||
select {
|
||||
case <-logctx.Done():
|
||||
case <-ctx.Done(): // a cancelled step does not wait on output nobody consumes
|
||||
}
|
||||
ppty.Close()
|
||||
ppty = nil
|
||||
}
|
||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||
return ExitCodeError(exitErr.ExitCode())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor {
|
||||
return e.ExecWithCmdLine(command, "", env, user, workdir)
|
||||
func (e *HostEnvironment) Exec(command []string, env map[string]string, _user, workdir string) common.Executor {
|
||||
return e.ExecWithCmdLine(command, "", env, workdir)
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor {
|
||||
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, workdir string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil {
|
||||
if err := e.exec(ctx, command, cmdline, env, workdir); err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("this step has been cancelled: %w", err)
|
||||
@@ -376,15 +367,13 @@ func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
return parseEnvFile(e, srcPath, env)
|
||||
return parseEnvFile(e, srcPath, env, runtime.GOOS == "windows")
|
||||
}
|
||||
|
||||
// removeAll is a var so tests can substitute a blocking stub.
|
||||
var removeAll = os.RemoveAll
|
||||
|
||||
// removeAllWithContext returns once the delete finishes or ctx is cancelled. On
|
||||
// cancellation the goroutine leaks: a delete inside a syscall cannot be
|
||||
// interrupted (see runWithTimeout).
|
||||
// Returns on cancel while an uninterruptible delete keeps running, freeing the runner's capacity slot.
|
||||
func removeAllWithContext(ctx context.Context, path string) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- removeAll(path) }()
|
||||
@@ -426,129 +415,22 @@ func removePathWithRetry(ctx context.Context, path string) error {
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// buildWindowsWorkspaceKillScript builds a PowerShell command that taskkills
|
||||
// every process tree whose ExecutablePath or CommandLine references one of the
|
||||
// given workspace dirs, releasing file handles for cleanup. Win32_Process
|
||||
// exposes both fields (Get-Process doesn't, wmic is deprecated); matching is on
|
||||
// the dir+separator prefix via ordinal String methods, so a name-prefix sibling
|
||||
// (job1 vs job10) is spared and path metacharacters stay literal.
|
||||
func buildWindowsWorkspaceKillScript(dirs []string) string {
|
||||
quoted := make([]string, len(dirs))
|
||||
for i, d := range dirs {
|
||||
// Single-quoted PowerShell literal; escape ' by doubling it.
|
||||
quoted[i] = "'" + strings.ReplaceAll(d, "'", "''") + "'"
|
||||
}
|
||||
|
||||
return `$paths = @(` + strings.Join(quoted, ",") + `)
|
||||
$selfPid = $PID
|
||||
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object {
|
||||
if ($_.ProcessId -eq $selfPid) { return $false }
|
||||
foreach ($p in $paths) {
|
||||
$prefix = $p + '\'
|
||||
if ($_.ExecutablePath -and $_.ExecutablePath.StartsWith($prefix, [System.StringComparison]::OrdinalIgnoreCase)) { return $true }
|
||||
if ($_.CommandLine -and $_.CommandLine.IndexOf($prefix, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) { return $true }
|
||||
}
|
||||
return $false
|
||||
} | ForEach-Object {
|
||||
& taskkill.exe /PID $_.ProcessId /T /F 2>$null | Out-Null
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) terminateRunningProcesses(ctx context.Context) {
|
||||
if runtime.GOOS != "windows" {
|
||||
return
|
||||
}
|
||||
|
||||
// Detached: exec.CommandContext won't start on a cancelled ctx, and a
|
||||
// server cancel has already cancelled the parent ctx.
|
||||
killCtx, killCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer killCancel()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// Dirs we own; a process referencing one is a leftover. ToolCache is shared
|
||||
// across jobs, and Workdir may be a caller-owned checkout.
|
||||
owned := []string{e.Path, e.TmpDir}
|
||||
if e.CleanWorkdir {
|
||||
owned = append(owned, e.Workdir)
|
||||
}
|
||||
dirs := make([]string, 0, len(owned))
|
||||
for _, d := range owned {
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := filepath.Abs(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dirs = append(dirs, abs)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
script := buildWindowsWorkspaceKillScript(dirs)
|
||||
|
||||
cmd := exec.CommandContext(killCtx, "powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
logger.Debugf("workspace process-tree kill via PowerShell failed: %v output=%s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Win32_Process exposes no working directory, so the scan above misses a
|
||||
// process that merely runs in a workspace dir while pinning a handle on it.
|
||||
if killed, err := process.KillProcessesWithCWDUnder(killCtx, dirs); err != nil {
|
||||
logger.Debugf("workspace process kill by working directory reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("terminated %d leftover process(es) by workspace working directory", killed)
|
||||
}
|
||||
}
|
||||
|
||||
// hostCleanupTimeout bounds each teardown phase so one stalled delete cannot
|
||||
// wedge the runner slot. A var so tests can shrink it.
|
||||
var hostCleanupTimeout = 30 * time.Second
|
||||
|
||||
// runWithTimeout returns context.DeadlineExceeded once timeout elapses, leaking
|
||||
// the goroutine: a delete blocked in a syscall (AV filter driver, dead network
|
||||
// mount) cannot be interrupted, and leaking scratch state beats losing the
|
||||
// runner's capacity slot forever. The idle stale-dir sweep reclaims it later.
|
||||
func runWithTimeout(fn func(), timeout time.Duration) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
fn()
|
||||
}()
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-timer.C:
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// End lingering processes before removing the workspace; on Windows their
|
||||
// file locks block cleanup. Closing the group is deterministic, the scan a net.
|
||||
// End lingering processes before removing the workspace, on Windows their file locks block cleanup.
|
||||
if err := e.procGroup.Load().Close(); err != nil {
|
||||
logger.Debugf("closing the job's process group failed: %v", err)
|
||||
}
|
||||
e.terminateRunningProcesses(ctx)
|
||||
|
||||
// Removes per-job misc state only, never the toolcache root. Bounded because
|
||||
// CleanUp is a caller-supplied, typically unbounded os.RemoveAll.
|
||||
if e.CleanUp != nil {
|
||||
logger.Debugf("running host environment cleanup callback")
|
||||
if err := runWithTimeout(e.CleanUp, hostCleanupTimeout); err != nil {
|
||||
logger.Warnf("host environment cleanup did not finish within %s; continuing job completion, scratch state may be leaked and is reclaimed by the idle stale-dir sweep", hostCleanupTimeout)
|
||||
} else {
|
||||
logger.Debugf("host environment cleanup callback finished")
|
||||
}
|
||||
if killed, err := process.KillTrackedProcesses(e.TrackingID); err != nil {
|
||||
logger.Debugf("killing the job's leftover processes reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("killed %d leftover process(es) of the job", killed)
|
||||
}
|
||||
|
||||
// Detach: a cancelled ctx would skip removePathWithRetry's retries,
|
||||
@@ -557,8 +439,8 @@ func (e *HostEnvironment) Remove() common.Executor {
|
||||
defer rmCancel()
|
||||
|
||||
var errs []error
|
||||
if err := removePathWithRetry(rmCtx, e.Path); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", e.Path, err)
|
||||
if err := removePathWithRetry(rmCtx, cmp.Or(e.Root, e.Path)); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", cmp.Or(e.Root, e.Path), err)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if e.CleanWorkdir {
|
||||
@@ -605,8 +487,7 @@ func (*HostEnvironment) GetPathVariableName() string {
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) DefaultPathVariable() string {
|
||||
v, _ := os.LookupEnv(e.GetPathVariableName())
|
||||
return v
|
||||
return os.Getenv(e.GetPathVariableName())
|
||||
}
|
||||
|
||||
func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
||||
@@ -616,30 +497,23 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
||||
// Reference for Arch values for runner.arch
|
||||
// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
|
||||
func goArchToActionArch(arch string) string {
|
||||
archMapper := map[string]string{
|
||||
return cmp.Or(map[string]string{
|
||||
"x86_64": "X64",
|
||||
"amd64": "X64",
|
||||
"386": "X86",
|
||||
"arm": "ARM",
|
||||
"aarch64": "ARM64",
|
||||
"arm64": "ARM64",
|
||||
}
|
||||
if arch, ok := archMapper[arch]; ok {
|
||||
return arch
|
||||
}
|
||||
return arch
|
||||
}[arch], arch)
|
||||
}
|
||||
|
||||
func goOsToActionOs(os string) string {
|
||||
osMapper := map[string]string{
|
||||
return cmp.Or(map[string]string{
|
||||
"linux": "Linux",
|
||||
"darwin": "macOS",
|
||||
"windows": "Windows",
|
||||
}
|
||||
if os, ok := osMapper[os]; ok {
|
||||
return os
|
||||
}
|
||||
return os
|
||||
"freebsd": "FreeBSD",
|
||||
}[os], os)
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any {
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
@@ -24,15 +24,19 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Type assert HostEnvironment implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &HostEnvironment{}
|
||||
|
||||
func TestActionPlatformNames(t *testing.T) {
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows")})
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows", "FreeBSD"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows"), goOsToActionOs("freebsd")})
|
||||
assert.Equal(t, []string{"X86", "X64", "ARM", "ARM64"}, []string{goArchToActionArch("386"), goArchToActionArch("amd64"), goArchToActionArch("arm"), goArchToActionArch("arm64")})
|
||||
}
|
||||
|
||||
func TestCopyDir(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
useGitIgnore, skipGitDir bool
|
||||
@@ -42,11 +46,6 @@ func TestCopyDir(t *testing.T) {
|
||||
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "dest")
|
||||
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
||||
|
||||
@@ -61,63 +60,68 @@ func TestCopyDir(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetContainerArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := t.Context()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
StdOut: os.Stdout,
|
||||
Workdir: path.Join("testdata", "scratch"),
|
||||
}
|
||||
_ = os.MkdirAll(e.Path, 0o700)
|
||||
_ = os.MkdirAll(e.TmpDir, 0o700)
|
||||
_ = os.MkdirAll(e.ToolCache, 0o700)
|
||||
_ = os.MkdirAll(e.ActPath, 0o700)
|
||||
e := &HostEnvironment{Path: t.TempDir()}
|
||||
expectedContent := []byte("sdde/7sh")
|
||||
err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
archive, err := e.GetContainerArchive(ctx, e.Path)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
defer archive.Close()
|
||||
reader := tar.NewReader(archive)
|
||||
h, err := reader.Next()
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", h.Name)
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedContent, content)
|
||||
_, err = reader.Next()
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
}
|
||||
|
||||
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
ctx := t.Context()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
StdOut: io.Discard,
|
||||
Workdir: filepath.Join(dir, "path"),
|
||||
}
|
||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
||||
assert.NoError(t, os.MkdirAll(p, 0o700)) //nolint:testifylint // test setup
|
||||
}
|
||||
for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
e := &HostEnvironment{Path: t.TempDir(), StdOut: writer, AllocatePTY: allocatePTY}
|
||||
t.Cleanup(func() { assert.NoError(t, e.Remove()(t.Context())) })
|
||||
|
||||
err := e.Exec([]string{"sh", "-c", "exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(ctx)
|
||||
var exitErr ExitCodeError
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
defer writer.Close()
|
||||
result <- e.Exec([]string{"sh", "-c", "printf 'failed\\n'; exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(t.Context())
|
||||
}()
|
||||
first := make([]byte, 1)
|
||||
_, err := io.ReadFull(reader, first)
|
||||
require.NoError(t, err)
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-result:
|
||||
t.Fatalf("Exec returned before its output was consumed: %v", err)
|
||||
default:
|
||||
}
|
||||
remaining, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
err = <-result
|
||||
var exitErr ExitCodeError
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
||||
assert.Equal(t, "failed\n", string(first)+string(remaining))
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
@@ -130,20 +134,13 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||
{name: "on", allocPTY: true, expect: "TTY"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Parallel()
|
||||
buf := &bytes.Buffer{}
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
Path: t.TempDir(),
|
||||
StdOut: buf,
|
||||
Workdir: filepath.Join(dir, "path"),
|
||||
AllocatePTY: tc.allocPTY,
|
||||
}
|
||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
||||
require.NoError(t, os.MkdirAll(p, 0o700))
|
||||
}
|
||||
|
||||
err := e.Exec(
|
||||
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
|
||||
@@ -156,218 +153,88 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
|
||||
ctx := common.WithLogger(t.Context(), logging.Default())
|
||||
base := t.TempDir()
|
||||
miscRoot := filepath.Join(base, "misc")
|
||||
path := filepath.Join(miscRoot, "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
workdir := filepath.Join(base, "workspace", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
Workdir: workdir,
|
||||
CleanUp: func() {
|
||||
_ = os.RemoveAll(miscRoot)
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
||||
for name, owned := range map[string]bool{"preserves caller workdir": false, "cleans owned workdir": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
miscRoot, workdir := t.TempDir(), t.TempDir()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(miscRoot, "hostexecutor"),
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: owned,
|
||||
Root: miscRoot,
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(miscRoot, "tmp"), 0o700))
|
||||
require.NoError(t, e.Remove()(t.Context()))
|
||||
assert.NoDirExists(t, miscRoot)
|
||||
_, err := os.Stat(workdir)
|
||||
if owned {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
require.NoError(t, e.Remove()(ctx))
|
||||
_, err := os.Stat(workdir)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemoveCleansWorkdirWhenOwned(t *testing.T) {
|
||||
ctx := common.WithLogger(t.Context(), logging.Default())
|
||||
base := t.TempDir()
|
||||
miscRoot := filepath.Join(base, "misc")
|
||||
path := filepath.Join(miscRoot, "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
workdir := filepath.Join(base, "workspace", "123", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: true,
|
||||
CleanUp: func() {
|
||||
_ = os.RemoveAll(miscRoot)
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
require.NoError(t, e.Remove()(ctx))
|
||||
_, err := os.Stat(workdir)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
stubDone := make(chan struct{})
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
stubDone := make(chan struct{})
|
||||
|
||||
orig := removeAll
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
// removeAllWithContext intentionally leaks the delete goroutine on timeout,
|
||||
// and that goroutine still references removeAll. Unblock it and wait for it
|
||||
// to return before restoring the var, so the restore can't race the read.
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
removeAll = orig
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := removeAllWithContext(ctx, t.TempDir())
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp guards against a stalled
|
||||
// CleanUp callback (e.g. an os.RemoveAll blocked by an AV/EDR filter driver or
|
||||
// an unresponsive mount) wedging the runner slot forever at "Cleaning up
|
||||
// container". Remove must time out the callback and complete job teardown.
|
||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp(t *testing.T) {
|
||||
// Keep the suite fast: shrink the per-phase teardown timeout for this test.
|
||||
orig := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = orig })
|
||||
|
||||
ctx := common.WithLogger(t.Context(), logging.Default())
|
||||
base := t.TempDir()
|
||||
path := filepath.Join(base, "misc", "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) }) // unblock the leaked goroutine at test end
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
CleanUp: func() {
|
||||
<-release // simulate a delete syscall stuck indefinitely
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck CleanUp callback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval guards against a
|
||||
// stalled os.RemoveAll on the misc/workspace paths (same AV/EDR wedge as
|
||||
// #1023) wedging job completion after the CleanUp callback has already timed
|
||||
// out or finished.
|
||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval(t *testing.T) {
|
||||
origTimeout := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||
|
||||
release := make(chan struct{})
|
||||
stubDone := make(chan struct{})
|
||||
|
||||
origRemoveAll := removeAll
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
// The stuck delete goroutine outlives the timed-out Remove and still reads
|
||||
// removeAll; unblock it and wait before restoring to avoid a restore/read race.
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
removeAll = origRemoveAll
|
||||
})
|
||||
|
||||
ctx := common.WithLogger(t.Context(), logging.Default())
|
||||
base := t.TempDir()
|
||||
path := filepath.Join(base, "misc", "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck path removal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
t.Run("single dir", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
||||
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
||||
// Self-PID guard is essential — without it the script could taskkill
|
||||
// the PowerShell process running it.
|
||||
assert.Contains(t, s, "$selfPid = $PID")
|
||||
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
||||
// Must match both ExecutablePath (binaries from the workspace) and
|
||||
// CommandLine (system binaries invoked with workspace paths in args),
|
||||
// both bounded by dir+separator so a name-prefix sibling is spared.
|
||||
assert.Contains(t, s, `$prefix = $p + '\'`)
|
||||
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
||||
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
||||
// Each matched PID must be tree-killed, not just stopped.
|
||||
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
||||
})
|
||||
|
||||
t.Run("multiple dirs comma-separated", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{
|
||||
`C:\work\path`,
|
||||
`C:\work\workdir`,
|
||||
`C:\Users\runner\AppData\Local\Temp\job-42`,
|
||||
orig := removeAll
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
removeAll = orig
|
||||
})
|
||||
assert.Contains(t, s, `'C:\work\path'`)
|
||||
assert.Contains(t, s, `'C:\work\workdir'`)
|
||||
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
||||
// Commas between entries — no trailing comma, no leading comma.
|
||||
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
||||
})
|
||||
|
||||
t.Run("path with single quote is escaped", func(t *testing.T) {
|
||||
// In PowerShell single-quoted strings the only special char is the
|
||||
// quote itself, escaped by doubling. A workspace path that ever
|
||||
// contained `'` would inject a command into the script otherwise.
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
||||
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
||||
// And it must NOT appear unescaped — otherwise the quote would
|
||||
// terminate the literal early.
|
||||
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
||||
})
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
||||
// A path containing [ ] ? * must be embedded verbatim and matched with
|
||||
// ordinal String methods, not -like, otherwise the metacharacters would
|
||||
// be interpreted as wildcards and the leftover process could escape.
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
||||
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
||||
assert.NotContains(t, s, "-like")
|
||||
assert.Contains(t, s, "StartsWith")
|
||||
assert.Contains(t, s, "IndexOf")
|
||||
})
|
||||
|
||||
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript(nil)
|
||||
// Empty array literal — script runs, matches nothing, is a no-op.
|
||||
assert.Contains(t, s, "$paths = @()")
|
||||
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
||||
err := removeAllWithContext(ctx, t.TempDir())
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
origTimeout, origRemoveAll := hostCleanupTimeout, removeAll
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
hostCleanupTimeout, removeAll = origTimeout, origRemoveAll
|
||||
})
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck path removal")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestJoinPath(t *testing.T) {
|
||||
assert.Equal(t, "//server/share/act/workflow/envs.txt", JoinPath("//server/share/act", "workflow/envs.txt"))
|
||||
assert.Equal(t, "/var/run/act/workflow/envs.txt", JoinPath("/var/run/act/", "workflow", "envs.txt"))
|
||||
for _, root := range []string{"//server/share/act", "/var/run/act", "C:/act"} {
|
||||
assert.Equal(t, root+"/actions/key/main.go.out", JoinPath(root+"/actions/key", "main.go.out"))
|
||||
assert.Equal(t, root+"/workflow", JoinPath(root+"/workflow/outputcmd.txt", ".."))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func TestHostEnvironmentTrackingCleanup(t *testing.T) {
|
||||
for _, tracking := range []string{"default", "", "shared-helper"} {
|
||||
t.Run("tracking="+tracking, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer listener.Close()
|
||||
stop := context.AfterFunc(ctx, func() { _ = listener.Close() })
|
||||
defer stop()
|
||||
workspace := t.TempDir()
|
||||
environment := &HostEnvironment{Path: t.TempDir(), TmpDir: workspace, Workdir: workspace, TrackingID: "gitea_test", StdOut: io.Discard}
|
||||
env := map[string]string{}
|
||||
for _, entry := range os.Environ() {
|
||||
if name, value, ok := strings.Cut(entry, "="); ok && !strings.EqualFold(name, "RUNNER_TRACKING_ID") {
|
||||
env[name] = value
|
||||
}
|
||||
}
|
||||
env["GITEA_TRACKING_HELPER"] = "step"
|
||||
env["GITEA_TRACKING_ADDRESS"] = listener.Addr().String()
|
||||
if tracking == "default" {
|
||||
env["RUNNER_TRACKING_ID"] = environment.TrackingID
|
||||
env["runner_tracking_id"] = "shared-helper"
|
||||
} else {
|
||||
env["runner_tracking_id"] = tracking
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
result <- environment.Exec([]string{os.Args[0], "-test.run=^TestHostTrackingHelper$"}, env, "", workspace)(ctx)
|
||||
}()
|
||||
var helpers []windows.Handle
|
||||
for range 2 {
|
||||
connection, err := listener.Accept()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = connection.Close() })
|
||||
require.NoError(t, connection.SetReadDeadline(time.Now().Add(20*time.Second)))
|
||||
var pid int
|
||||
_, err = fmt.Fscanln(connection, &pid)
|
||||
require.NoError(t, err)
|
||||
handle, err := windows.OpenProcess(windows.SYNCHRONIZE|windows.PROCESS_TERMINATE, false, uint32(pid))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = windows.TerminateProcess(handle, 1)
|
||||
_, _ = windows.WaitForSingleObject(handle, 5000)
|
||||
_ = windows.CloseHandle(handle)
|
||||
})
|
||||
helpers = append(helpers, handle)
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
require.NoError(t, err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
require.NoError(t, environment.Remove()(ctx))
|
||||
for _, handle := range helpers {
|
||||
if tracking == "default" {
|
||||
result, err := windows.WaitForSingleObject(handle, 5000)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, windows.WAIT_OBJECT_0, result)
|
||||
} else {
|
||||
result, err := windows.WaitForSingleObject(handle, 0)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, windows.WAIT_TIMEOUT, result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostTrackingHelper(t *testing.T) {
|
||||
mode := os.Getenv("GITEA_TRACKING_HELPER")
|
||||
if mode == "" {
|
||||
return
|
||||
}
|
||||
if mode != "descendant" {
|
||||
child := exec.Command(os.Args[0], "-test.run=^TestHostTrackingHelper$")
|
||||
child.Env = append(os.Environ(), "GITEA_TRACKING_HELPER="+map[string]string{"step": "helper", "helper": "descendant"}[mode])
|
||||
require.NoError(t, child.Start())
|
||||
if mode == "step" {
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
connection, err := net.Dial("tcp4", os.Getenv("GITEA_TRACKING_ADDRESS"))
|
||||
require.NoError(t, err)
|
||||
_, err = fmt.Fprintln(connection, os.Getpid())
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(io.Discard, connection)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, connection.Close())
|
||||
os.Exit(0)
|
||||
}
|
||||
+123
-13
@@ -7,12 +7,15 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -23,6 +26,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/spf13/pflag"
|
||||
@@ -36,6 +40,7 @@ type KubernetesOptions struct {
|
||||
MaxLifetime time.Duration
|
||||
ForcePull bool
|
||||
RunnerUUID string
|
||||
Transfer *KubernetesTransfer
|
||||
}
|
||||
|
||||
// KubernetesPod runs a job container and its service containers as one pod.
|
||||
@@ -46,6 +51,10 @@ type KubernetesPod struct {
|
||||
job *kubernetesContainer
|
||||
services []*kubernetesContainer
|
||||
arch, path string
|
||||
ips []net.IP
|
||||
transfer *transferServer
|
||||
transferImage string
|
||||
transferHelper string // empty when copies go through the API server
|
||||
}
|
||||
|
||||
type kubernetesContainer struct {
|
||||
@@ -55,6 +64,7 @@ type kubernetesContainer struct {
|
||||
name string
|
||||
input *NewContainerInput
|
||||
probe map[string]any
|
||||
gzip bool // the container can gunzip, so copies into it go compressed
|
||||
}
|
||||
|
||||
type containerStatus struct {
|
||||
@@ -74,6 +84,7 @@ type podStatus struct {
|
||||
Spec struct{ Containers []struct{} }
|
||||
Status struct {
|
||||
Phase, Reason, Message string
|
||||
PodIPs []struct{ IP string }
|
||||
Conditions []struct{ Type, Status, Message string }
|
||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||
}
|
||||
@@ -129,14 +140,24 @@ func (p *KubernetesPod) create(ctx context.Context) error {
|
||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||
}
|
||||
metadata := map[string]any{"name": name, "labels": labels}
|
||||
var runnerImage string
|
||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||
var runner struct{ Metadata struct{ Name, UID string } }
|
||||
var runner struct {
|
||||
Metadata struct{ Name, UID string }
|
||||
Spec struct{ Containers []struct{ Image string } }
|
||||
}
|
||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||
if len(runner.Spec.Containers) == 1 {
|
||||
runnerImage = runner.Spec.Containers[0].Image
|
||||
}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||
}
|
||||
}
|
||||
if err := p.setupTransfer(runnerImage); err != nil {
|
||||
return err
|
||||
}
|
||||
secretData := map[string]string{}
|
||||
manifest, err := p.manifest(name, secretData, metadata)
|
||||
if err != nil {
|
||||
@@ -205,6 +226,18 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
||||
}
|
||||
}
|
||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||
if p.transferHelper != "" {
|
||||
transfer := map[string]any{
|
||||
"name": "gitea-runner-transfer", "image": p.transferImage, "terminationMessagePolicy": "FallbackToLogsOnError",
|
||||
"command": []string{"gitea-runner", KubernetesTransferCommand, "install", p.transferHelper},
|
||||
"volumeMounts": []any{map[string]any{"name": "act", "mountPath": p.job.GetActPath()}},
|
||||
"securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}},
|
||||
}
|
||||
if p.options.ForcePull {
|
||||
transfer["imagePullPolicy"] = "Always"
|
||||
}
|
||||
spec["initContainers"] = []any{transfer}
|
||||
}
|
||||
if len(hostnames) > 0 {
|
||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||
}
|
||||
@@ -341,7 +374,13 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if pod.Status.Phase == "Failed" {
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
||||
reason := cmp.Or(pod.Status.Message, pod.Status.Reason)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
if terminated := status.State.Terminated; reason == "" && terminated != nil && terminated.ExitCode != 0 {
|
||||
reason = fmt.Sprintf("container %s exited with %d: %s", status.Name, terminated.ExitCode, strings.TrimSpace(terminated.Message))
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, reason)
|
||||
}
|
||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
@@ -354,6 +393,9 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
if started {
|
||||
for _, ip := range pod.Status.PodIPs {
|
||||
p.ips = append(p.ips, net.ParseIP(ip.IP))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
@@ -387,16 +429,44 @@ func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
||||
if err := c.pod.waitStarted(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard); err != nil {
|
||||
output, err := c.pod.probe(ctx, c.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
machine, path, _ := strings.Cut(stdout.String(), "\n")
|
||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
||||
probed := strings.SplitN(output, "\n", 3)
|
||||
for len(probed) < 3 {
|
||||
probed = append(probed, "")
|
||||
}
|
||||
c.pod.arch, c.gzip, c.pod.path = goArchToActionArch(probed[0]), probed[1] == "gzip", probed[2]
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// probe retries exec while it fails short of the container, as a new node runs pods before its kubelet has a serving certificate.
|
||||
func (p *KubernetesPod) probe(ctx context.Context, container string) (string, error) {
|
||||
var failure string
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
|
||||
var stdout bytes.Buffer
|
||||
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s\n%s' "$(uname -m)" "$(command -v gzip >/dev/null && echo gzip)" "$PATH"`}, nil, &stdout, io.Discard)
|
||||
var exitErr ExitCodeError
|
||||
var apiErr *apiError
|
||||
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
|
||||
errors.As(err, &apiErr) && apiErr.code < http.StatusInternalServerError {
|
||||
return stdout.String(), err
|
||||
}
|
||||
if err.Error() != failure {
|
||||
failure = err.Error()
|
||||
common.Logger(ctx).Infof("Waiting for exec into pod %s: %v", p.name, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (*kubernetesContainer) Pull(bool) common.Executor {
|
||||
return func(context.Context) error { return nil }
|
||||
}
|
||||
@@ -427,27 +497,67 @@ func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, w
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
||||
// compressCopyMin is the size from which Copy compresses and uses the transfer helper: below it that saves next to nothing on the wire and costs a process in the pod.
|
||||
const compressCopyMin = 64 << 10
|
||||
|
||||
// extract streams the tar write produces into destPath, gzipped with compress, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, compress bool, write func(io.Writer) error) error {
|
||||
if compress && c.pod.transferHelper != "" {
|
||||
request, done := c.pod.transfer.offer(c.pod.ips, write)
|
||||
stdin, _ := json.Marshal(request)
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{c.pod.transferHelper, KubernetesTransferCommand, "fetch", destPath}, bytes.NewReader(stdin), io.Discard, &stderr)
|
||||
taken, writeErr := done()
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
if taken || !c.pod.options.Transfer.Fallback {
|
||||
return fmt.Errorf("copy to %s over the pod network: %w: %s", destPath, errors.Join(writeErr, err), strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
c.pod.transferHelper = "" // a pod that cannot reach the runner once will not later
|
||||
logging.Warnf("Copying into pod %s over the pod network failed, copying through the API server instead: %v: %s", c.pod.name, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
compress = compress && c.gzip
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
go func() { writer.CloseWithError(write(writer)) }()
|
||||
go func() {
|
||||
if compress {
|
||||
writer.CloseWithError(writeGzip(writer, write))
|
||||
} else {
|
||||
writer.CloseWithError(write(writer))
|
||||
}
|
||||
}()
|
||||
script := `mkdir -p "$1" && tar -xf - -C "$1"`
|
||||
if compress {
|
||||
script = `mkdir -p "$1" && gzip -dc | tar -xf - -C "$1"` // gzip -dc rather than tar -z, which GNU tar runs gzip for anyway
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", script, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeGzip gzips what write produces into w, at a level fast enough to not slow copies down and nearly as small as the default.
|
||||
func writeGzip(w io.Writer, write func(io.Writer) error) error {
|
||||
compressed, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||
return errors.Join(write(compressed), compressed.Close())
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
size := 0
|
||||
for _, file := range files {
|
||||
size += len(file.Body)
|
||||
}
|
||||
return c.extract(ctx, destPath, size >= compressCopyMin, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0, hostOS == "windows")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -475,7 +585,7 @@ func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath s
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
return parseEnvFile(c, srcPath, env)
|
||||
return parseEnvFile(c, srcPath, env, false)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
|
||||
@@ -43,8 +43,9 @@ type podManifest struct {
|
||||
}
|
||||
|
||||
type fakeCluster struct {
|
||||
mu sync.Mutex
|
||||
requests map[string][]string
|
||||
mu sync.Mutex
|
||||
requests map[string][]string
|
||||
execFailures []http.HandlerFunc
|
||||
}
|
||||
|
||||
func (f *fakeCluster) record(request, content string) {
|
||||
@@ -53,7 +54,42 @@ func (f *fakeCluster) record(request, content string) {
|
||||
f.requests[request] = append(f.requests[request], content)
|
||||
}
|
||||
|
||||
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
||||
func (f *fakeCluster) failExec(failures ...http.HandlerFunc) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.execFailures = failures
|
||||
}
|
||||
|
||||
func (f *fakeCluster) nextExecFailure() http.HandlerFunc {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if len(f.execFailures) == 0 {
|
||||
return nil
|
||||
}
|
||||
failure := f.execFailures[0]
|
||||
f.execFailures = f.execFailures[1:]
|
||||
return failure
|
||||
}
|
||||
|
||||
// execFailure answers like the API server, code 0 for a failure reported after the websocket upgrade.
|
||||
func execFailure(t *testing.T, code int, status string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if code != 0 {
|
||||
w.WriteHeader(code)
|
||||
_, _ = io.WriteString(w, status)
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.CloseNow() }()
|
||||
_ = conn.Write(r.Context(), websocket.MessageBinary, append([]byte{3}, status...))
|
||||
}
|
||||
}
|
||||
|
||||
func fakeKubernetes(t *testing.T, podStatus string, env ...string) (string, *fakeCluster) {
|
||||
cluster := &fakeCluster{requests: map[string][]string{}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||
@@ -70,7 +106,11 @@ func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
||||
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
|
||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
||||
cluster.record("exec", r.URL.RawQuery)
|
||||
serveLocalExec(t, w, r)
|
||||
if failure := cluster.nextExecFailure(); failure != nil {
|
||||
failure(w, r)
|
||||
} else {
|
||||
serveLocalExec(t, w, r, env)
|
||||
}
|
||||
case strings.HasSuffix(r.URL.Path, "/log"):
|
||||
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
|
||||
@@ -92,7 +132,7 @@ users: [{name: test, user: {tokenFile: token}}]
|
||||
return kubeconfig, cluster
|
||||
}
|
||||
|
||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request, env []string) {
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
@@ -114,7 +154,7 @@ func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
||||
stdin.Write(frame[1:])
|
||||
}
|
||||
cmd := exec.Command(command[0], command[1:]...)
|
||||
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
|
||||
cmd.Env = append(append(os.Environ(), "COPYFILE_DISABLE=1"), env...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
||||
status := `{"status":"Success"}`
|
||||
@@ -143,15 +183,17 @@ func TestKubernetesServiceHostnames(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}},
|
||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
path := workdir + ":" + os.Getenv("PATH")
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}},
|
||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`, "PATH="+path)
|
||||
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
||||
{"spec": map[string]any{
|
||||
"automountServiceAccountToken": true,
|
||||
@@ -212,8 +254,13 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
||||
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
|
||||
cluster.failExec(
|
||||
execFailure(t, http.StatusInternalServerError, `{"message":"error dialing backend: remote error: tls: internal error"}`),
|
||||
execFailure(t, 0, `{"status":"Failure","reason":"InternalError","message":"Internal error occurred: error sending request: remote error: tls: internal error"}`),
|
||||
)
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
assert.Len(t, cluster.requests["exec"], 3)
|
||||
assert.True(t, pod.job.gzip)
|
||||
|
||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||
@@ -230,7 +277,9 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.NotContains(t, query, "hunter2")
|
||||
}
|
||||
|
||||
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||
assert.NotContains(t, lastExec(), "gzip")
|
||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||
require.NoError(t, err)
|
||||
reader := tar.NewReader(archive)
|
||||
@@ -243,6 +292,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
require.NoError(t, archive.Close())
|
||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||
require.Error(t, err)
|
||||
large := strings.Repeat("a", compressCopyMin)
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/large", Mode: 0o644, Body: large})(ctx))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
content, err = os.ReadFile(workdir + "/act/workflow/large")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large, string(content))
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
@@ -251,11 +306,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
assert.Contains(t, lastExec(), "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
||||
assert.Equal(t, path, pathEnv["PATH"])
|
||||
|
||||
info, err := service.Inspect(ctx)
|
||||
require.NoError(t, err)
|
||||
@@ -275,6 +331,38 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||
}
|
||||
|
||||
func TestKubernetesCopiesUncompressedWithoutGzip(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
workdir, bin := t.TempDir(), t.TempDir()
|
||||
for _, tool := range []string{"uname", "tar", "mkdir"} { // all the job container needs but gzip, as on a minimal image
|
||||
found, err := exec.LookPath(tool)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Symlink(found, filepath.Join(bin, tool)))
|
||||
}
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}}]}}`, "PATH="+bin)
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "node", WorkingDir: workdir})
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
assert.False(t, pod.job.gzip)
|
||||
assert.Equal(t, bin, pod.path)
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.NotContains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "gzip")
|
||||
}
|
||||
|
||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||
@@ -295,11 +383,33 @@ func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
}
|
||||
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
|
||||
}
|
||||
running, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[{"name":"job","state":{"running":{}}}]}}`)
|
||||
for _, test := range []struct {
|
||||
failure http.HandlerFunc
|
||||
err string
|
||||
}{
|
||||
{execFailure(t, http.StatusForbidden, `{"message":"pods \"job\" is forbidden"}`), "is forbidden"},
|
||||
{execFailure(t, 0, `{"status":"Failure","details":{"causes":[{"reason":"ExitCode","message":"126"}]}}`), ExitCodeError(126).Error()},
|
||||
} {
|
||||
cluster.failExec(test.failure)
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: running})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), test.err)
|
||||
}
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||
failed, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Failed","initContainerStatuses":[
|
||||
{"name":"gitea-runner-transfer","state":{"terminated":{"exitCode":1,"message":"exec format error\n"}}}]}}`)
|
||||
pod, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: failed})
|
||||
require.NoError(t, err)
|
||||
job = pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "container gitea-runner-transfer exited with 1: exec format error")
|
||||
|
||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
// KubernetesTransfer has job pods fetch large copies from the runner over the pod network instead of through the API server.
|
||||
type KubernetesTransfer struct {
|
||||
Host string // empty uses the outbound IP
|
||||
Port uint16 // 0 picks a free port
|
||||
Image string // has gitea-runner on its PATH, for an init container to copy into each job pod
|
||||
Fallback bool // copy through the API server when a job pod cannot reach the runner, else fail
|
||||
}
|
||||
|
||||
// KubernetesTransferCommand is the hidden runner subcommand job pods run the transfer helper with.
|
||||
const KubernetesTransferCommand = "kubernetes-transfer"
|
||||
|
||||
// transferRequest goes to the helper on the exec stream's stdin, keeping the token out of the command line and the audit log.
|
||||
type transferRequest struct {
|
||||
URL string `json:"url"`
|
||||
Pin string `json:"pin"` // hex SHA-256 of the runner's certificate
|
||||
}
|
||||
|
||||
type transferServer struct {
|
||||
base string
|
||||
pin string
|
||||
mu sync.Mutex
|
||||
pending map[string]transferOffer
|
||||
}
|
||||
|
||||
type transferOffer struct {
|
||||
clients []net.IP
|
||||
write func(io.Writer) error
|
||||
result chan error
|
||||
}
|
||||
|
||||
var (
|
||||
transferServersMu sync.Mutex
|
||||
transferServers = map[string]*transferServer{}
|
||||
)
|
||||
|
||||
// startTransferServer returns the runner's server for these settings, started on first use.
|
||||
func startTransferServer(host string, port uint16) (*transferServer, error) {
|
||||
transferServersMu.Lock()
|
||||
defer transferServersMu.Unlock()
|
||||
configured := net.JoinHostPort(host, strconv.Itoa(int(port)))
|
||||
if server, ok := transferServers[configured]; ok {
|
||||
return server, nil
|
||||
}
|
||||
if host == "" {
|
||||
ip := common.GetOutboundIP()
|
||||
if ip == nil {
|
||||
return nil, errors.New("no outbound IP for job pods to reach the runner at, set kubernetes.transfer.host")
|
||||
}
|
||||
host = ip.String()
|
||||
}
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(1)} // trusted by its pin alone, so no CA, names or validity
|
||||
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port))))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen for job pod transfers: %w", err)
|
||||
}
|
||||
pin := sha256.Sum256(certificate)
|
||||
server := &transferServer{base: "https://" + listener.Addr().String() + "/", pin: hex.EncodeToString(pin[:]), pending: map[string]transferOffer{}}
|
||||
httpServer := &http.Server{
|
||||
Handler: server,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{certificate}, PrivateKey: privateKey}}, MinVersion: tls.VersionTLS13},
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
IdleTimeout: 30 * time.Second,
|
||||
}
|
||||
go func() { _ = httpServer.ServeTLS(listener, "", "") }()
|
||||
transferServers[configured] = server
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// offer serves write once to one of clients until done, which reports whether a client took it and what write failed with.
|
||||
func (s *transferServer) offer(clients []net.IP, write func(io.Writer) error) (request transferRequest, done func() (taken bool, err error)) {
|
||||
token := rand.Text()
|
||||
result := make(chan error, 1)
|
||||
s.mu.Lock()
|
||||
s.pending[token] = transferOffer{clients: clients, write: write, result: result}
|
||||
s.mu.Unlock()
|
||||
return transferRequest{URL: s.base + token, Pin: s.pin}, func() (bool, error) {
|
||||
s.mu.Lock()
|
||||
_, pending := s.pending[token]
|
||||
delete(s.pending, token)
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case err := <-result:
|
||||
return true, err
|
||||
default:
|
||||
return !pending, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *transferServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
token := strings.TrimPrefix(r.URL.Path, "/")
|
||||
s.mu.Lock()
|
||||
offer, ok := s.pending[token]
|
||||
ok = ok && r.Method == http.MethodGet && slices.ContainsFunc(offer.clients, net.ParseIP(host).Equal)
|
||||
if ok {
|
||||
delete(s.pending, token)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
err := writeGzip(w, offer.write)
|
||||
offer.result <- err
|
||||
if err != nil {
|
||||
panic(http.ErrAbortHandler) // a cut-off body fails the fetch, a short tar might not
|
||||
}
|
||||
}
|
||||
|
||||
// FetchKubernetesTransfer reads a transferRequest from request and extracts the tar it points to into dest.
|
||||
func FetchKubernetesTransfer(ctx context.Context, request io.Reader, dest string) error {
|
||||
var transfer transferRequest
|
||||
if err := json.NewDecoder(request).Decode(&transfer); err != nil {
|
||||
return fmt.Errorf("read transfer request: %w", err)
|
||||
}
|
||||
client := &http.Client{Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: time.Minute,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: true, // VerifyConnection checks the pin instead
|
||||
VerifyConnection: func(state tls.ConnectionState) error {
|
||||
if sum := sha256.Sum256(state.PeerCertificates[0].Raw); hex.EncodeToString(sum[:]) != transfer.Pin {
|
||||
return errors.New("runner certificate does not match the pin")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}}
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, transfer.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Do(httpRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("fetch: %s", response.Status)
|
||||
}
|
||||
if err := os.MkdirAll(dest, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", dest)
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = response.Body, os.Stderr, os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(io.Discard, response.Body) // tar stops at the first zero blocks, a cut-off body after them still fails
|
||||
return err
|
||||
}
|
||||
|
||||
// InstallKubernetesTransferHelper copies the running binary to target.
|
||||
func InstallKubernetesTransferHelper(target string) error {
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := os.Open(executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer source.Close()
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(target) // a file left in a reused act volume would keep its mode
|
||||
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, source)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
// setupTransfer turns on transfers for the pod, with the runner pod's image unless one is set.
|
||||
func (p *KubernetesPod) setupTransfer(runnerImage string) error {
|
||||
settings := p.options.Transfer
|
||||
if settings == nil {
|
||||
return nil
|
||||
}
|
||||
image := cmp.Or(settings.Image, runnerImage)
|
||||
var err error
|
||||
var server *transferServer
|
||||
if image == "" {
|
||||
err = errors.New("set kubernetes.transfer.image, the runner is no single-container pod in kubernetes.namespace to take it from")
|
||||
} else {
|
||||
server, err = startTransferServer(settings.Host, settings.Port)
|
||||
}
|
||||
if err != nil {
|
||||
if !settings.Fallback {
|
||||
return fmt.Errorf("kubernetes.transfer: %w", err)
|
||||
}
|
||||
logging.Warnf("Job pod copies go through the API server: %v", err)
|
||||
return nil
|
||||
}
|
||||
p.transfer, p.transferImage = server, image
|
||||
p.transferHelper = path.Join(p.job.GetActPath(), "bin", "gitea-runner")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const transferHelperEnv = "GITEA_RUNNER_TEST_TRANSFER_HELPER"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv(transferHelperEnv) == "1" && len(os.Args) == 4 && os.Args[1] == KubernetesTransferCommand && os.Args[2] == "fetch" {
|
||||
if err := FetchKubernetesTransfer(context.Background(), os.Stdin, os.Args[3]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fetch(t *testing.T, request transferRequest) error {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(request)
|
||||
require.NoError(t, err)
|
||||
return FetchKubernetesTransfer(t.Context(), bytes.NewReader(body), t.TempDir())
|
||||
}
|
||||
|
||||
func TestTransferServer(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("extracts with tar")
|
||||
}
|
||||
server, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
again, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, server, again)
|
||||
loopback := []net.IP{net.ParseIP("127.0.0.1")}
|
||||
nothing := func(io.Writer) error { return nil }
|
||||
|
||||
t.Run("serves an offer gzipped once to an IP of its job pod", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("fd00::1"), loopback[0]}, nothing)
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableCompression: true}}
|
||||
response, err := client.Get(request.URL)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, response.Body.Close())
|
||||
assert.Equal(t, "gzip", response.Header.Get("Content-Encoding"))
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.True(t, taken)
|
||||
})
|
||||
|
||||
t.Run("another pod can neither take nor block an offer", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("10.1.2.3")}, nothing)
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
|
||||
t.Run("a failed write fails the fetch even past the end blocks tar stops at, and done reports it", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, func(w io.Writer) error {
|
||||
_, _ = w.Write(make([]byte, 1024))
|
||||
_, _ = io.CopyN(w, rand.Reader, 1<<20)
|
||||
return errors.New("source gone")
|
||||
})
|
||||
require.Error(t, fetch(t, request))
|
||||
taken, err := done()
|
||||
assert.True(t, taken)
|
||||
require.EqualError(t, err, "source gone")
|
||||
})
|
||||
|
||||
t.Run("the helper refuses a runner whose certificate does not match its pin", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, nothing)
|
||||
request.Pin = strings.Repeat("0", 64)
|
||||
require.ErrorContains(t, fetch(t, request), "does not match the pin")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKubernetesPodTransferImage(t *testing.T) {
|
||||
newPod := func(image string, fallback bool) *KubernetesPod {
|
||||
pod := &KubernetesPod{options: KubernetesOptions{Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: image, Fallback: fallback}}}
|
||||
pod.JobContainer(&NewContainerInput{})
|
||||
return pod
|
||||
}
|
||||
|
||||
pod := newPod("", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "gitea/runner:latest", pod.transferImage)
|
||||
|
||||
pod = newPod("registry.example/runner:1", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "registry.example/runner:1", pod.transferImage)
|
||||
|
||||
pod = newPod("", true)
|
||||
require.NoError(t, pod.setupTransfer(""))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
|
||||
pod = newPod("", false)
|
||||
require.ErrorContains(t, pod.setupTransfer(""), "kubernetes.transfer.image")
|
||||
}
|
||||
|
||||
func TestKubernetesPodCopiesLargeFilesOverPodNetwork(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","podIPs":[{"ip":"127.0.0.1"}],"containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}}]}}`, transferHelperEnv+"=1")
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, ForcePull: true, Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: "gitea/runner:latest", Fallback: true}})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Name: "job", Image: "node", WorkingDir: workdir, Stdout: io.Discard, Stderr: io.Discard})
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
|
||||
var manifest struct {
|
||||
Spec struct {
|
||||
InitContainers []struct {
|
||||
Image, ImagePullPolicy string
|
||||
Command []string
|
||||
VolumeMounts []struct{ Name, MountPath string }
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||
require.Len(t, manifest.Spec.InitContainers, 1)
|
||||
initContainer := manifest.Spec.InitContainers[0]
|
||||
assert.Equal(t, "gitea/runner:latest", initContainer.Image)
|
||||
assert.Equal(t, "Always", initContainer.ImagePullPolicy)
|
||||
assert.Equal(t, []string{"gitea-runner", KubernetesTransferCommand, "install", "/var/run/act/bin/gitea-runner"}, initContainer.Command)
|
||||
assert.Equal(t, []struct{ Name, MountPath string }{{"act", "/var/run/act"}}, initContainer.VolumeMounts)
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
|
||||
helper := filepath.Join(t.TempDir(), "bin", "gitea-runner")
|
||||
require.NoError(t, InstallKubernetesTransferHelper(helper))
|
||||
pod.transferHelper = helper
|
||||
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||
large := &FileEntry{Name: "large", Mode: 0o644, Body: strings.Repeat("a", compressCopyMin)}
|
||||
copyLarge := func(dir string) error { return job.Copy(filepath.Join(workdir, dir)+"/", large)(ctx) }
|
||||
require.NoError(t, copyLarge("transferred"))
|
||||
content, err := os.ReadFile(filepath.Join(workdir, "transferred", "large"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large.Body, string(content))
|
||||
assert.Contains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
assert.NotContains(t, lastExec(), "127.0.0.1")
|
||||
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "event.json", Mode: 0o644, Body: `{}`})(ctx))
|
||||
assert.NotContains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "file"), nil, 0o644))
|
||||
require.ErrorContains(t, copyLarge("file"), "over the pod network")
|
||||
assert.Equal(t, helper, pod.transferHelper)
|
||||
|
||||
unreachable := filepath.Join(workdir, "unreachable")
|
||||
require.NoError(t, os.WriteFile(unreachable, []byte("#!/bin/sh\necho 'connection refused' >&2\nexit 1\n"), 0o755))
|
||||
pod.transferHelper = unreachable
|
||||
pod.options.Transfer.Fallback = false
|
||||
require.ErrorContains(t, copyLarge("direct"), "connection refused")
|
||||
assert.NoFileExists(t, filepath.Join(workdir, "direct", "large"))
|
||||
|
||||
pod.options.Transfer.Fallback = true
|
||||
require.NoError(t, copyLarge("auto"))
|
||||
assert.FileExists(t, filepath.Join(workdir, "auto", "large"))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
}
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
|
||||
type LinuxContainerEnvironmentExtensions struct{}
|
||||
|
||||
var windowsPathRegex = regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
||||
|
||||
// Resolves the equivalent host path inside the container
|
||||
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
||||
// For use in docker volumes and binds
|
||||
@@ -31,8 +33,6 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
|
||||
return ""
|
||||
}
|
||||
|
||||
// Test if the path is a windows path
|
||||
windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
||||
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
|
||||
|
||||
// Return as-is if no match
|
||||
@@ -43,11 +43,7 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
|
||||
// Convert to WSL2-compatible path if it is a windows path
|
||||
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
|
||||
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
|
||||
driveLetter := strings.ToLower(windowsPathComponents[1])
|
||||
translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
||||
// Should make something like /mnt/c/Users/person/My Folder/MyActProject
|
||||
result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
|
||||
return result
|
||||
return "/mnt/" + strings.ToLower(windowsPathComponents[1]) + "/" + strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
||||
}
|
||||
|
||||
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
||||
@@ -69,10 +65,41 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
|
||||
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
|
||||
const DefaultToolCache = "/opt/hostedtoolcache"
|
||||
|
||||
type runnerArchKey struct{}
|
||||
|
||||
func WithRunnerArch(ctx context.Context) context.Context { // shares one architecture lookup per run
|
||||
if ctx.Value(runnerArchKey{}) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, runnerArchKey{}, &runnerArchCache{lock: make(chan struct{}, 1)})
|
||||
}
|
||||
|
||||
type runnerArchCache struct {
|
||||
lock chan struct{} // lets cancelled jobs stop waiting for the cache
|
||||
arch string
|
||||
}
|
||||
|
||||
func cachedRunnerArch(ctx context.Context) string {
|
||||
cache, ok := ctx.Value(runnerArchKey{}).(*runnerArchCache)
|
||||
if !ok {
|
||||
return RunnerArch(ctx)
|
||||
}
|
||||
select {
|
||||
case cache.lock <- struct{}{}:
|
||||
defer func() { <-cache.lock }()
|
||||
case <-ctx.Done():
|
||||
return ""
|
||||
}
|
||||
if cache.arch == "" {
|
||||
cache.arch = RunnerArch(ctx)
|
||||
}
|
||||
return cache.arch
|
||||
}
|
||||
|
||||
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
|
||||
return map[string]any{
|
||||
"os": "Linux",
|
||||
"arch": RunnerArch(ctx),
|
||||
"arch": cachedRunnerArch(ctx),
|
||||
"temp": "/tmp",
|
||||
"tool_cache": DefaultToolCache,
|
||||
}
|
||||
|
||||
@@ -5,15 +5,13 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestContainerPath(t *testing.T) {
|
||||
@@ -23,42 +21,27 @@ func TestContainerPath(t *testing.T) {
|
||||
workDir string
|
||||
}
|
||||
|
||||
linuxcontainerext := &LinuxContainerEnvironmentExtensions{}
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
tests := []containerPathJob{
|
||||
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
||||
{"/home/act", `/home/act/`, ""},
|
||||
{cwd, ".", ""},
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
logging.Error(err)
|
||||
}
|
||||
|
||||
rootDrive := os.Getenv("SystemDrive")
|
||||
rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
|
||||
for _, v := range []containerPathJob{
|
||||
tests = []containerPathJob{
|
||||
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
|
||||
{"/mnt/f/work/dir", `F:\work\dir`, ""},
|
||||
{"/mnt/c/windows/to/unix", "windows\\to\\unix", rootDrive + "\\"},
|
||||
{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", rootDrive + "\\"},
|
||||
} {
|
||||
if v.workDir != "" {
|
||||
t.Chdir(v.workDir)
|
||||
}
|
||||
|
||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
||||
{"/mnt/" + strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + "/act", "act", rootDrive + "\\"},
|
||||
}
|
||||
|
||||
t.Chdir(cwd)
|
||||
} else {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
logging.Error(err)
|
||||
}
|
||||
for _, v := range []containerPathJob{
|
||||
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
||||
{"/home/act", `/home/act/`, ""},
|
||||
{cwd, ".", ""},
|
||||
} {
|
||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
||||
}
|
||||
for _, test := range tests {
|
||||
if test.workDir != "" {
|
||||
t.Chdir(test.workDir)
|
||||
}
|
||||
assert.Equal(t, test.destinationPath, (&LinuxContainerEnvironmentExtensions{}).ToContainerPath(test.sourcePath))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,5 +50,4 @@ type typeAssertMockContainer struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
}
|
||||
|
||||
// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &typeAssertMockContainer{}
|
||||
@@ -7,6 +7,7 @@ package container
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -18,8 +19,14 @@ import (
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor {
|
||||
localEnv := *env
|
||||
// parseEnvFile reads a GITHUB_ENV style file, splitting lines like GitHub's runner on Windows or elsewhere.
|
||||
func parseEnvFile(e Container, srcPath string, env *map[string]string, windows bool) common.Executor {
|
||||
return ParseEnvFile(e, srcPath, func(name, value string) {
|
||||
(*env)[name] = value
|
||||
}, windows)
|
||||
}
|
||||
|
||||
func ParseEnvFile(e Container, srcPath string, setEnv func(name, value string), windows bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
||||
if err != nil {
|
||||
@@ -38,8 +45,22 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
||||
s := bufio.NewScanner(decoded)
|
||||
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
|
||||
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||
scanned := 0
|
||||
s.Split(func(data []byte, atEOF bool) (int, []byte, error) {
|
||||
if index := bytes.IndexByte(data[scanned:], '\n'); index >= 0 {
|
||||
advance := scanned + index + 1
|
||||
scanned = 0
|
||||
return advance, data[:advance], nil
|
||||
}
|
||||
scanned = len(data)
|
||||
if atEOF && len(data) > 0 {
|
||||
scanned = 0
|
||||
return len(data), data, nil
|
||||
}
|
||||
return 0, nil, nil
|
||||
})
|
||||
for s.Scan() {
|
||||
line := s.Text()
|
||||
line := trimEnvLineEnding(s.Text(), windows)
|
||||
// GitHub's runner ignores blank lines
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
@@ -48,14 +69,17 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
||||
multiLineEnv := strings.Index(line, "<<")
|
||||
switch {
|
||||
case singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv):
|
||||
localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:]
|
||||
setEnv(line[:singleLineEnv], line[singleLineEnv+1:])
|
||||
case multiLineEnv != -1:
|
||||
var multiLineEnvContent []string
|
||||
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
||||
if multiLineEnv == 0 || multiLineEnvDelimiter == "" {
|
||||
return fmt.Errorf("invalid format '%v', name and delimiter must not be empty", line)
|
||||
}
|
||||
delimiterFound := false
|
||||
for s.Scan() {
|
||||
content := s.Text()
|
||||
if content == multiLineEnvDelimiter {
|
||||
if trimEnvLineEnding(content, windows) == multiLineEnvDelimiter {
|
||||
delimiterFound = true
|
||||
break
|
||||
}
|
||||
@@ -67,7 +91,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
||||
if !delimiterFound {
|
||||
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
||||
}
|
||||
localEnv[line[:multiLineEnv]] = strings.Join(multiLineEnvContent, "\n")
|
||||
setEnv(line[:multiLineEnv], trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows))
|
||||
default:
|
||||
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
||||
}
|
||||
@@ -75,7 +99,13 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
||||
if err := s.Err(); err != nil {
|
||||
return fmt.Errorf("reading env file: %w", err)
|
||||
}
|
||||
env = &localEnv
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func trimEnvLineEnding(line string, windows bool) string {
|
||||
if windows && strings.HasSuffix(line, "\r\n") {
|
||||
return line[:len(line)-2]
|
||||
}
|
||||
return strings.TrimSuffix(line, "\n")
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -12,127 +13,100 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
)
|
||||
|
||||
func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
||||
t.Helper()
|
||||
e := &HostEnvironment{Path: t.TempDir()}
|
||||
return e, filepath.Join(e.Path, "envfile")
|
||||
}
|
||||
|
||||
func TestParseEnvFileSingleLine(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\nBAZ=qux\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "qux", env["BAZ"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileMultiLine(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content := "FOO<<EOF\nline1\nline2\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "line1\nline2", env["FOO"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileLargeValueWithinLimit(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
big := strings.Repeat("x", 2*1024*1024)
|
||||
content := "FOO<<EOF\n" + big + "\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, big, env["FOO"])
|
||||
func TestParseEnvFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||
require.NoError(t, err)
|
||||
return string(content)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name, content string
|
||||
want map[string]string
|
||||
}{
|
||||
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
{"repeated names", "FOO=first\nfoo=middle\nFOO=last\n", map[string]string{"FOO": "last", "foo": "middle"}},
|
||||
{"single line CRLF and final bare CR", "FOO=bar\r\nBAZ=qux\r", map[string]string{"FOO": "bar\r", "BAZ": "qux\r"}},
|
||||
{"single line without final newline", "FOO=bar", map[string]string{"FOO": "bar"}},
|
||||
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
|
||||
{"multiline without final newline", "FOO<<EOF\nline1\r\nline2\r\nEOF", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||
{"empty multiline without final newline", "FOO<<EOF\nEOF", map[string]string{"FOO": ""}},
|
||||
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
|
||||
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
|
||||
{"multiline CRLF", "FOO<<EOF\r\nline1\r\nline2\r\nEOF\r\n", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||
{"multiline mixed line endings", "FOO<<EOF\nline1\r\nline2\n\r\nEOF\n", map[string]string{"FOO": "line1\r\nline2\n\r"}},
|
||||
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
|
||||
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
|
||||
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
envPath := filepath.Join(hostEnv.Path, test.name)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, false)(t.Context()))
|
||||
assert.Equal(t, test.want, env)
|
||||
})
|
||||
}
|
||||
t.Run("ordered entries preserve repeated names", func(t *testing.T) {
|
||||
envPath := filepath.Join(hostEnv.Path, "ordered")
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=first\r\nfoo<<EOF\r\nmiddle\r\nEOF\r\nFOO=last\r\n"), 0o600))
|
||||
var entries [][2]string
|
||||
require.NoError(t, ParseEnvFile(hostEnv, envPath, func(name, value string) {
|
||||
entries = append(entries, [2]string{name, value})
|
||||
}, true)(t.Context()))
|
||||
assert.Equal(t, [][2]string{
|
||||
{"FOO", "first"}, {"foo", "middle"}, {"FOO", "last"},
|
||||
}, entries)
|
||||
})
|
||||
t.Run("Windows line endings", func(t *testing.T) {
|
||||
envPath := filepath.Join(hostEnv.Path, "windows")
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
||||
assert.Equal(t, map[string]string{"FOO": "bar", "MULTI": "line1\r\nline2"}, env)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
tooBig := strings.Repeat("x", 17*1024*1024) // over the 16 MiB cap
|
||||
content := "FOO<<EOF\n" + tooBig + "\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
t.Parallel()
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
envPath := filepath.Join(hostEnv.Path, "envfile")
|
||||
const header, footer = "FOO<<EOF\n", "\nEOF\n"
|
||||
content := bytes.Repeat([]byte("x"), len(header)+17*1024*1024+len(footer))
|
||||
copy(content, header)
|
||||
copy(content[len(content)-len(footer):], footer)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
||||
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||
assert.Contains(t, err.Error(), "reading env file")
|
||||
}
|
||||
|
||||
// Regression test: a blank line used to fail the job at "Complete Job", after
|
||||
// every step had already been recorded as successful.
|
||||
func TestParseEnvFileBlankLines(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("\nFOO=bar\n\n \nBAZ=qux\n\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "qux", env["BAZ"])
|
||||
}
|
||||
|
||||
// blank lines inside a heredoc value are content, not separators
|
||||
func TestParseEnvFileMultiLineKeepsBlankLines(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\n\nline2\nEOF\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "line1\n\nline2", env["FOO"])
|
||||
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n\nline2\nEOF\n"), 0o600))
|
||||
env = map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "\nline2", env["FOO"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileUTF8BOM(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content := append([]byte{0xEF, 0xBB, 0xBF}, []byte("FOO=bar\n")...)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
}
|
||||
|
||||
// Windows host mode: PowerShell 5.1 redirection writes UTF-16, which used to be
|
||||
// unrecognisable as KEY=VALUE, so the writes were silently ignored.
|
||||
func TestParseEnvFileUTF16(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
encoder *encoding.Encoder
|
||||
}{
|
||||
{"little endian", unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder()},
|
||||
{"big endian", unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder()},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content, err := tt.encoder.Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
for name, content := range map[string]string{
|
||||
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
||||
"empty name": "<<EOF\nline1\nEOF\n",
|
||||
"empty delimiter": "FOO<<\nline1\n\n",
|
||||
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
envPath := filepath.Join(hostEnv.Path, name)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("APPLIED=before-error\n"+content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "line1", env["MULTI"])
|
||||
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "delimiter")
|
||||
assert.Equal(t, map[string]string{"APPLIED": "before-error"}, env)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvFileMissingDelimiter(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "delimiter")
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*Fil
|
||||
}
|
||||
|
||||
// writeDirTar prefixes entries with dstPath for extraction at root.
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int, executable bool) error {
|
||||
logger := common.Logger(ctx)
|
||||
tw := tar.NewWriter(w)
|
||||
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
||||
@@ -47,7 +47,7 @@ func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useG
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
UseGitIgnore: useGitIgnore,
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:], Executable: executable},
|
||||
}).CollectFiles(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
-1
@@ -1 +0,0 @@
|
||||
testfile
|
||||
@@ -27,23 +27,25 @@ type Handler interface {
|
||||
}
|
||||
|
||||
type TarCollector struct {
|
||||
TarWriter *tar.Writer
|
||||
UID int
|
||||
GID int
|
||||
DstDir string
|
||||
TarWriter *tar.Writer
|
||||
UID int
|
||||
GID int
|
||||
DstDir string
|
||||
Executable bool // Windows has no exec bit, so mark everything executable like docker build
|
||||
}
|
||||
|
||||
func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error {
|
||||
// create a new dir/file header
|
||||
header, err := tar.FileInfoHeader(fi, linkName)
|
||||
header, err := tar.FileInfoHeader(fi, filepath.ToSlash(linkName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update the name to correctly reflect the desired destination when untaring
|
||||
header.Name = path.Join(tc.DstDir, fpath)
|
||||
header.Mode = int64(fi.Mode())
|
||||
header.ModTime = fi.ModTime()
|
||||
if tc.Executable {
|
||||
header.Mode = header.Mode&0o755 | 0o111
|
||||
}
|
||||
header.Uid = tc.UID
|
||||
header.Gid = tc.GID
|
||||
|
||||
@@ -58,10 +60,8 @@ func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
|
||||
}
|
||||
|
||||
// copy file data into tar writer
|
||||
if _, err := io.Copy(tc.TarWriter, f); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err = io.Copy(tc.TarWriter, f)
|
||||
return err
|
||||
}
|
||||
|
||||
type CopyCollector struct {
|
||||
@@ -87,10 +87,8 @@ func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string
|
||||
return err
|
||||
}
|
||||
defer df.Close()
|
||||
if _, err := io.Copy(df, f); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err = io.Copy(df, f)
|
||||
return err
|
||||
}
|
||||
|
||||
type FileCollector struct {
|
||||
|
||||
@@ -28,10 +28,12 @@ func gitCmd(t *testing.T, args ...string) {
|
||||
}
|
||||
|
||||
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
||||
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
|
||||
gitCmd(t, "init", "-q", dir)
|
||||
repoDir := t.TempDir()
|
||||
gitCmd(t, "init", "-q", "--template=", repoDir)
|
||||
for _, dir := range []string{"sub", "nested"} {
|
||||
require.NoError(t, os.CopyFS(filepath.Join(repoDir, dir, ".git"), os.DirFS(filepath.Join(repoDir, ".git"))))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "ign"), 0o755))
|
||||
for name, body := range map[string]string{
|
||||
".gitignore": ".*\nign/\n",
|
||||
".env": "",
|
||||
@@ -43,17 +45,18 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
"sub/kept": "",
|
||||
"nested/kept": "",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
||||
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
||||
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
||||
for _, gitlink := range []string{"sub", "uninitialized"} {
|
||||
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
|
||||
}
|
||||
gitCmd(t, "-C", repoDir, "update-index", "--add",
|
||||
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,sub",
|
||||
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,uninitialized",
|
||||
)
|
||||
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Chmod(filepath.Join(repoDir, "renamed"), 0o777|os.ModeSticky))
|
||||
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
||||
expected["test.env"] = ".env"
|
||||
}
|
||||
@@ -69,6 +72,9 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
for header, err := tr.Next(); err != io.EOF; header, err = tr.Next() {
|
||||
require.NoError(t, err)
|
||||
files[header.Name] = header.Linkname
|
||||
if header.Name == "renamed" && runtime.GOOS != "windows" {
|
||||
assert.Equal(t, int64(0o1777), header.Mode)
|
||||
}
|
||||
}
|
||||
assert.Contains(t, files, ".git/HEAD")
|
||||
maps.DeleteFunc(files, func(name, _ string) bool { return strings.Contains("/"+name, "/.git/") })
|
||||
|
||||
@@ -28,12 +28,12 @@ var (
|
||||
|
||||
type pinnedKey struct{}
|
||||
|
||||
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
|
||||
func WithPinnedCheckout(ctx context.Context) context.Context {
|
||||
// WithPinnedCheckout keeps a run's sha, ref and repository fixed when HEAD moves, in pins unless ctx has some.
|
||||
func WithPinnedCheckout(ctx context.Context, pins *sync.Map) context.Context {
|
||||
if ctx.Value(pinnedKey{}) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
|
||||
return context.WithValue(ctx, pinnedKey{}, pins)
|
||||
}
|
||||
|
||||
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
||||
@@ -57,12 +57,29 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
case "pull_request_target":
|
||||
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||
case "pull_request", "pull_request_review", "pull_request_review_comment":
|
||||
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", ghc.Event["number"])
|
||||
number := ghc.Event["number"]
|
||||
if number == nil {
|
||||
number = model.NestedMapLookup(ghc.Event, "pull_request", "number")
|
||||
}
|
||||
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", number)
|
||||
if ghc.EventName == "pull_request" && ghc.Event["action"] == "closed" && model.NestedMapLookup(ghc.Event, "pull_request", "merged") == true {
|
||||
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||
}
|
||||
case "deployment", "deployment_status":
|
||||
ghc.Ref = model.AsString(model.NestedMapLookup(ghc.Event, "deployment", "ref"))
|
||||
case "release":
|
||||
ghc.Ref = "refs/tags/" + model.AsString(model.NestedMapLookup(ghc.Event, "release", "tag_name"))
|
||||
case "push", "create", "workflow_dispatch":
|
||||
case "create":
|
||||
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||
if ghc.Ref != "" && !strings.HasPrefix(ghc.Ref, "refs/") {
|
||||
switch model.AsString(ghc.Event["ref_type"]) {
|
||||
case "branch":
|
||||
ghc.Ref = "refs/heads/" + ghc.Ref
|
||||
case "tag":
|
||||
ghc.Ref = "refs/tags/" + ghc.Ref
|
||||
}
|
||||
}
|
||||
case "push", "workflow_dispatch":
|
||||
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||
default:
|
||||
defaultBranch := model.AsString(model.NestedMapLookup(ghc.Event, "repository", "default_branch"))
|
||||
@@ -117,8 +134,7 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
|
||||
if ghc.Sha == "" {
|
||||
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
|
||||
_, revision, err := findGitRevision(ctx, repoPath)
|
||||
return revision, err
|
||||
return findGitRevision(ctx, repoPath)
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf("unable to get git revision: %v", err)
|
||||
@@ -132,12 +148,12 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
// remote in repoPath when it is not set yet, and derives its owner.
|
||||
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
if ghc.Repository == "" {
|
||||
repo, err := findGithubRepo(ctx, repoPath)
|
||||
repo, err := pinned(ctx, "repo:"+repoPath, func() (string, error) { return findGithubRepo(ctx, repoPath) })
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
|
||||
return
|
||||
}
|
||||
ghc.Repository = repo
|
||||
}
|
||||
ghc.RepositoryOwner = strings.Split(ghc.Repository, "/")[0]
|
||||
ghc.RepositoryOwner, _, _ = strings.Cut(ghc.Repository, "/")
|
||||
}
|
||||
@@ -7,30 +7,21 @@ package ghcontext
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSetRef(t *testing.T) {
|
||||
logging.SetLevel(logging.LevelDebug)
|
||||
|
||||
oldFindGitRef := findGitRef
|
||||
oldFindGitRevision := findGitRevision
|
||||
defer func() { findGitRef = oldFindGitRef }()
|
||||
defer func() { findGitRevision = oldFindGitRevision }()
|
||||
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "refs/heads/master", nil
|
||||
}
|
||||
|
||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
||||
return "", "1234fakesha", nil
|
||||
}
|
||||
|
||||
tables := []struct {
|
||||
eventName string
|
||||
event map[string]any
|
||||
@@ -51,6 +42,30 @@ func TestSetRef(t *testing.T) {
|
||||
ref: "refs/pull/1234/merge",
|
||||
refName: "1234/merge",
|
||||
},
|
||||
{
|
||||
eventName: "pull_request_review",
|
||||
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||
ref: "refs/pull/1234/merge",
|
||||
refName: "1234/merge",
|
||||
},
|
||||
{
|
||||
eventName: "pull_request",
|
||||
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": true}},
|
||||
ref: "refs/heads/master",
|
||||
refName: "master",
|
||||
},
|
||||
{
|
||||
eventName: "pull_request",
|
||||
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": false}},
|
||||
ref: "refs/pull/1234/merge",
|
||||
refName: "1234/merge",
|
||||
},
|
||||
{
|
||||
eventName: "pull_request_review_comment",
|
||||
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||
ref: "refs/pull/1234/merge",
|
||||
refName: "1234/merge",
|
||||
},
|
||||
{
|
||||
eventName: "deployment",
|
||||
event: map[string]any{
|
||||
@@ -79,6 +94,24 @@ func TestSetRef(t *testing.T) {
|
||||
ref: "refs/heads/somebranch",
|
||||
refName: "somebranch",
|
||||
},
|
||||
{
|
||||
eventName: "create",
|
||||
event: map[string]any{"ref": "feature/new", "ref_type": "branch"},
|
||||
ref: "refs/heads/feature/new",
|
||||
refName: "feature/new",
|
||||
},
|
||||
{
|
||||
eventName: "create",
|
||||
event: map[string]any{"ref": "v1", "ref_type": "tag"},
|
||||
ref: "refs/tags/v1",
|
||||
refName: "v1",
|
||||
},
|
||||
{
|
||||
eventName: "create",
|
||||
event: map[string]any{"ref": "refs/tags/v1", "ref_type": "tag"},
|
||||
ref: "refs/tags/v1",
|
||||
refName: "v1",
|
||||
},
|
||||
{
|
||||
eventName: "unknown",
|
||||
event: map[string]any{
|
||||
@@ -114,7 +147,7 @@ func TestSetRef(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Run("no-default-branch", func(t *testing.T) {
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "", errors.New("no default branch")
|
||||
}
|
||||
|
||||
@@ -130,19 +163,15 @@ func TestSetRef(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSetSha(t *testing.T) {
|
||||
logging.SetLevel(logging.LevelDebug)
|
||||
|
||||
oldFindGitRef := findGitRef
|
||||
oldFindGitRevision := findGitRevision
|
||||
oldFindGithubRepo := findGithubRepo
|
||||
defer func() { findGitRef = oldFindGitRef }()
|
||||
defer func() { findGitRevision = oldFindGitRevision }()
|
||||
defer func() { findGithubRepo = oldFindGithubRepo }()
|
||||
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
return "refs/heads/master", nil
|
||||
}
|
||||
|
||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
||||
return "", "1234fakesha", nil
|
||||
findGitRevision = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "1234fakesha", nil
|
||||
}
|
||||
|
||||
tables := []struct {
|
||||
@@ -177,11 +206,6 @@ func TestSetSha(t *testing.T) {
|
||||
},
|
||||
sha: "deployment-sha",
|
||||
},
|
||||
{
|
||||
eventName: "release",
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
{
|
||||
eventName: "push",
|
||||
event: map[string]any{
|
||||
@@ -195,18 +219,12 @@ func TestSetSha(t *testing.T) {
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
{
|
||||
eventName: "no-event",
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
t.Run(table.eventName, func(t *testing.T) {
|
||||
ghc := &model.GithubContext{
|
||||
EventName: table.eventName,
|
||||
BaseRef: "master",
|
||||
Event: table.event,
|
||||
}
|
||||
|
||||
@@ -216,16 +234,19 @@ func TestSetSha(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
|
||||
ctx := WithPinnedCheckout(t.Context())
|
||||
t.Run("pinned checkout keeps the first sha, ref and repository", func(t *testing.T) {
|
||||
ctx := WithPinnedCheckout(t.Context(), &sync.Map{})
|
||||
for _, head := range []string{"first", "moved"} {
|
||||
findGitRevision = func(context.Context, string) (string, string, error) { return "", head, nil }
|
||||
findGitRevision = func(context.Context, string) (string, error) { return head, nil }
|
||||
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
||||
findGithubRepo = func(context.Context, string) (string, error) { return "owner/" + head, nil }
|
||||
ghc := &model.GithubContext{Event: map[string]any{}}
|
||||
SetSha(ctx, ghc, "/some/dir")
|
||||
SetRef(ctx, ghc, "/some/dir")
|
||||
SetRepositoryAndOwner(ctx, ghc, "/some/dir")
|
||||
assert.Equal(t, "first", ghc.Sha)
|
||||
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
||||
assert.Equal(t, "owner/first", ghc.Repository)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func lookPathWindows(file string, env map[string]string, lookup func(string, map[string]string) (string, error)) (string, error) {
|
||||
found, err := lookup(file, env)
|
||||
isBash := strings.EqualFold(file, "bash") || strings.EqualFold(file, "bash.exe")
|
||||
if !isBash || err == nil && !strings.EqualFold(filepath.Base(filepath.Dir(found)), "System32") { // System32's bash is the WSL launcher, Git's bash replaces it
|
||||
return found, err
|
||||
}
|
||||
if git, gitErr := lookup("git.exe", env); gitErr == nil {
|
||||
for _, relative := range []string{"../bin/bash.exe", "../usr/bin/bash.exe", "bash.exe"} {
|
||||
if bash, bashErr := lookup(filepath.Join(filepath.Dir(git), relative), env); bashErr == nil {
|
||||
return bash, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLookPathWindowsFallsBackToGitBash(t *testing.T) {
|
||||
root := "C:" + string(filepath.Separator)
|
||||
system32 := filepath.Join(root, "Windows", "System32")
|
||||
gitCmd, gitBin := filepath.Join(root, "Git", "cmd"), filepath.Join(root, "Git", "bin")
|
||||
msys := filepath.Join(root, "msys64", "usr", "bin")
|
||||
for _, test := range []struct {
|
||||
name, command, want string
|
||||
path []string
|
||||
}{
|
||||
{"wsl launcher first", "bash", filepath.Join(gitBin, "bash.exe"), []string{system32, gitCmd}},
|
||||
{"no bash on path", "BASH.EXE", filepath.Join(gitBin, "bash.exe"), []string{gitCmd}},
|
||||
{"own bash first", "bash", filepath.Join(msys, "bash.exe"), []string{msys, system32, gitCmd}},
|
||||
{"no git", "bash", filepath.Join(system32, "bash.exe"), []string{system32}},
|
||||
{"other command", "sh", filepath.Join(system32, "sh.exe"), []string{system32, gitCmd}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
files := map[string]bool{
|
||||
filepath.Join(system32, "bash.exe"): true, filepath.Join(system32, "sh.exe"): true,
|
||||
filepath.Join(gitCmd, "git.exe"): true, filepath.Join(gitBin, "bash.exe"): true, filepath.Join(msys, "bash.exe"): true,
|
||||
}
|
||||
lookup := func(file string, env map[string]string) (string, error) {
|
||||
candidates := []string{file}
|
||||
if !strings.ContainsAny(file, `:\/`) {
|
||||
candidates = nil
|
||||
for directory := range strings.SplitSeq(env["PATH"], ";") {
|
||||
candidates = append(candidates, filepath.Join(directory, strings.TrimSuffix(strings.ToLower(file), ".exe")+".exe"))
|
||||
}
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if files[filepath.Clean(candidate)] {
|
||||
return filepath.Clean(candidate), nil
|
||||
}
|
||||
}
|
||||
return "", fs.ErrNotExist
|
||||
}
|
||||
got, err := lookPathWindows(test.command, map[string]string{"PATH": strings.Join(test.path, ";")}, lookup)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,35 +13,23 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLookPath2SearchesPathAndEmptyElement(t *testing.T) {
|
||||
func TestLookPath2SearchesPathAndEmptyElementOrUsesDirectPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
exe := filepath.Join(dir, "tool")
|
||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := LookPath2("tool", map[string]string{"PATH": string(filepath.ListSeparator) + dir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookPath2DirectPathDoesNotSearchPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
exe := filepath.Join(dir, "tool")
|
||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := LookPath2(exe, map[string]string{"PATH": ""})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
for file, searchPath := range map[string]string{"tool": string(filepath.ListSeparator) + dir, exe: ""} {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
got, err := LookPath2(file, map[string]string{"PATH": searchPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,21 +28,15 @@ func chkStat(file string) error {
|
||||
}
|
||||
|
||||
func hasExt(file string) bool {
|
||||
i := strings.LastIndex(file, ".")
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
return strings.LastIndexAny(file, `:\/`) < i
|
||||
return strings.LastIndex(file, ".") > strings.LastIndexAny(file, `:\/`)
|
||||
}
|
||||
|
||||
func findExecutable(file string, exts []string) (string, error) {
|
||||
if len(exts) == 0 {
|
||||
return file, chkStat(file)
|
||||
}
|
||||
if hasExt(file) {
|
||||
if chkStat(file) == nil {
|
||||
return file, nil
|
||||
}
|
||||
if hasExt(file) && chkStat(file) == nil {
|
||||
return file, nil
|
||||
}
|
||||
for _, e := range exts {
|
||||
if f := file + e; chkStat(f) == nil {
|
||||
@@ -59,6 +53,10 @@ func findExecutable(file string, exts []string) (string, error) {
|
||||
// a suitable candidate.
|
||||
// The result may be an absolute path or a path relative to the current directory.
|
||||
func LookPath2(file string, env map[string]string) (string, error) {
|
||||
return lookPathWindows(file, env, lookPath)
|
||||
}
|
||||
|
||||
func lookPath(file string, env map[string]string) (string, error) {
|
||||
var exts []string
|
||||
x := getenv(env, `PATHEXT`)
|
||||
if x != "" {
|
||||
@@ -82,9 +80,6 @@ func LookPath2(file string, env map[string]string) (string, error) {
|
||||
return "", &Error{file, err}
|
||||
}
|
||||
}
|
||||
if f, err := findExecutable(filepath.Join(".", file), exts); err == nil {
|
||||
return f, nil
|
||||
}
|
||||
path := getenv(env, "path")
|
||||
for _, dir := range filepath.SplitList(path) {
|
||||
if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lookpath
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLookPath2IgnoresTheCurrentDirectory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "tool.exe"), nil, 0o755))
|
||||
t.Chdir(dir)
|
||||
_, err := LookPath2("tool", map[string]string{"PATH": t.TempDir()})
|
||||
require.EqualError(t, err, ErrNotFound.Error())
|
||||
}
|
||||
+159
-116
@@ -8,7 +8,6 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -20,12 +19,15 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
_ "embed"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/kballard/go-shellquote"
|
||||
)
|
||||
@@ -47,11 +49,13 @@ type fileWriter func(filename string, data []byte, perm fs.FileMode) error
|
||||
type runAction func(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor
|
||||
|
||||
//go:embed res/trampoline.js
|
||||
var trampoline embed.FS
|
||||
var trampoline string
|
||||
|
||||
var (
|
||||
ContainerImageExistsLocally = container.ImageExistsLocally
|
||||
ContainerNewDockerBuildExecutor = container.NewDockerBuildExecutor
|
||||
dockerActionNamePattern = regexp.MustCompile("[^a-zA-Z0-9]")
|
||||
inputEnvPattern = regexp.MustCompile("[^A-Z0-9-]")
|
||||
)
|
||||
|
||||
func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) {
|
||||
@@ -71,52 +75,52 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
||||
reader, closer, err = readFile("action.yaml")
|
||||
addError("action.yaml", err)
|
||||
if os.IsNotExist(err) {
|
||||
_, closer, err := readFile("Dockerfile")
|
||||
addError("Dockerfile", err)
|
||||
dockerfile := "Dockerfile"
|
||||
_, closer, err := readFile(dockerfile)
|
||||
addError(dockerfile, err)
|
||||
if os.IsNotExist(err) {
|
||||
dockerfile = "dockerfile"
|
||||
_, closer, err = readFile(dockerfile)
|
||||
addError(dockerfile, err)
|
||||
}
|
||||
if err == nil {
|
||||
closer.Close()
|
||||
action := &model.Action{
|
||||
Name: "(Synthetic)",
|
||||
Runs: model.ActionRuns{
|
||||
Using: "docker",
|
||||
Image: "Dockerfile",
|
||||
Image: dockerfile,
|
||||
},
|
||||
}
|
||||
logger.Debugf("Using synthetic action %v for Dockerfile", action)
|
||||
logger.Debugf("Using synthetic action %v for %s", action, dockerfile)
|
||||
return action, nil
|
||||
}
|
||||
if step.With != nil {
|
||||
if val, ok := step.With["args"]; ok {
|
||||
var b []byte
|
||||
if b, err = trampoline.ReadFile("res/trampoline.js"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err2 := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), b, 0o644)
|
||||
if err2 != nil {
|
||||
return nil, err2
|
||||
}
|
||||
action := &model.Action{
|
||||
Name: "(Synthetic)",
|
||||
Inputs: map[string]model.Input{
|
||||
"cwd": {
|
||||
Description: "(Actual working directory)",
|
||||
Required: false,
|
||||
Default: filepath.Join(actionDir, actionPath),
|
||||
},
|
||||
"command": {
|
||||
Description: "(Actual program)",
|
||||
Required: false,
|
||||
Default: val,
|
||||
},
|
||||
},
|
||||
Runs: model.ActionRuns{
|
||||
Using: "node12",
|
||||
Main: "trampoline.js",
|
||||
},
|
||||
}
|
||||
logger.Debugf("Using synthetic action %v", action)
|
||||
return action, nil
|
||||
if val, ok := step.With["args"]; ok {
|
||||
if err := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), []byte(trampoline), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
action := &model.Action{
|
||||
Name: "(Synthetic)",
|
||||
Inputs: map[string]model.Input{
|
||||
"args": {},
|
||||
"cwd": {
|
||||
Description: "(Actual working directory)",
|
||||
Required: false,
|
||||
Default: filepath.Join(actionDir, actionPath),
|
||||
},
|
||||
"command": {
|
||||
Description: "(Actual program)",
|
||||
Required: false,
|
||||
Default: val,
|
||||
},
|
||||
},
|
||||
Runs: model.ActionRuns{
|
||||
Using: "node12",
|
||||
Main: "trampoline.js",
|
||||
},
|
||||
}
|
||||
logger.Debugf("Using synthetic action %v", action)
|
||||
return action, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -125,8 +129,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
||||
}
|
||||
defer closer.Close()
|
||||
|
||||
action, err := model.ReadAction(reader)
|
||||
return action, err
|
||||
return model.ReadAction(reader)
|
||||
}
|
||||
|
||||
func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actionPath, containerActionDir string) error {
|
||||
@@ -151,12 +154,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
if !rc.Config.NoActionPatch {
|
||||
// Other jobs share the directory, so patch under the copy's lock.
|
||||
defer patchActions(ctx, actionScriptPaths(filepath.Join(actionDir, actionPath), step.getActionModel()))()
|
||||
if err := rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx); err != nil { // an exported commit has no .git to apply gitignore with
|
||||
return err
|
||||
}
|
||||
|
||||
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx) // an exported commit has no .git to apply gitignore with
|
||||
if rc.Config.NoActionPatch {
|
||||
return nil
|
||||
}
|
||||
patched := patchedScripts(ctx, actionDir, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))
|
||||
if len(patched) == 0 {
|
||||
return nil
|
||||
}
|
||||
return rc.JobContainer.Copy(containerActionDirCopy, patched...)(ctx)
|
||||
}
|
||||
|
||||
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
|
||||
@@ -166,19 +174,20 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
actionPath := ""
|
||||
if remoteAction != nil && remoteAction.Path != "" {
|
||||
if remoteAction != nil {
|
||||
actionPath = remoteAction.Path
|
||||
}
|
||||
|
||||
action := step.getActionModel()
|
||||
|
||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *step.getEnv())
|
||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *step.getEnv())
|
||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
warnUnexpectedInputs(ctx, action, stepModel.With)
|
||||
printRunActionHeader(ctx, stepModel, action, *step.getEnv(), rc)
|
||||
|
||||
actionLocation := path.Join(actionDir, actionPath)
|
||||
actionName, containerActionDir := getContainerActionPaths(step)
|
||||
|
||||
logger.Debugf("type=%v actionDir=%s actionPath=%s workdir=%s actionCacheDir=%s actionName=%s containerActionDir=%s", stepModel.Type(), actionDir, actionPath, rc.Config.Workdir, rc.ActionCacheDir(), actionName, containerActionDir)
|
||||
@@ -189,14 +198,14 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil {
|
||||
return err
|
||||
}
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Main))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Main))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
return rc.JobContainer.Exec(containerArgs, *step.getEnv(), "", "")(ctx)
|
||||
case x.IsDocker():
|
||||
location := actionLocation
|
||||
location := path.Join(actionDir, actionPath)
|
||||
if remoteAction == nil {
|
||||
location = containerActionDir
|
||||
}
|
||||
@@ -215,12 +224,9 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Main + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Main}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Main}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return fmt.Errorf("the runs.using key must be one of: %v, got %s", []string{
|
||||
@@ -259,16 +265,14 @@ func dockerActionImageTag(repository, actionName string, localAction bool) strin
|
||||
name = path.Join(repository, actionName)
|
||||
}
|
||||
// The human-readable name is sanitized by collapsing every non-alphanumeric character to "-".
|
||||
sanitized := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(name, "-")
|
||||
sanitized := dockerActionNamePattern.ReplaceAllString(name, "-")
|
||||
if localAction {
|
||||
// For local actions a short hash of the raw repository and action path is appended so the tag stays unique per repository.
|
||||
sum := sha256.Sum256([]byte(repository + "\x00" + actionName))
|
||||
sanitized += "-" + hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
// "-dockeraction" ensures that "./", "./test " won't get converted to "act-:latest", "act-test-:latest" which are invalid docker image names
|
||||
image := fmt.Sprintf("%s-dockeraction:%s", sanitized, "latest")
|
||||
image = "act-" + strings.TrimLeft(image, "-")
|
||||
return strings.ToLower(image)
|
||||
return strings.ToLower("act-" + strings.TrimLeft(sanitized+"-dockeraction:latest", "-"))
|
||||
}
|
||||
|
||||
// One image per step and job: its stages share a build, and concurrent jobs never run each other's.
|
||||
@@ -279,7 +283,7 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
||||
}
|
||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||
if !ok {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
image = dockerActionImageTag(getStepGithubContext(ctx, step).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||
job.dockerActionImages[step.getStepModel()] = image
|
||||
}
|
||||
@@ -291,6 +295,9 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
||||
func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, basedir string, localAction bool, stage stepStage) error {
|
||||
logger := common.Logger(ctx)
|
||||
rc := step.getRunContext()
|
||||
if rc.isWindowsHost() {
|
||||
return errWindowsHostContainer
|
||||
}
|
||||
action := step.getActionModel()
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
@@ -312,9 +319,12 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
return err
|
||||
}
|
||||
|
||||
correctArchExists, err := ContainerImageExistsLocally(ctx, image, rc.Config.ContainerArchitecture)
|
||||
if err != nil {
|
||||
return err
|
||||
correctArchExists := anyArchExists
|
||||
if anyArchExists && rc.Config.ContainerArchitecture != "" && rc.Config.ContainerArchitecture != "any" {
|
||||
correctArchExists, err = ContainerImageExistsLocally(ctx, image, rc.Config.ContainerArchitecture)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if anyArchExists && !correctArchExists {
|
||||
@@ -329,46 +339,39 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
|
||||
if !correctArchExists || rc.Config.ForceRebuild {
|
||||
logger.Debugf("image '%s' for architecture '%s' will be built from context '%s", image, rc.Config.ContainerArchitecture, contextDir)
|
||||
var buildContext io.ReadCloser
|
||||
if localAction {
|
||||
buildContext, err = rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
||||
if err != nil {
|
||||
return err
|
||||
prepImage = common.Executor(func(ctx context.Context) error {
|
||||
input := container.NewDockerBuildExecutorInput{
|
||||
ContextDir: contextDir,
|
||||
Dockerfile: fileName,
|
||||
ImageTag: image,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
BuildArgs: rc.proxyBuildArgs(),
|
||||
}
|
||||
defer buildContext.Close()
|
||||
}
|
||||
prepImage = ContainerNewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
|
||||
ContextDir: contextDir,
|
||||
Dockerfile: fileName,
|
||||
ImageTag: image,
|
||||
BuildContext: buildContext,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
BuildArgs: rc.proxyBuildArgs(),
|
||||
})
|
||||
if buildContext == nil {
|
||||
// Held across the whole build: the daemon drains contextDir lazily.
|
||||
inner := prepImage
|
||||
prepImage = func(ctx context.Context) error {
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir)
|
||||
if localAction {
|
||||
buildContext, err := rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer buildContext.Close()
|
||||
input.BuildContext = buildContext
|
||||
} else {
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir) // held across the whole build, the daemon drains contextDir lazily
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
return inner(ctx)
|
||||
}
|
||||
}
|
||||
return ContainerNewDockerBuildExecutor(input)(ctx)
|
||||
}).Retry(3)
|
||||
} else {
|
||||
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
||||
}
|
||||
}
|
||||
cmd, err := shellquote.Split(step.getStepModel().With["args"])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ee, err := evalDockerEnv(ctx, step, action)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var cmd []string
|
||||
if action.Runs.Args != nil {
|
||||
// a fresh slice, the manifest is evaluated again for every stage
|
||||
cmd = make([]string, len(action.Runs.Args))
|
||||
@@ -377,6 +380,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
return fmt.Errorf("unable to interpolate runs.args: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
cmd, err = shellquote.Split(stepInput(step.getStepModel(), "args"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
entrypoint, err := dockerEntrypoint(step, stage)
|
||||
if err != nil {
|
||||
@@ -406,7 +414,7 @@ func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
|
||||
default:
|
||||
entrypoint = runs.Entrypoint
|
||||
if entrypoint == "" {
|
||||
if fields := strings.Fields(step.getStepModel().With["entrypoint"]); len(fields) > 0 {
|
||||
if fields := strings.Fields(stepInput(step.getStepModel(), "entrypoint")); len(fields) > 0 {
|
||||
return fields, nil
|
||||
}
|
||||
}
|
||||
@@ -424,9 +432,15 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
|
||||
var err error
|
||||
inputs := make(map[string]string)
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
// Set Defaults
|
||||
providedInputs := make(map[string]bool, len(step.getStepModel().With))
|
||||
for key := range step.getStepModel().With {
|
||||
providedInputs[strings.ToLower(key)] = true
|
||||
}
|
||||
eval := rc.NewStepExpressionEvaluator(ctx, step)
|
||||
for k, input := range action.Inputs {
|
||||
if providedInputs[strings.ToLower(k)] {
|
||||
continue
|
||||
}
|
||||
if inputs[k], err = eval.Interpolate(ctx, input.Default); err != nil {
|
||||
return nil, fmt.Errorf("unable to interpolate the default of input %s: %w", k, err)
|
||||
}
|
||||
@@ -441,6 +455,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
}
|
||||
}
|
||||
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
||||
aliasGiteaEnv(*step.getEnv())
|
||||
return ee, nil
|
||||
}
|
||||
|
||||
@@ -485,20 +500,20 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
|
||||
}
|
||||
|
||||
func populateEnvsFromSavedState(env *map[string]string, step actionStep, rc *RunContext) {
|
||||
state, ok := rc.IntraActionState[step.getStepModel().ID]
|
||||
if ok {
|
||||
for name, value := range state {
|
||||
envName := "STATE_" + name
|
||||
(*env)[envName] = value
|
||||
}
|
||||
for name, value := range rc.IntraActionState[step.getStepModel().ID] {
|
||||
(*env)["STATE_"+name] = value
|
||||
}
|
||||
}
|
||||
|
||||
func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *model.Action, rc *RunContext) error {
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
func inputEnvKey(inputID string) string {
|
||||
return "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
}
|
||||
|
||||
func populateEnvsFromInput(ctx context.Context, step step, action *model.Action) error {
|
||||
env := step.getEnv()
|
||||
eval := step.getRunContext().NewStepExpressionEvaluator(ctx, step)
|
||||
for inputID, input := range action.Inputs {
|
||||
envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
envKey = "INPUT_" + envKey
|
||||
envKey := inputEnvKey(inputID)
|
||||
if _, ok := (*env)[envKey]; !ok {
|
||||
var err error
|
||||
if (*env)[envKey], err = eval.Interpolate(ctx, input.Default); err != nil {
|
||||
@@ -509,6 +524,33 @@ func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *
|
||||
return nil
|
||||
}
|
||||
|
||||
func stepInput(step *model.Step, name string) string {
|
||||
value, _ := common.LookupFold(step.With, name, true)
|
||||
return value
|
||||
}
|
||||
|
||||
func warnUnexpectedInputs(ctx context.Context, action *model.Action, inputs map[string]string) {
|
||||
docker := action.Runs.Using.IsDocker()
|
||||
var unexpectedInputs []string
|
||||
for input := range inputs {
|
||||
_, declared := common.LookupFold(action.Inputs, input, true)
|
||||
containerInput := docker && (strings.EqualFold(input, "entryPoint") || strings.EqualFold(input, "args"))
|
||||
if !declared && !containerInput {
|
||||
unexpectedInputs = append(unexpectedInputs, input)
|
||||
}
|
||||
}
|
||||
if len(unexpectedInputs) == 0 {
|
||||
return
|
||||
}
|
||||
validInputs := slices.Sorted(maps.Keys(action.Inputs))
|
||||
if docker {
|
||||
validInputs = append([]string{"entryPoint", "args"}, validInputs...)
|
||||
}
|
||||
slices.Sort(unexpectedInputs)
|
||||
common.Logger(ctx).With(rawOutputField, true).Warnf("##[warning]%s", EscapeCommandData(fmt.Sprintf(
|
||||
"Unexpected input(s) '%s', valid inputs are ['%s']", strings.Join(unexpectedInputs, "', '"), strings.Join(validInputs, "', '"))))
|
||||
}
|
||||
|
||||
func getContainerActionPaths(step actionStep) (string, string) {
|
||||
rc := step.getRunContext()
|
||||
if sar, ok := step.(*stepActionRemote); ok { // a copy per action, as its repository's directory is shared
|
||||
@@ -597,17 +639,18 @@ func runPreStep(step actionStep) common.Executor {
|
||||
x := action.Runs.Using
|
||||
if !x.IsComposite() {
|
||||
// defaults in pre steps were missing, however provided inputs are available
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
printRunActionHeader(ctx, step.getStepModel(), action, *step.getEnv(), rc)
|
||||
switch {
|
||||
case x.IsNode():
|
||||
if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Pre))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Pre))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
@@ -637,12 +680,9 @@ func runPreStep(step actionStep) common.Executor {
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Pre + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Pre}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Pre}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return nil
|
||||
@@ -652,8 +692,11 @@ func runPreStep(step actionStep) common.Executor {
|
||||
|
||||
func shouldRunPostStep(step actionStep) common.Conditional {
|
||||
return func(ctx context.Context) bool {
|
||||
if remote, ok := step.(*stepActionRemote); ok && remote.preStepStarted && !remote.action.Runs.Using.IsComposite() {
|
||||
return true
|
||||
}
|
||||
log := common.Logger(ctx)
|
||||
stepResults := step.getRunContext().getStepsContext()
|
||||
stepResults := step.getRunContext().StepResults
|
||||
stepResult := stepResults[step.getStepModel().ID]
|
||||
|
||||
if stepResult == nil {
|
||||
@@ -699,12 +742,15 @@ func runPostStep(step actionStep) common.Executor {
|
||||
actionDir, actionPath, _, containerActionDir := actionStagePaths(step)
|
||||
|
||||
x := action.Runs.Using
|
||||
if !x.IsComposite() { // a composite's children print their own
|
||||
logger.With(rawOutputField, true).Info("Post job cleanup.")
|
||||
}
|
||||
switch {
|
||||
case x.IsNode():
|
||||
|
||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||
|
||||
containerArgs := nodeActionCommand(path.Join(containerActionDir, action.Runs.Post))
|
||||
containerArgs := nodeActionCommand(container.JoinPath(containerActionDir, action.Runs.Post))
|
||||
logger.Debugf("executing remote job container: %s", containerArgs)
|
||||
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
@@ -731,12 +777,9 @@ func runPostStep(step actionStep) common.Executor {
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Post + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Post}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Post}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{container.JoinPath(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
|
||||
default:
|
||||
|
||||
Loaded 100 of 272 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user