mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 14:55:44 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
|||||||
|
|
||||||
/bin
|
/bin
|
||||||
/dist
|
/dist
|
||||||
|
/.tmp
|
||||||
|
/.git/hooks
|
||||||
|
/.git/logs
|
||||||
|
/.git/rr-cache
|
||||||
/.env
|
/.env
|
||||||
/.runner
|
/.runner
|
||||||
|
/.runner.lock
|
||||||
/config.yaml
|
/config.yaml
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: go-setup
|
||||||
|
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||||
|
id: setup-go
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: false
|
||||||
|
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||||
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||||
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: cache-seeder
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- go.mod
|
||||||
|
- go.sum
|
||||||
|
- .golangci.yml
|
||||||
|
- Makefile
|
||||||
|
- .gitea/actions/go-setup/action.yml
|
||||||
|
- .gitea/workflows/cache-seeder.yml
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
seed:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- run: make deps-tools lint build
|
||||||
|
- name: compile the test binaries without running them
|
||||||
|
run: |
|
||||||
|
make test GOTEST_FLAGS='-race -exec true'
|
||||||
|
go test -tags e2e -exec true ./e2e/...
|
||||||
@@ -21,7 +21,7 @@ jobs:
|
|||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 26
|
||||||
- run: make lint-pr-title
|
- run: make lint-pr-title
|
||||||
env:
|
env:
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release --nightly
|
args: release --nightly
|
||||||
@@ -79,7 +79,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
passphrase: ${{ secrets.PASSPHRASE }}
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release
|
args: release
|
||||||
@@ -82,7 +82,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
+11
-15
@@ -17,25 +17,24 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||||
- name: pre-pull test images
|
- name: pre-pull test images
|
||||||
env:
|
env:
|
||||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:d6aa754f16b3197301076f047b5def2f02ea1dbbc2ca920407d46d7ec7f87b20 # renovate: datasource=docker
|
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||||
run: |
|
run: |
|
||||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||||
for attempt in 1 2 3; do
|
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||||
docker pull "$image" && break
|
for attempt in 1 2 3; do
|
||||||
[ "$attempt" = 3 ] || sleep 5
|
docker pull "$image" && break
|
||||||
done
|
[ "$attempt" = 3 ] || sleep 5
|
||||||
|
done
|
||||||
|
fi
|
||||||
docker tag "$image" "${image%@*}"
|
docker tag "$image" "${image%@*}"
|
||||||
done
|
done
|
||||||
- name: lint
|
- name: lint
|
||||||
@@ -68,10 +67,7 @@ jobs:
|
|||||||
- gitea/gitea:main-nightly
|
- gitea/gitea:main-nightly
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
- name: e2e compatibility
|
- name: e2e compatibility
|
||||||
|
|||||||
+10
-2
@@ -57,7 +57,15 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
|||||||
|
|
||||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||||
parallel.
|
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||||
|
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||||
|
|
||||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||||
|
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||||
|
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||||
|
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||||
|
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||||
|
Docker and Kubernetes.
|
||||||
|
|
||||||
|
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||||
+5
-3
@@ -12,12 +12,14 @@ ENV GOPROXY=${GOPROXY:-}
|
|||||||
COPY . /opt/src/runner
|
COPY . /opt/src/runner
|
||||||
WORKDIR /opt/src/runner
|
WORKDIR /opt/src/runner
|
||||||
|
|
||||||
RUN make clean && make build
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
make clean && make build
|
||||||
|
|
||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.1-dind AS dind
|
FROM docker:29.9.2-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -39,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ GO ?= go
|
|||||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||||
|
|
||||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||||
@@ -15,8 +15,8 @@ DOCKER_IMAGE ?= gitea/runner
|
|||||||
DOCKER_TAG ?= nightly
|
DOCKER_TAG ?= nightly
|
||||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||||
|
|
||||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||||
|
|
||||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||||
|
|
||||||
@@ -154,7 +154,7 @@ E2E_CONCURRENCY ?= 8
|
|||||||
|
|
||||||
.PHONY: test-e2e
|
.PHONY: test-e2e
|
||||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||||
|
|
||||||
.PHONY: install
|
.PHONY: install
|
||||||
install: $(GOFILES) ## install the runner binary via `go install`
|
install: $(GOFILES) ## install the runner binary via `go install`
|
||||||
|
|||||||
@@ -303,6 +303,8 @@ Use `uses: builtin:checkout` to check out a repository without downloading `acti
|
|||||||
|
|
||||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||||
|
|
||||||
|
As on GitHub, a job restores caches saved for its own ref, its pull request's base branch and the default branch, so a cache saved by a pull request is only restored by that pull request.
|
||||||
|
|
||||||
**Sharing a cache between runners**
|
**Sharing a cache between runners**
|
||||||
|
|
||||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||||
|
|||||||
@@ -6,7 +6,5 @@
|
|||||||
//
|
//
|
||||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||||
//
|
//
|
||||||
// TODO: Authorization
|
|
||||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
|
||||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||||
package artifactcache
|
package artifactcache
|
||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
@@ -58,6 +59,10 @@ type credKey struct{}
|
|||||||
type JobCredential struct {
|
type JobCredential struct {
|
||||||
Repo string `json:"repo"`
|
Repo string `json:"repo"`
|
||||||
|
|
||||||
|
Ref string `json:"ref"`
|
||||||
|
BaseRef string `json:"base_ref"`
|
||||||
|
DefaultRef string `json:"default_ref"`
|
||||||
|
|
||||||
// Results is the instance whose artifact service this server forwards for the job, and
|
// Results is the instance whose artifact service this server forwards for the job, and
|
||||||
// InsecureTLS how the runner reaches it; see results.go.
|
// InsecureTLS how the runner reaches it; see results.go.
|
||||||
Results string `json:"-"`
|
Results string `json:"-"`
|
||||||
@@ -67,6 +72,10 @@ type JobCredential struct {
|
|||||||
PublicURL string `json:"public_url"`
|
PublicURL string `json:"public_url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cred JobCredential) owns(cache *Cache) bool {
|
||||||
|
return cache.Repo == cred.Repo && cache.Ref == cred.Ref
|
||||||
|
}
|
||||||
|
|
||||||
// credEntry holds a registered job's credential along with an active
|
// credEntry holds a registered job's credential along with an active
|
||||||
// registration count. RegisterJob is reference-counted so that if two tasks
|
// registration count. RegisterJob is reference-counted so that if two tasks
|
||||||
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
||||||
@@ -652,7 +661,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, keys, version)
|
cache, err := h.lookupCache(db, cred, keys, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -668,26 +677,27 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
|
// lookupCache returns the entry to restore or (nil, nil), dropping matches whose blob was pruned.
|
||||||
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
|
func (h *Handler) lookupCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||||
// entry is dropped on the way out.
|
for {
|
||||||
func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
cache, err := findCache(db, cred, keys, version)
|
||||||
cache, err := findCache(db, repo, keys, version)
|
if err != nil || cache == nil {
|
||||||
if err != nil || cache == nil {
|
return nil, err
|
||||||
return nil, err
|
}
|
||||||
|
ok, err := h.storage.Exist(cache.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||||
|
// this call and the GET that follows it.
|
||||||
|
h.touch(db, cache)
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
|
if !h.deleteCache(db, cache) {
|
||||||
|
return nil, nil //nolint:nilnil // absence is not an error here
|
||||||
|
}
|
||||||
}
|
}
|
||||||
ok, err := h.storage.Exist(cache.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
h.deleteCache(db, cache)
|
|
||||||
return nil, nil //nolint:nilnil // absence is not an error here
|
|
||||||
}
|
|
||||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
|
||||||
// this call and the GET that follows it.
|
|
||||||
h.touch(db, cache)
|
|
||||||
return cache, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// POST /_apis/artifactcache/caches
|
// POST /_apis/artifactcache/caches
|
||||||
@@ -699,7 +709,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size}
|
cache := &Cache{Repo: cred.Repo, Ref: cred.Ref, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||||
if cache.Size == 0 {
|
if cache.Size == 0 {
|
||||||
cache.Size = -1
|
cache.Size = -1
|
||||||
}
|
}
|
||||||
@@ -710,9 +720,10 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache.CreatedAt = now
|
cache.CreatedAt = now.Unix()
|
||||||
cache.UsedAt = now
|
cache.CreatedAtNanos = now.Nanosecond()
|
||||||
|
cache.UsedAt = now.Unix()
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -747,7 +758,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cache.Repo != cred.Repo {
|
if !cred.owns(cache) {
|
||||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -778,6 +789,17 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, 400, err)
|
h.responseJSON(w, r, 400, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var request struct {
|
||||||
|
Size *int64 `json:"size"`
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||||
|
if err == nil && len(body) > 0 {
|
||||||
|
err = json.Unmarshal(body, &request)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
db, err := h.openDB()
|
db, err := h.openDB()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -795,7 +817,7 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cache.Repo != cred.Repo {
|
if !cred.owns(cache) {
|
||||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -805,6 +827,14 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.Size != nil {
|
||||||
|
if *request.Size < 0 || cache.Size >= 0 && cache.Size != *request.Size {
|
||||||
|
h.responseJSON(w, r, http.StatusBadRequest, errors.New("invalid cache size"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cache.Size = *request.Size
|
||||||
|
}
|
||||||
|
|
||||||
db.Close()
|
db.Close()
|
||||||
|
|
||||||
if err := h.commitCache(cache); err != nil {
|
if err := h.commitCache(cache); err != nil {
|
||||||
@@ -1050,25 +1080,35 @@ func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time
|
|||||||
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if not found, return (nil, nil) instead of an error.
|
// findCache tries every key in one ref before the next ref, and returns (nil, nil) on a miss.
|
||||||
func findCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
func findCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||||
for _, prefix := range keys {
|
refs := []string{cred.Ref}
|
||||||
// if a key in the list matches exactly, don't return partial matches
|
for _, ref := range []string{cred.BaseRef, cred.DefaultRef} {
|
||||||
exact, err := findExactCache(db, repo, prefix, version, true)
|
if ref != "" && !slices.Contains(refs, ref) {
|
||||||
if err != nil {
|
refs = append(refs, ref)
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
if exact != nil {
|
}
|
||||||
return exact, nil
|
caches, err := db.find(func(cache *Cache) bool {
|
||||||
}
|
return cache.Repo == cred.Repo && cache.Version == version && cache.Complete && slices.Contains(refs, cache.Ref) &&
|
||||||
caches, err := db.find(func(cache *Cache) bool {
|
slices.ContainsFunc(keys, func(prefix string) bool { return strings.HasPrefix(cache.Key, prefix) })
|
||||||
return cache.Repo == repo && strings.HasPrefix(cache.Key, prefix) && cache.Version == version && cache.Complete
|
})
|
||||||
})
|
if err != nil {
|
||||||
if err != nil {
|
return nil, fmt.Errorf("find cache: %w", err)
|
||||||
return nil, fmt.Errorf("find cache: %w", err)
|
}
|
||||||
}
|
for _, ref := range refs {
|
||||||
if cache := latest(caches, func(cache *Cache) int64 { return cache.CreatedAt }); cache != nil {
|
for _, prefix := range keys {
|
||||||
return cache, nil
|
var exact, partial []*Cache
|
||||||
|
for _, cache := range caches {
|
||||||
|
if cache.Ref == ref && cache.Key == prefix {
|
||||||
|
exact = append(exact, cache)
|
||||||
|
} else if cache.Ref == ref && strings.HasPrefix(cache.Key, prefix) {
|
||||||
|
partial = append(partial, cache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// if a key in the list matches exactly, don't return partial matches
|
||||||
|
if cache := cmp.Or(latest(exact, (*Cache).createdAtNano), latest(partial, (*Cache).createdAtNano)); cache != nil {
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
||||||
@@ -1078,7 +1118,9 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
|||||||
if len(caches) == 0 {
|
if len(caches) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return slices.MaxFunc(caches, func(left, right *Cache) int { return cmp.Compare(stamp(left), stamp(right)) })
|
return slices.MaxFunc(caches, func(left, right *Cache) int {
|
||||||
|
return cmp.Or(cmp.Compare(stamp(left), stamp(right)), cmp.Compare(left.ID, right.ID))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
|
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
|
||||||
@@ -1088,16 +1130,16 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
|||||||
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
||||||
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
||||||
// touches UsedAt on every part.
|
// touches UsedAt on every part.
|
||||||
func findExactCache(db *cacheDB, repo, key, version string, complete bool) (*Cache, error) {
|
func findExactCache(db *cacheDB, cred JobCredential, key, version string, complete bool) (*Cache, error) {
|
||||||
caches, err := db.find(func(cache *Cache) bool {
|
caches, err := db.find(func(cache *Cache) bool {
|
||||||
return cache.Key == key && cache.Repo == repo && cache.Version == version && cache.Complete == complete
|
return cred.owns(cache) && cache.Key == key && cache.Version == version && cache.Complete == complete
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("find cache: %w", err)
|
return nil, fmt.Errorf("find cache: %w", err)
|
||||||
}
|
}
|
||||||
stamp := func(cache *Cache) int64 { return cache.UsedAt }
|
stamp := func(cache *Cache) int64 { return cache.UsedAt }
|
||||||
if complete {
|
if complete {
|
||||||
stamp = func(cache *Cache) int64 { return cache.CreatedAt }
|
stamp = (*Cache).createdAtNano
|
||||||
}
|
}
|
||||||
return latest(caches, stamp), nil
|
return latest(caches, stamp), nil
|
||||||
}
|
}
|
||||||
@@ -1257,17 +1299,18 @@ func (h *Handler) evictExpired(db *cacheDB) {
|
|||||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictSuperseded removes entries a newer one with the same key and version replaced. The
|
// evictSuperseded removes entries a newer one with the same repo, ref, key and version replaced.
|
||||||
// grouping includes Repo so two repos sharing a (key, version) do not evict each other.
|
|
||||||
func (h *Handler) evictSuperseded(db *cacheDB) {
|
func (h *Handler) evictSuperseded(db *cacheDB) {
|
||||||
type entry struct{ repo, key, version string }
|
type entry struct{ repo, ref, key, version string }
|
||||||
groups := make(map[entry][]*Cache)
|
groups := make(map[entry][]*Cache)
|
||||||
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
||||||
group := entry{cache.Repo, cache.Key, cache.Version}
|
group := entry{cache.Repo, cache.Ref, cache.Key, cache.Version}
|
||||||
groups[group] = append(groups[group], cache)
|
groups[group] = append(groups[group], cache)
|
||||||
}
|
}
|
||||||
for _, caches := range groups {
|
for _, caches := range groups {
|
||||||
slices.SortFunc(caches, func(left, right *Cache) int { return cmp.Compare(left.CreatedAt, right.CreatedAt) })
|
slices.SortFunc(caches, func(left, right *Cache) int {
|
||||||
|
return cmp.Or(cmp.Compare(left.createdAtNano(), right.createdAtNano()), cmp.Compare(left.ID, right.ID))
|
||||||
|
})
|
||||||
for _, cache := range caches[:len(caches)-1] {
|
for _, cache := range caches[:len(caches)-1] {
|
||||||
if inUse(cache) {
|
if inUse(cache) {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
@@ -47,10 +48,12 @@ func signArtifactURL(h *Handler, id int64) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ExternalURL(t *testing.T) {
|
func TestHandler_ExternalURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
|
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_Upstream(t *testing.T) {
|
func TestHandler_Upstream(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var upstream *httptest.Server
|
var upstream *httptest.Server
|
||||||
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
assert.Equal(t, http.MethodPut, r.Method)
|
assert.Equal(t, http.MethodPut, r.Method)
|
||||||
@@ -100,6 +103,7 @@ func TestHandler_Upstream(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler(t *testing.T) {
|
func TestHandler(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
const version = "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
const version = "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
@@ -302,6 +306,37 @@ func TestHandler(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("chunked commit validates size omitted at reservation", func(t *testing.T) {
|
||||||
|
for _, size := range []int64{5, 10} {
|
||||||
|
t.Run(strconv.FormatInt(size, 10), func(t *testing.T) {
|
||||||
|
id := reserveCache(t, testClient, base, strings.ToLower(t.Name()), version, 0)
|
||||||
|
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), strings.NewReader("cache"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
req.Header.Set("Content-Range", "bytes 0-4/*")
|
||||||
|
resp, err := testClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
resp, err = testClient.Post(fmt.Sprintf("%s/caches/%d", base, id), "application/json", io.MultiReader(strings.NewReader(fmt.Sprintf(`{"size":%d}`, size))))
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
if size == 5 {
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
} else {
|
||||||
|
require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
cache, err := db.get(id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, size == 5, cache.Complete)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("get with bad id", func(t *testing.T) {
|
t.Run("get with bad id", func(t *testing.T) {
|
||||||
resp, err := testClient.Get(base + "/artifacts/invalid_id")
|
resp, err := testClient.Get(base + "/artifacts/invalid_id")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -438,6 +473,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_gcCache(t *testing.T) {
|
func TestHandler_gcCache(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir, Policy: Policy{Retention: testRetention}})
|
handler, err := StartHandler(Options{Dir: dir, Policy: Policy{Retention: testRetention}})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -540,6 +576,7 @@ func TestHandler_gcCache(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_evictPolicy(t *testing.T) {
|
func TestHandler_evictPolicy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
stale := func(d time.Duration) int64 { return now.Add(-d).Unix() }
|
stale := func(d time.Duration) int64 { return now.Add(-d).Unix() }
|
||||||
mib := func(n int64) int64 { return n * miB }
|
mib := func(n int64) int64 { return n * miB }
|
||||||
@@ -637,6 +674,7 @@ func TestHandler_evictPolicy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_evictForFreeSpace(t *testing.T) {
|
func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
free := func(n int64) func(string) (uint64, error) {
|
free := func(n int64) func(string) (uint64, error) {
|
||||||
return func(string) (uint64, error) { return uint64(n), nil }
|
return func(string) (uint64, error) { return uint64(n), nil }
|
||||||
}
|
}
|
||||||
@@ -676,6 +714,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
cache := &Cache{Key: "stuck", Complete: true, UsedAt: time.Now().Add(-(testRetention + time.Hour)).Unix()}
|
cache := &Cache{Key: "stuck", Complete: true, UsedAt: time.Now().Add(-(testRetention + time.Hour)).Unix()}
|
||||||
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
|
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
|
||||||
|
|
||||||
@@ -690,6 +729,7 @@ func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
wanted := &Cache{Repo: testRepo, Key: "wanted", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-3 * time.Hour).Unix()}
|
wanted := &Cache{Repo: testRepo, Key: "wanted", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-3 * time.Hour).Unix()}
|
||||||
other := &Cache{Repo: testRepo, Key: "other", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-2 * time.Hour).Unix()}
|
other := &Cache{Repo: testRepo, Key: "other", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-2 * time.Hour).Unix()}
|
||||||
newest := &Cache{Repo: testRepo, Key: "newest", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
newest := &Cache{Repo: testRepo, Key: "newest", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
@@ -709,7 +749,67 @@ func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
|||||||
assert.Equal(t, []string{wanted.Key}, keptKeys(t, handler, []*Cache{wanted, other}), "the entry just promised to a job survives, the next least recently used goes instead")
|
assert.Equal(t, []string{wanted.Key}, keptKeys(t, handler, []*Cache{wanted, other}), "the entry just promised to a job survives, the next least recently used goes instead")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandler_PrefixUsesNewestReservationWithinSecond(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
created := time.Now().Add(-time.Hour).Unix()
|
||||||
|
older := &Cache{Repo: testRepo, Key: "deps-old", Version: "v", Complete: true, CreatedAt: created}
|
||||||
|
newer := &Cache{Repo: testRepo, Key: "deps-new", Version: "v", Complete: true, CreatedAt: created}
|
||||||
|
handler := newTestHandler(t, Policy{}, older, newer)
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
cache, err := findCache(db, JobCredential{Repo: testRepo}, []string{"deps-"}, "v")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, newer, cache)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_RestoreRefs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
const main, release, pull = "refs/heads/main", "refs/heads/release", "refs/pull/1/head"
|
||||||
|
now := time.Now().Unix()
|
||||||
|
entry := func(ref, key string, age int64) *Cache {
|
||||||
|
return &Cache{Repo: testRepo, Ref: ref, Key: key, Version: "v", Complete: true, CreatedAt: now - 3600 + age, UsedAt: now}
|
||||||
|
}
|
||||||
|
exactOnMain := entry(main, "deps-exact", -1)
|
||||||
|
onRelease := entry(release, "deps-release", 0)
|
||||||
|
onMain := entry(main, "deps-main", 1)
|
||||||
|
onPull := entry(pull, "deps-pull", 2)
|
||||||
|
unscoped := entry("", "deps-unscoped", 3)
|
||||||
|
handler := newTestHandler(t, Policy{}, exactOnMain, onRelease, onMain, onPull, unscoped, entry("refs/pull/2/head", "deps-lost", 4))
|
||||||
|
for _, cache := range []*Cache{exactOnMain, onRelease, onMain, onPull, unscoped} {
|
||||||
|
writeBlob(t, handler, cache.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
cred JobCredential
|
||||||
|
keys []string
|
||||||
|
want *Cache
|
||||||
|
}{
|
||||||
|
{"the default branch never restores what a pull request wrote", JobCredential{Ref: main, DefaultRef: main}, []string{"deps-"}, onMain},
|
||||||
|
{"a pull request restores its own entry first", JobCredential{Ref: pull, BaseRef: release, DefaultRef: main}, []string{"deps-"}, onPull},
|
||||||
|
{"a pull request falls back past a lost blob to its base branch before the default branch", JobCredential{Ref: "refs/pull/2/head", BaseRef: release, DefaultRef: main}, []string{"deps-"}, onRelease},
|
||||||
|
{"a branch does not restore a sibling branch", JobCredential{Ref: "refs/heads/other", DefaultRef: main}, []string{"deps-"}, onMain},
|
||||||
|
{"a prefix in the own ref wins over an exact key in the default branch", JobCredential{Ref: release, DefaultRef: main}, []string{"deps-exact", "deps-"}, onRelease},
|
||||||
|
{"a job with a ref never restores entries saved without one", JobCredential{Ref: "refs/heads/other"}, []string{"deps-"}, nil},
|
||||||
|
{"a job without a ref only restores entries saved without one", JobCredential{}, []string{"deps-"}, unscoped},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
tc.cred.Repo = testRepo
|
||||||
|
cache, err := handler.lookupCache(db, tc.cred, tc.keys, "v")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.want, cache)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandler_evictOnCommit(t *testing.T) {
|
func TestHandler_evictOnCommit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
handler := newTestHandler(t, Policy{RepoSizeLimit: 4 * miB}, full)
|
handler := newTestHandler(t, Policy{RepoSizeLimit: 4 * miB}, full)
|
||||||
|
|
||||||
@@ -719,6 +819,7 @@ func TestHandler_evictOnCommit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_gcCacheInterval(t *testing.T) {
|
func TestHandler_gcCacheInterval(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
cache := &Cache{Key: "temp", UsedAt: time.Now().Add(-time.Hour).Unix()}
|
cache := &Cache{Key: "temp", UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
handler := newTestHandler(t, Policy{SweepInterval: 30 * time.Minute}, cache)
|
handler := newTestHandler(t, Policy{SweepInterval: 30 * time.Minute}, cache)
|
||||||
|
|
||||||
@@ -728,6 +829,7 @@ func TestHandler_gcCacheInterval(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
fresh := &Cache{Repo: testRepo, Key: "fresh", Version: "v", Complete: true, UsedAt: time.Now().Unix()}
|
fresh := &Cache{Repo: testRepo, Key: "fresh", Version: "v", Complete: true, UsedAt: time.Now().Unix()}
|
||||||
pending := &Cache{Repo: testRepo, Key: "pending", Version: "v", UsedAt: time.Now().Add(-time.Second).Unix()}
|
pending := &Cache{Repo: testRepo, Key: "pending", Version: "v", UsedAt: time.Now().Add(-time.Second).Unix()}
|
||||||
handler := newTestHandler(t, Policy{}, fresh, pending)
|
handler := newTestHandler(t, Policy{}, fresh, pending)
|
||||||
@@ -794,6 +896,7 @@ func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_RejectsBearer(t *testing.T) {
|
func TestHandler_RejectsBearer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -830,6 +933,7 @@ func TestHandler_RejectsBearer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_UnregisterRevokes(t *testing.T) {
|
func TestHandler_UnregisterRevokes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
token string
|
token string
|
||||||
@@ -871,12 +975,14 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_CrossRepoIsolation(t *testing.T) {
|
func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer handler.Close()
|
defer handler.Close()
|
||||||
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
||||||
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
||||||
|
handler.RegisterJob("token-a-pull", JobCredential{Repo: "owner/repoA", Ref: "refs/pull/1/head"})
|
||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
key := "shared-key"
|
key := "shared-key"
|
||||||
@@ -911,16 +1017,24 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
|||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
for _, client := range []*http.Client{clientB, {Transport: &bearerTransport{token: "token-a-pull"}}} {
|
||||||
require.NoError(t, err)
|
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
||||||
req.Header.Set("Content-Range", "bytes 0-5/*")
|
require.NoError(t, err)
|
||||||
resp, err = clientB.Do(req)
|
req.Header.Set("Content-Range", "bytes 0-5/*")
|
||||||
require.NoError(t, err)
|
resp, err = client.Do(req)
|
||||||
resp.Body.Close()
|
require.NoError(t, err)
|
||||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
resp.Body.Close()
|
||||||
|
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||||
|
|
||||||
|
resp, err = client.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ArtifactSignature(t *testing.T) {
|
func TestHandler_ArtifactSignature(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -952,6 +1066,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
|
|
||||||
first, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"})
|
first, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"})
|
||||||
@@ -968,6 +1083,7 @@ func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
insert := func(by *Handler, cache *Cache) {
|
insert := func(by *Handler, cache *Cache) {
|
||||||
db, err := by.openDB()
|
db, err := by.openDB()
|
||||||
@@ -985,6 +1101,7 @@ func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1018,51 +1135,24 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
|||||||
assert.Equal(t, content, body)
|
assert.Equal(t, content, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_GC_PerRepoDedup(t *testing.T) {
|
func TestHandler_GC_DedupPerRepoAndRef(t *testing.T) {
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
t.Parallel()
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer handler.Close()
|
|
||||||
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"})
|
|
||||||
handler.RegisterJob("tok-b", JobCredential{Repo: "owner/repoB"})
|
|
||||||
|
|
||||||
key := "shared-dedup-key"
|
|
||||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
||||||
|
|
||||||
db, err := handler.openDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
||||||
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
|
entries := []*Cache{
|
||||||
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
|
{Repo: "owner/repoA", Key: "shared", Version: "v", Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1},
|
||||||
require.NoError(t, handler.insertCache(db, a))
|
{Repo: "owner/repoB", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||||
require.NoError(t, handler.insertCache(db, b))
|
{Repo: "owner/repoA", Ref: "refs/pull/1/head", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||||
require.NoError(t, handler.storage.Write(a.ID, 0, strings.NewReader("a")))
|
}
|
||||||
_, err = handler.storage.Commit(a.ID, 1)
|
handler := newTestHandler(t, Policy{}, entries...)
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, handler.storage.Write(b.ID, 0, strings.NewReader("b")))
|
|
||||||
_, err = handler.storage.Commit(b.ID, 1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
handler.gcAt = time.Time{}
|
handler.gcAt = time.Time{}
|
||||||
handler.gcCache()
|
handler.gcCache()
|
||||||
|
|
||||||
db, err = handler.openDB()
|
assert.Len(t, keptKeys(t, handler, entries), len(entries))
|
||||||
require.NoError(t, err)
|
|
||||||
defer db.Close()
|
|
||||||
after, err := db.find(func(cache *Cache) bool { return cache.Key == key && cache.Version == version })
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := make(map[string]bool)
|
|
||||||
for _, c := range after {
|
|
||||||
repos[c.Repo] = true
|
|
||||||
}
|
|
||||||
assert.True(t, repos["owner/repoA"], "repoA's cache must survive dedup against repoB")
|
|
||||||
assert.True(t, repos["owner/repoB"], "repoB's cache must survive dedup against repoA")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1077,6 +1167,7 @@ func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
const secret = "internal-secret"
|
const secret = "internal-secret"
|
||||||
handler, err := StartHandler(Options{Dir: dir, InternalSecret: secret})
|
handler, err := StartHandler(Options{Dir: dir, InternalSecret: secret})
|
||||||
@@ -1122,10 +1213,10 @@ func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","ref":"refs/pull/1/head","base_ref":"refs/heads/release","default_ref":"refs/heads/main","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||||
cred, ok := handler.lookupCredential("via-internal-api")
|
cred, ok := handler.lookupCredential("via-internal-api")
|
||||||
require.True(t, ok)
|
require.True(t, ok)
|
||||||
assert.Equal(t, JobCredential{Repo: testRepo, PublicURL: "http://runner.example"}, cred)
|
assert.Equal(t, JobCredential{Repo: testRepo, Ref: "refs/pull/1/head", BaseRef: "refs/heads/release", DefaultRef: "refs/heads/main", PublicURL: "http://runner.example"}, cred)
|
||||||
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
||||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||||
// existing and silently never saved.
|
// existing and silently never saved.
|
||||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if existing != nil {
|
} else if existing != nil {
|
||||||
@@ -84,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||||
owner := hashedToken(bearerToken(r))
|
owner := hashedToken(bearerToken(r))
|
||||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||||
@@ -100,15 +100,17 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache := &Cache{
|
cache := &Cache{
|
||||||
Repo: cred.Repo,
|
Repo: cred.Repo,
|
||||||
Key: req.Key,
|
Ref: cred.Ref,
|
||||||
Version: req.Version,
|
Key: req.Key,
|
||||||
Owner: owner,
|
Version: req.Version,
|
||||||
Size: -1, // the size is only known at finalize time
|
Owner: owner,
|
||||||
CreatedAt: now,
|
Size: -1, // the size is only known at finalize time
|
||||||
UsedAt: now,
|
CreatedAt: now.Unix(),
|
||||||
|
CreatedAtNanos: now.Nanosecond(),
|
||||||
|
UsedAt: now.Unix(),
|
||||||
}
|
}
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -136,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||||
cache = nil // not the reservation this job made, so not this job's to commit
|
cache = nil // not the reservation this job made, so not this job's to commit
|
||||||
}
|
}
|
||||||
if err == nil && cache == nil {
|
if err == nil && cache == nil {
|
||||||
// A retry whose first response was lost finds the entry already committed.
|
// A retry whose first response was lost finds the entry already committed.
|
||||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -185,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -103,6 +104,7 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||||
@@ -150,6 +152,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||||
require.Equal(t, true, saved["ok"])
|
require.Equal(t, true, saved["ok"])
|
||||||
@@ -228,9 +231,28 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||||
|
clients := map[string]*http.Client{"x": testClient}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||||
|
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||||
|
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
require.Equal(t, true, created["ok"])
|
||||||
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||||
|
}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||||
|
}
|
||||||
|
for content, client := range clients {
|
||||||
|
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -267,6 +289,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
Key string `json:"key" `
|
Key string `json:"key" `
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
@@ -11,13 +13,19 @@ type Request struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Cache struct {
|
type Cache struct {
|
||||||
ID uint64 `json:"id"`
|
ID uint64 `json:"id"`
|
||||||
Repo string `json:"repo"`
|
Repo string `json:"repo"`
|
||||||
Key string `json:"key"`
|
Ref string `json:"ref,omitzero"`
|
||||||
Version string `json:"version"`
|
Key string `json:"key"`
|
||||||
Size int64 `json:"cacheSize"`
|
Version string `json:"version"`
|
||||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
Size int64 `json:"cacheSize"`
|
||||||
Complete bool `json:"complete"`
|
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||||
UsedAt int64 `json:"usedAt"`
|
Complete bool `json:"complete"`
|
||||||
CreatedAt int64 `json:"createdAt"`
|
UsedAt int64 `json:"usedAt"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cache *Cache) createdAtNano() int64 {
|
||||||
|
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||||
}
|
}
|
||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFrontResultsService(t *testing.T) {
|
func TestFrontResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var gotHost, gotPath, gotProto string
|
var gotHost, gotPath, gotProto string
|
||||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -19,6 +20,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var objects sync.Map
|
var objects sync.Map
|
||||||
var uploads atomic.Int32
|
var uploads atomic.Int32
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
@@ -51,14 +53,33 @@ func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
|||||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
assert.EqualValues(t, 1, uploads.Load())
|
assert.EqualValues(t, 1, uploads.Load())
|
||||||
|
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
cache, err := saver.s3.fetch(t.Context(), id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||||
require.Equal(t, true, found["ok"])
|
require.Equal(t, true, found["ok"])
|
||||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
|
for _, cache := range []*Cache{
|
||||||
|
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||||
|
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||||
|
} {
|
||||||
|
require.NoError(t, saver.s3.publish(cache))
|
||||||
|
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||||
|
}
|
||||||
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
|
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||||
|
require.Equal(t, true, found["ok"])
|
||||||
|
assert.Equal(t, "deps-new", found["matched_key"])
|
||||||
|
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
objects.Clear()
|
objects.Clear()
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||||
@@ -71,6 +92,7 @@ func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
for _, failingPart := range []string{"", "2"} {
|
for _, failingPart := range []string{"", "2"} {
|
||||||
var parts, aborts atomic.Int32
|
var parts, aborts atomic.Int32
|
||||||
var completion struct {
|
var completion struct {
|
||||||
@@ -110,6 +132,7 @@ func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|||||||
+21
-7
@@ -13,6 +13,7 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -84,16 +85,25 @@ func uploads(router *http.ServeMux, baseDir string) {
|
|||||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
var start int64
|
||||||
appendUpload := req.Header.Get("Content-Range")
|
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if start == 0 {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
}
|
}
|
||||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
if req.Body == nil {
|
if req.Body == nil {
|
||||||
panic(errors.New("no body given"))
|
panic(errors.New("no body given"))
|
||||||
}
|
}
|
||||||
@@ -120,7 +130,7 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
runID := req.PathValue("runId")
|
runID := req.PathValue("runId")
|
||||||
|
|
||||||
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,6 +155,9 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
|
|
||||||
var files []ContainerItem
|
var files []ContainerItem
|
||||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !entry.IsDir() {
|
if !entry.IsDir() {
|
||||||
rel, err := filepath.Rel(safePath, path)
|
rel, err := filepath.Rel(safePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -157,12 +170,12 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
files = append(files, ContainerItem{
|
files = append(files, ContainerItem{
|
||||||
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||||
ItemType: "file",
|
ItemType: "file",
|
||||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, filepath.ToSlash(rel)),
|
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: net.JoinHostPort(addr, port),
|
Addr: net.JoinHostPort(addr, port),
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
router := http.NewServeMux()
|
router := http.NewServeMux()
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
server := httptest.NewServer(router)
|
server := httptest.NewServer(router)
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -48,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
return resp.StatusCode, data
|
return resp.StatusCode, data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("no-artifacts", func(t *testing.T) {
|
||||||
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
var list NamedFileContainerResourceURLResponse
|
||||||
|
require.NoError(t, json.Unmarshal(data, &list))
|
||||||
|
require.Zero(t, list.Count)
|
||||||
|
require.Empty(t, list.Value)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("upload-and-download", func(t *testing.T) {
|
t.Run("upload-and-download", func(t *testing.T) {
|
||||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||||
|
|
||||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
@@ -81,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(data, &items))
|
require.NoError(t, json.Unmarshal(data, &items))
|
||||||
require.Len(t, items.Value, 1)
|
require.Len(t, items.Value, 1)
|
||||||
require.Equal(t, "file", items.Value[0].ItemType)
|
require.Equal(t, "file", items.Value[0].ItemType)
|
||||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
require.Equal(t, item, items.Value[0].Path)
|
||||||
|
|
||||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(stored))
|
require.Equal(t, content, string(stored))
|
||||||
})
|
})
|
||||||
@@ -95,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
t.Run("content-range", func(t *testing.T) {
|
t.Run("content-range", func(t *testing.T) {
|
||||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
|
||||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
for range 2 {
|
||||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "first-second", string(stored))
|
require.Equal(t, "first-second", string(stored))
|
||||||
|
|
||||||
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||||
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, stored)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||||
@@ -156,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("v4", func(t *testing.T) {
|
||||||
|
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||||
|
call := func(method, body string) (int, map[string]any) {
|
||||||
|
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||||
|
var out map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||||
|
return status, out
|
||||||
|
}
|
||||||
|
put := func(rawURL, body string) int {
|
||||||
|
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
download := func() (string, http.Header) {
|
||||||
|
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, signed)
|
||||||
|
resp, err := client.Get(signed["signedUrl"].(string))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(body), resp.Header
|
||||||
|
}
|
||||||
|
|
||||||
|
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
upload := created["signedUploadUrl"].(string)
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||||
|
for attempt := range 2 {
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusBadRequest, status)
|
||||||
|
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status, finalized)
|
||||||
|
id := finalized["artifactId"].(string)
|
||||||
|
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
require.Equal(t, id, refinalized["artifactId"])
|
||||||
|
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusConflict, status)
|
||||||
|
require.Equal(t, "already_exists", conflict["code"])
|
||||||
|
|
||||||
|
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
artifacts := listed["artifacts"].([]any)
|
||||||
|
require.Len(t, artifacts, 1)
|
||||||
|
artifact := artifacts[0].(map[string]any)
|
||||||
|
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||||
|
delete(artifact, "createdAt")
|
||||||
|
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||||
|
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||||
|
require.Empty(t, other["artifacts"])
|
||||||
|
|
||||||
|
body, header := download()
|
||||||
|
require.Equal(t, "hello world", body)
|
||||||
|
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusNotFound, status)
|
||||||
|
require.Equal(t, "not_found", missing["code"])
|
||||||
|
|
||||||
|
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||||
|
require.Equal(t, http.StatusCreated, status)
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
body, header = download()
|
||||||
|
require.Equal(t, "gz", body)
|
||||||
|
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
rerun := http.NewServeMux()
|
||||||
|
registerV4(rerun, artifactPath)
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||||
|
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSafeResolve(t *testing.T) {
|
func TestSafeResolve(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||||
|
dir string
|
||||||
|
mu sync.Mutex
|
||||||
|
lastID int64
|
||||||
|
created map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
zipMimeType = "application/zip"
|
||||||
|
maxTwirpBytes = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerV4(router *http.ServeMux, baseDir string) {
|
||||||
|
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||||
|
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||||
|
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
type twirpError struct {
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *twirpError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
func notFound(name string) error {
|
||||||
|
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(format string, args ...any) error {
|
||||||
|
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req Req
|
||||||
|
var resp Resp
|
||||||
|
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||||
|
if err != nil {
|
||||||
|
err = badRequest("malformed request: %v", err)
|
||||||
|
} else {
|
||||||
|
resp, err = handle(r, req)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err != nil {
|
||||||
|
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||||
|
errors.As(err, &terr)
|
||||||
|
w.WriteHeader(terr.status)
|
||||||
|
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.MarshalWrite(w, resp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||||
|
sum := sha256.Sum256([]byte(name))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) runDir(runID string) string {
|
||||||
|
return filepath.Join(s.dir, fileName(runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) artifactDir(runID, name string) string {
|
||||||
|
return filepath.Join(s.runDir(runID), fileName(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) urlDir(r *http.Request) string {
|
||||||
|
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMeta(dir string) (*monolithArtifact, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var meta monolithArtifact
|
||||||
|
return &meta, json.Unmarshal(data, &meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||||
|
data, err := json.Marshal(meta)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||||
|
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.created[dir] {
|
||||||
|
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.lastID++
|
||||||
|
if err := writeMeta(dir, &monolithArtifact{
|
||||||
|
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||||
|
DatabaseID: s.lastID,
|
||||||
|
Name: req.Name,
|
||||||
|
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||||
|
}); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.created[dir] = true
|
||||||
|
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockList struct {
|
||||||
|
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||||
|
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||||
|
var err error
|
||||||
|
switch comp := r.URL.Query().Get("comp"); comp {
|
||||||
|
case "block":
|
||||||
|
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||||
|
case "blocklist":
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||||
|
case "": // Put Blob, which docker/build-push-action uses
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||||
|
_, err := io.Copy(out, r.Body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(name string, body io.Reader) error {
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, body)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||||
|
out, err := os.CreateTemp(dir, "blob")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||||
|
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(out.Name()))
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||||
|
var list blockList
|
||||||
|
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range list.Blocks {
|
||||||
|
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendBlock(out io.Writer, name string) error {
|
||||||
|
in, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
} else if meta.Digest != "" {
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
size, err := io.Copy(hash, blob)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if size != req.Size {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||||
|
}
|
||||||
|
if req.Hash != "" && req.Hash != digest {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||||
|
}
|
||||||
|
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
meta.Size, meta.Digest = size, digest
|
||||||
|
if err := writeMeta(dir, meta); err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||||
|
run := s.runDir(req.WorkflowRunBackendID)
|
||||||
|
entries, err := os.ReadDir(run)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return listArtifactsResponse{}, err
|
||||||
|
}
|
||||||
|
var resp listArtifactsResponse
|
||||||
|
for _, entry := range entries {
|
||||||
|
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||||
|
if err != nil || meta.Digest == "" ||
|
||||||
|
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||||
|
}
|
||||||
|
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||||
|
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||||
|
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil || meta.Digest == "" {
|
||||||
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||||
|
contentType, filename := string(data), meta.Name
|
||||||
|
if !strings.Contains(contentType, "/") {
|
||||||
|
contentType = zipMimeType
|
||||||
|
}
|
||||||
|
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||||
|
filename += ".zip"
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||||
|
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return deleteArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
delete(s.created, dir)
|
||||||
|
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||||
|
now := time.Now()
|
||||||
|
claims, _ := json.Marshal(map[string]any{
|
||||||
|
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||||
|
"ac": `[{"Scope":"","Permission":3}]`,
|
||||||
|
"nbf": now.Unix(),
|
||||||
|
"exp": now.Add(lifetime).Unix(),
|
||||||
|
"jti": rand.Text(),
|
||||||
|
})
|
||||||
|
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
|
||||||
|
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||||
|
createArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
SignedUploadURL string `json:"signedUploadUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeArtifactRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
Size int64 `json:",string"`
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
finalizeArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
ArtifactID int64 `json:"artifactId,string"`
|
||||||
|
}
|
||||||
|
|
||||||
|
listArtifactsRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
NameFilter string
|
||||||
|
IDFilter int64 `json:",string"`
|
||||||
|
}
|
||||||
|
listArtifactsResponse struct {
|
||||||
|
Artifacts []monolithArtifact `json:"artifacts"`
|
||||||
|
}
|
||||||
|
monolithArtifact struct {
|
||||||
|
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||||
|
DatabaseID int64 `json:"databaseId,string"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size int64 `json:"size,string"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||||
|
Digest string `json:"digest"` // empty until the upload is finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
getSignedArtifactURLRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
getSignedArtifactURLResponse struct {
|
||||||
|
SignedURL string `json:"signedUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
createArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactResponse = finalizeArtifactResponse
|
||||||
|
)
|
||||||
@@ -164,7 +164,10 @@ func (e Executor) Finally(finally Executor) Executor {
|
|||||||
err := e(ctx)
|
err := e(ctx)
|
||||||
err2 := finally(ctx)
|
err2 := finally(ctx)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -134,9 +135,13 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
mainErr := errors.New("main failed")
|
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||||
require.EqualError(t, err, "error occurred running finally: cleanup failed (original error: main failed)")
|
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||||
|
require.ErrorIs(t, err, finalErr)
|
||||||
|
if mainErr != nil {
|
||||||
|
require.ErrorIs(t, err, mainErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
@@ -28,6 +29,13 @@ import (
|
|||||||
|
|
||||||
var actionFetches sync.Map
|
var actionFetches sync.Map
|
||||||
|
|
||||||
|
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||||
|
|
||||||
|
type treeFile struct {
|
||||||
|
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
type actionFetch struct {
|
type actionFetch struct {
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
dir, sha string
|
dir, sha string
|
||||||
@@ -86,9 +94,14 @@ func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, s
|
|||||||
record := filepath.Join(input.Dir, "ref")
|
record := filepath.Join(input.Dir, "ref")
|
||||||
data, _ := os.ReadFile(record)
|
data, _ := os.ReadFile(record)
|
||||||
sha, etag, _ := strings.Cut(string(data), "\n")
|
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||||
if !isFullObjectID(sha) || !exists(tree(sha)) {
|
switch {
|
||||||
|
case !isFullObjectID(sha):
|
||||||
sha, etag = "", ""
|
sha, etag = "", ""
|
||||||
} else if input.OfflineMode || isFullObjectID(input.Ref) {
|
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||||
|
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||||
|
case !treeIntact(input.Dir, sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||||
return tree(sha), sha, nil
|
return tree(sha), sha, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,7 +186,7 @@ func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, toke
|
|||||||
newEtag := response.Header.Get("ETag")
|
newEtag := response.Header.Get("ETag")
|
||||||
gz, err := gzip.NewReader(response.Body)
|
gz, err := gzip.NewReader(response.Body)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||||
}
|
}
|
||||||
var netErr net.Error
|
var netErr net.Error
|
||||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||||
@@ -191,18 +204,18 @@ func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, e
|
|||||||
clone := input
|
clone := input
|
||||||
clone.Dir = filepath.Join(input.Dir, "git")
|
clone.Dir = filepath.Join(input.Dir, "git")
|
||||||
sha, _, err := resolveClone(ctx, clone)
|
sha, _, err := resolveClone(ctx, clone)
|
||||||
if err != nil || exists(filepath.Join(input.Dir, sha)) {
|
if err != nil || treeIntact(input.Dir, sha) {
|
||||||
return sha, err
|
return sha, err
|
||||||
}
|
}
|
||||||
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
|
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||||
return sha, err
|
return sha, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||||
temp, err := os.MkdirTemp(dir, "tmp-")
|
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -215,6 +228,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
defer root.Close()
|
defer root.Close()
|
||||||
|
|
||||||
sha := ""
|
sha := ""
|
||||||
|
sizes := map[string]int64{}
|
||||||
reader := tar.NewReader(archive)
|
reader := tar.NewReader(archive)
|
||||||
for {
|
for {
|
||||||
header, err := reader.Next()
|
header, err := reader.Next()
|
||||||
@@ -225,7 +239,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
|
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||||
return sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -241,10 +255,10 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
case tar.TypeDir:
|
case tar.TypeDir:
|
||||||
err = root.MkdirAll(name, 0o755)
|
err = root.MkdirAll(name, 0o755)
|
||||||
case tar.TypeReg:
|
case tar.TypeReg:
|
||||||
err = writeTreeFile(root, name, header.Mode, reader)
|
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||||
case tar.TypeSymlink:
|
case tar.TypeSymlink:
|
||||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||||
err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -257,20 +271,76 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
if err := root.Close(); err != nil {
|
if err := root.Close(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return sha, os.Rename(temp, filepath.Join(dir, sha))
|
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
|
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||||
|
|
||||||
|
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||||
|
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||||
|
files := make([]treeFile, 0, len(sizes))
|
||||||
|
for name, size := range sizes {
|
||||||
|
files = append(files, treeFile{[]byte(name), size})
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tree := filepath.Join(dir, sha)
|
||||||
|
if !exists(tree) {
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||||
|
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||||
|
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||||
|
func treeIntact(dir, sha string) bool {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||||
|
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var files []treeFile
|
||||||
|
if json.Unmarshal(data, &files) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||||
perm := os.FileMode(0o644)
|
perm := os.FileMode(0o644)
|
||||||
if mode&0o111 != 0 {
|
if mode&0o111 != 0 {
|
||||||
perm = 0o755
|
perm = 0o755
|
||||||
}
|
}
|
||||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
_, err = io.Copy(file, content)
|
n, err := io.Copy(file, content)
|
||||||
return errors.Join(err, file.Close())
|
return n, errors.Join(err, file.Close())
|
||||||
}
|
}
|
||||||
|
|
||||||
func exists(name string) bool {
|
func exists(name string) bool {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -41,6 +42,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||||
|
expectedSha := gitRevParse(t, remote, "main")
|
||||||
|
|
||||||
gitPath, err := exec.LookPath("git")
|
gitPath, err := exec.LookPath("git")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -48,6 +50,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
var archiveRequests []http.Header
|
var archiveRequests []http.Header
|
||||||
var mirror *httptest.Server
|
var mirror *httptest.Server
|
||||||
|
archiveCache := map[string][]byte{}
|
||||||
requests, archives, failures := 0, "", 0
|
requests, archives, failures := 0, "", 0
|
||||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
@@ -84,7 +87,12 @@ func TestFetchAction(t *testing.T) {
|
|||||||
if archives != "unprefixed" {
|
if archives != "unprefixed" {
|
||||||
args = append(args, "--prefix=action/")
|
args = append(args, "--prefix=action/")
|
||||||
}
|
}
|
||||||
archive, _ := runGit(request.Context(), remote, nil, args...)
|
archiveKey := strings.Join(args, "\x00")
|
||||||
|
archive, ok := archiveCache[archiveKey]
|
||||||
|
if !ok {
|
||||||
|
archive, _ = runGit(request.Context(), remote, nil, args...)
|
||||||
|
archiveCache[archiveKey] = archive
|
||||||
|
}
|
||||||
writer.Header().Set("ETag", etag)
|
writer.Header().Set("ETag", etag)
|
||||||
_, _ = writer.Write(archive)
|
_, _ = writer.Write(archive)
|
||||||
})
|
})
|
||||||
@@ -107,7 +115,6 @@ func TestFetchAction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||||
expectedSha := gitRevParse(t, remote, "main")
|
|
||||||
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||||
t.Run(scenario, func(t *testing.T) {
|
t.Run(scenario, func(t *testing.T) {
|
||||||
serve("prefixed", 0)
|
serve("prefixed", 0)
|
||||||
@@ -180,7 +187,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
input := newInput("v1")
|
input := newInput("v1")
|
||||||
tree, sha, err := FetchAction(t.Context(), input)
|
tree, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "v1"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||||
@@ -196,9 +203,10 @@ func TestFetchAction(t *testing.T) {
|
|||||||
|
|
||||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||||
|
expectedSha = gitRevParse(t, remote, "v1")
|
||||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "v1"), movedSha)
|
assert.Equal(t, expectedSha, movedSha)
|
||||||
assert.NotEqual(t, tree, moved)
|
assert.NotEqual(t, tree, moved)
|
||||||
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
|
||||||
@@ -206,10 +214,17 @@ func TestFetchAction(t *testing.T) {
|
|||||||
_, offlineSha, err := FetchAction(t.Context(), input)
|
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, movedSha, offlineSha)
|
assert.Equal(t, movedSha, offlineSha)
|
||||||
|
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||||
|
reused, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, moved, reused)
|
||||||
|
|
||||||
requests, headers := served()
|
requests, headers := served()
|
||||||
assert.Equal(t, 3, requests)
|
assert.Equal(t, 4, requests)
|
||||||
require.Len(t, headers, 3)
|
require.Len(t, headers, 4)
|
||||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||||
assert.Empty(t, headers[0].Get("If-None-Match"))
|
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||||
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||||
@@ -218,7 +233,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||||
serve("prefixed", 0)
|
serve("prefixed", 0)
|
||||||
sha := gitRevParse(t, remote, "main")
|
sha := expectedSha
|
||||||
input := newInput(sha)
|
input := newInput(sha)
|
||||||
for range 2 {
|
for range 2 {
|
||||||
tree, got, err := FetchAction(t.Context(), input)
|
tree, got, err := FetchAction(t.Context(), input)
|
||||||
@@ -233,12 +248,74 @@ func TestFetchAction(t *testing.T) {
|
|||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||||
|
sha := gitRevParse(t, remote, "main")
|
||||||
|
for _, mode := range []string{"prefixed", "none"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
for _, damage := range []func() error{
|
||||||
|
func() error { return os.Truncate(yml, 4) },
|
||||||
|
func() error { return os.Remove(tree + treeSizes) },
|
||||||
|
} {
|
||||||
|
require.NoError(t, damage(), mode)
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, "name: action\n", string(body), mode)
|
||||||
|
}
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 3, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.Truncate(yml, 4))
|
||||||
|
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||||
|
require.NoError(t, err)
|
||||||
|
release := sync.OnceFunc(unlock)
|
||||||
|
defer release()
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||||
|
close(waiting)
|
||||||
|
return AcquireCloneLock(ctx, dir)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||||
|
case <-waiting:
|
||||||
|
}
|
||||||
|
info, err := os.Stat(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||||
|
release()
|
||||||
|
require.NoError(t, <-done)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "name: action\n", string(body))
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||||
serve("redirect", 2)
|
serve("redirect", 2)
|
||||||
input := newInput("main")
|
input := newInput("main")
|
||||||
_, sha, err := FetchAction(t.Context(), input)
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
require.Len(t, headers, 4)
|
require.Len(t, headers, 4)
|
||||||
@@ -251,7 +328,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
serve(mode, 0)
|
serve(mode, 0)
|
||||||
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||||
require.NoError(t, err, mode)
|
require.NoError(t, err, mode)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha, mode)
|
assert.Equal(t, expectedSha, sha, mode)
|
||||||
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||||
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
@@ -266,7 +343,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
input.URL = "https://example.invalid/org/action"
|
input.URL = "https://example.invalid/org/action"
|
||||||
_, sha, err := FetchAction(t.Context(), input)
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
assert.Len(t, headers, 1)
|
assert.Len(t, headers, 1)
|
||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
@@ -306,10 +383,30 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
|||||||
require.NoError(t, writer.Close())
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
_, err := extractTree(&archive, dir, false)
|
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||||
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||||
|
}
|
||||||
|
sha := strings.Repeat("a", 40)
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||||
|
_, err := writer.Write([]byte("x"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||||
|
assert.True(t, treeIntact(dir, sha))
|
||||||
|
}
|
||||||
+18
-3
@@ -10,6 +10,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -320,9 +321,18 @@ func cachedRef(ref string) string {
|
|||||||
|
|
||||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||||
if !isFullObjectID(ref) {
|
if !isFullObjectID(ref) {
|
||||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
if strings.ContainsAny(ref, "\x00\n") {
|
||||||
return sha, nil
|
for _, revision := range revisions {
|
||||||
|
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||||
|
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||||
|
if isFullObjectID(sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -427,6 +437,10 @@ func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||||
|
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
@@ -438,6 +452,7 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
|||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
|
cmd.Stdin = stdin
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||||
name, _, _ := strings.Cut(entry, "=")
|
name, _, _ := strings.Cut(entry, "=")
|
||||||
return slices.Contains(LocalEnvVars, name)
|
return slices.Contains(LocalEnvVars, name)
|
||||||
|
|||||||
+59
-28
@@ -29,6 +29,31 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var gitFixtureDir string
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
var err error
|
||||||
|
gitFixtureDir, err = os.MkdirTemp("", "runner-git-fixture-")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
code := 1
|
||||||
|
if err = gitCmd("init", "--quiet", "--template=", "--bare", "--initial-branch=main", filepath.Join(gitFixtureDir, "remote")); err == nil {
|
||||||
|
err = gitCmd("init", "--quiet", "--template=", "--initial-branch=main", filepath.Join(gitFixtureDir, "work"))
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
code = m.Run()
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(gitFixtureDir); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
code = 1
|
||||||
|
}
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
func TestFindGitSlug(t *testing.T) {
|
func TestFindGitSlug(t *testing.T) {
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
remoteURL string
|
remoteURL string
|
||||||
@@ -141,9 +166,8 @@ func TestGitFindRef(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
dir := filepath.Join(basedir, name)
|
dir := filepath.Join(basedir, name)
|
||||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
|
require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/master\n"), 0o644))
|
||||||
require.NoError(t, os.RemoveAll(filepath.Join(dir, ".git", "hooks")))
|
|
||||||
for _, command := range tt.commands {
|
for _, command := range tt.commands {
|
||||||
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
||||||
}
|
}
|
||||||
@@ -180,6 +204,7 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||||
t.Setenv("GIT_DIR", t.TempDir())
|
t.Setenv("GIT_DIR", t.TempDir())
|
||||||
|
|
||||||
|
var clones sync.WaitGroup
|
||||||
for name, tt := range map[string]struct {
|
for name, tt := range map[string]struct {
|
||||||
Err error
|
Err error
|
||||||
Ref string
|
Ref string
|
||||||
@@ -207,31 +232,34 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
Ref: fullSha[:4],
|
Ref: fullSha[:4],
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
clones.Go(func() {
|
||||||
dir := t.TempDir()
|
t.Run(name, func(t *testing.T) {
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
dir := t.TempDir()
|
||||||
URL: "https://example.invalid/action",
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
Ref: tt.Ref,
|
URL: "https://example.invalid/action",
|
||||||
Dir: dir,
|
Ref: tt.Ref,
|
||||||
})
|
Dir: dir,
|
||||||
|
})
|
||||||
|
|
||||||
err := clone(t.Context())
|
err := clone(t.Context())
|
||||||
if tt.Err != nil {
|
if tt.Err != nil {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Equal(t, tt.Err, err)
|
assert.Equal(t, tt.Err, err)
|
||||||
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
marker := filepath.Join(dir, "marker")
|
marker := filepath.Join(dir, "marker")
|
||||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||||
require.NoError(t, clone(t.Context()))
|
require.NoError(t, clone(t.Context()))
|
||||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||||
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
clones.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||||
@@ -476,9 +504,12 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
|||||||
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
remoteDir, workDir := t.TempDir(), t.TempDir()
|
remoteDir, workDir := t.TempDir(), t.TempDir()
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
require.NoError(t, os.CopyFS(remoteDir, os.DirFS(filepath.Join(gitFixtureDir, "remote"))))
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
require.NoError(t, os.CopyFS(workDir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
config, err := os.ReadFile(filepath.Join(workDir, ".git", "config"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".git", "config"),
|
||||||
|
fmt.Appendf(config, "\n[remote \"origin\"]\n\turl = %s\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", strconv.Quote(remoteDir)), 0o644))
|
||||||
for _, message := range messages {
|
for _, message := range messages {
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -74,8 +74,10 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
||||||
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
||||||
|
if err != nil {
|
||||||
return logDockerResponse(logger, resp.Body, err != nil)
|
return err
|
||||||
|
}
|
||||||
|
return logDockerResponse(logger, resp.Body, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,13 +8,35 @@ package container
|
|||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||||
|
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
writer.Header().Set("API-Version", "1.47")
|
||||||
|
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||||
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
|
writer.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(daemon.Close)
|
||||||
|
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||||
|
t.Setenv("DOCKER_CONFIG", t.TempDir())
|
||||||
|
t.Setenv("DOCKER_AUTH_CONFIG", "")
|
||||||
|
|
||||||
|
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||||
|
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||||
|
})(t.Context()), "build rejected")
|
||||||
|
}
|
||||||
|
|
||||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
for name, content := range map[string]string{
|
for name, content := range map[string]string{
|
||||||
|
|||||||
@@ -5,14 +5,17 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
|
mobyclient "github.com/moby/moby/client"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -21,48 +24,47 @@ func init() {
|
|||||||
logging.SetLevel(logging.LevelDebug)
|
logging.SetLevel(logging.LevelDebug)
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
|
func loadScratchImage(t *testing.T, platform string) string {
|
||||||
// network or emulation since there is nothing to run) and returns its tag, removing it after
|
|
||||||
// the test.
|
|
||||||
func buildScratchImage(t *testing.T, platform string) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
tag := fmt.Sprintf("act-test-exists-%s:latest", strings.TrimPrefix(platform, "linux/"))
|
tag := fmt.Sprintf("act-test-exists-%s-%d:latest", strings.TrimPrefix(platform, "linux/"), time.Now().UnixNano())
|
||||||
cmd := exec.Command("docker", "build", "--platform", platform, "-t", tag, "-")
|
var archive bytes.Buffer
|
||||||
cmd.Stdin = strings.NewReader("FROM scratch\nLABEL act-test=1\n")
|
require.NoError(t, writeFilesTar(t.Context(), &archive, 0, 0,
|
||||||
// Force BuildKit: it records the requested architecture in the image config for a
|
&FileEntry{Name: "config.json", Mode: 0o600, Body: fmt.Sprintf(`{"architecture":%q,"os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`, strings.TrimPrefix(platform, "linux/"))},
|
||||||
// FROM-scratch build, whereas the classic builder ignores --platform and tags it with the
|
&FileEntry{Name: "manifest.json", Mode: 0o600, Body: fmt.Sprintf(`[{"Config":"config.json","RepoTags":[%q],"Layers":[]}]`, tag)},
|
||||||
// host arch, which would break the per-platform existence assertions below.
|
))
|
||||||
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
|
cli, err := GetDockerClient(t.Context())
|
||||||
out, err := cmd.CombinedOutput()
|
require.NoError(t, err)
|
||||||
require.NoError(t, err, string(out))
|
t.Cleanup(func() { _ = cli.Close() })
|
||||||
t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() })
|
response, err := cli.ImageLoad(t.Context(), &archive)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer response.Close()
|
||||||
|
require.NoError(t, logDockerResponse(common.Logger(t.Context()), response, false))
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = cli.ImageRemove(context.WithoutCancel(t.Context()), tag, mobyclient.ImageRemoveOptions{Force: true})
|
||||||
|
})
|
||||||
return tag
|
return tag
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageExistsLocally(t *testing.T) {
|
func TestImageExistsLocally(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireDocker(t)
|
requireDocker(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
// a non-existent image is reported absent
|
|
||||||
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, missing)
|
assert.False(t, missing)
|
||||||
|
|
||||||
// Build tiny images for two architectures locally so per-platform existence can be checked
|
amd64Ref := loadScratchImage(t, "linux/amd64")
|
||||||
// offline (formerly pulled node:24-bookworm-slim for amd64 and arm64 over the network).
|
arm64Ref := loadScratchImage(t, "linux/arm64")
|
||||||
amd64Ref := buildScratchImage(t, "linux/amd64")
|
|
||||||
arm64Ref := buildScratchImage(t, "linux/arm64")
|
|
||||||
|
|
||||||
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, amd64Exists)
|
assert.True(t, amd64Exists)
|
||||||
|
|
||||||
// a non-host architecture image is detected for its own architecture
|
|
||||||
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, arm64Exists)
|
assert.True(t, arm64Exists)
|
||||||
|
|
||||||
// a present image is reported absent for a different platform
|
|
||||||
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, wrongPlatform)
|
assert.False(t, wrongPlatform)
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
|||||||
defer dockerResponse.Close()
|
defer dockerResponse.Close()
|
||||||
|
|
||||||
scanner := bufio.NewScanner(dockerResponse)
|
scanner := bufio.NewScanner(dockerResponse)
|
||||||
|
scanner.Buffer(nil, 64*1024*1024) // a build echoes long instructions in one record
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Bytes()
|
line := scanner.Bytes()
|
||||||
var msg dockerMessage
|
var msg dockerMessage
|
||||||
@@ -65,7 +66,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return scanner.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
||||||
|
|||||||
@@ -85,14 +85,14 @@ networks:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return createDockerNetwork(ctx, cli, name, opts)
|
return createDockerNetwork(ctx, cli, name, opts)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||||
@@ -157,7 +157,7 @@ func isAddressPoolExhausted(err error) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -167,7 +167,7 @@ func NewDockerNetworkConnectExecutor(networkName, container string) common.Execu
|
|||||||
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -182,14 +182,14 @@ func connectNetwork(ctx context.Context, cli client.APIClient, networkName, cont
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return disconnectNetwork(ctx, cli, networkName, container)
|
return disconnectNetwork(ctx, cli, networkName, container)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -201,14 +201,14 @@ func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, c
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return removeNetwork(ctx, cli, name)
|
return removeNetwork(ctx, cli, name)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
||||||
|
|||||||
+104
-29
@@ -40,7 +40,9 @@ const (
|
|||||||
jobLabel = "com.gitea.runner.job."
|
jobLabel = "com.gitea.runner.job."
|
||||||
maxCreateBody = 8 << 20
|
maxCreateBody = 8 << 20
|
||||||
|
|
||||||
dockerProxyProbeTimeout = 5 * time.Second
|
dockerProxyProbeTimeout = 5 * time.Second
|
||||||
|
dockerProxyProbeWorkerTimeout = time.Minute
|
||||||
|
dockerProxyProbeTarget = "/gitea-runner-probe"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -48,20 +50,27 @@ var (
|
|||||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
var dockerProxyDirectories sync.Map
|
var dockerProxyProbes sync.Map
|
||||||
|
|
||||||
type dockerProxyDirectory struct {
|
type dockerProxyDirectory struct {
|
||||||
dir, daemonDir string
|
dir, daemonDir string
|
||||||
socket os.FileInfo
|
socket os.FileInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type dockerProxyProbe struct {
|
||||||
|
directory dockerProxyDirectory
|
||||||
|
done chan struct{}
|
||||||
|
seen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||||
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
cli, err := GetDockerClient(probeCtx)
|
cli, err := client.New(client.FromEnv) // negotiates on first request, so a cached probe costs no round trip
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -76,42 +85,72 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
|||||||
}
|
}
|
||||||
workdir, _ := os.Getwd()
|
workdir, _ := os.Getwd()
|
||||||
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||||
cached, _ := dockerProxyDirectories.Load(key)
|
probe, err := probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||||
directory, seen := cached.(dockerProxyDirectory)
|
|
||||||
seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
|
|
||||||
dir, daemonDir := directory.dir, directory.daemonDir
|
|
||||||
if !seen {
|
|
||||||
dir, daemonDir = runnerContainerWorkdir(probeCtx, cli)
|
|
||||||
seen = daemonDir != ""
|
|
||||||
if !seen {
|
|
||||||
if dir, err = filepath.Abs(os.TempDir()); err == nil {
|
|
||||||
daemonDir = dir
|
|
||||||
seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if !seen {
|
if !probe.seen {
|
||||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
proxy, err := StartDockerProxy(daemonSocket, dir, job)
|
proxy, err := StartDockerProxy(daemonSocket, probe.directory.dir, job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
dockerProxyDirectories.Delete(key)
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
dockerProxyDirectories.Store(key, dockerProxyDirectory{dir, daemonDir, socket})
|
proxy.Socket = probe.directory.daemonDir + strings.TrimPrefix(proxy.Socket, probe.directory.dir)
|
||||||
proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
|
|
||||||
return proxy
|
return proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (*dockerProxyProbe, error) {
|
||||||
|
for {
|
||||||
|
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
|
||||||
|
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
|
||||||
|
if loaded {
|
||||||
|
probe, _ = actual.(*dockerProxyProbe)
|
||||||
|
if !os.SameFile(probe.directory.socket, socket) { // daemon replacement invalidates cached paths
|
||||||
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
go func() { // outlives callers' deadlines so a slow daemon still gets its result cached
|
||||||
|
defer cli.Close()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeWorkerTimeout)
|
||||||
|
defer cancel()
|
||||||
|
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||||
|
probe.seen = probe.directory.daemonDir != ""
|
||||||
|
if !probe.seen {
|
||||||
|
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
|
||||||
|
probe.directory.daemonDir = probe.directory.dir
|
||||||
|
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if probe.seen {
|
||||||
|
dockerProxyProbes.LoadOrStore(key, probe)
|
||||||
|
} else {
|
||||||
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
|
}
|
||||||
|
close(probe.done)
|
||||||
|
removeStaleProbeContainers(cli)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return &dockerProxyProbe{}, ctx.Err()
|
||||||
|
case <-probe.done:
|
||||||
|
}
|
||||||
|
if ctx.Err() == nil && errors.Is(probe.err, context.DeadlineExceeded) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return probe, probe.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
||||||
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
||||||
workdir, err := os.Getwd()
|
workdir, err := os.Getwd()
|
||||||
@@ -162,19 +201,25 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
|||||||
if len(images.Items) == 0 {
|
if len(images.Items) == 0 {
|
||||||
return false, errors.New("no image available for the docker proxy probe")
|
return false, errors.New("no image available for the docker proxy probe")
|
||||||
}
|
}
|
||||||
created, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
var created client.ContainerCreateResult
|
||||||
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
|
for _, image := range images.Items {
|
||||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
|
||||||
}},
|
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||||
})
|
{Type: mount.TypeBind, Source: daemonDir, Target: dockerProxyProbeTarget, ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if !cerrdefs.IsNotFound(err) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
if cerrdefs.IsInvalidArgument(err) {
|
if cerrdefs.IsInvalidArgument(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, dockerProxyProbeTarget)
|
||||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||||
@@ -183,6 +228,36 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
|||||||
return seen, err
|
return seen, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeStaleProbeContainers removes probe containers left by abandoned creates once any probe in flight when listed has ended.
|
||||||
|
func removeStaleProbeContainers(cli client.APIClient) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||||
|
defer cancel()
|
||||||
|
listed, err := cli.ContainerList(ctx, client.ContainerListOptions{
|
||||||
|
All: true,
|
||||||
|
Filters: make(client.Filters).Add("status", string(container.StateCreated)).Add("volume", dockerProxyProbeTarget),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
common.Logger(ctx).Warnf("listing stale docker proxy probe containers failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
listed.Items = slices.DeleteFunc(listed.Items, func(stale container.Summary) bool {
|
||||||
|
return stale.State != container.StateCreated || !slices.ContainsFunc(stale.Mounts, func(mountPoint container.MountPoint) bool {
|
||||||
|
return mountPoint.Type == mount.TypeBind && !mountPoint.RW && mountPoint.Destination == dockerProxyProbeTarget
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if len(listed.Items) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(dockerProxyProbeWorkerTimeout + dockerProxyProbeTimeout)
|
||||||
|
for _, stale := range listed.Items {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||||
|
if err := (&containerReference{cli: cli, id: stale.ID}).remove()(ctx); err != nil {
|
||||||
|
common.Logger(ctx).Warnf("removing stale docker proxy probe container failed: %v", err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||||
info, err := os.Stat(daemonSocket)
|
info, err := os.Stat(daemonSocket)
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
@@ -93,6 +94,8 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
_, _ = w.Write([]byte(`[]`))
|
||||||
case strings.HasSuffix(r.URL.Path, "/json"):
|
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
case strings.HasSuffix(r.URL.Path, "/archive"):
|
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||||
@@ -234,7 +237,8 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
t.Skip("automatic proxy discovery requires Linux")
|
t.Skip("automatic proxy discovery requires Linux")
|
||||||
}
|
}
|
||||||
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||||
t.Setenv("TMPDIR", shortTempDir(t))
|
tempDir := shortTempDir(t)
|
||||||
|
t.Setenv("TMPDIR", tempDir)
|
||||||
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||||
for _, job := range []string{"first", "second"} {
|
for _, job := range []string{"first", "second"} {
|
||||||
proxy := NewDockerProxy(t.Context(), job)
|
proxy := NewDockerProxy(t.Context(), job)
|
||||||
@@ -242,12 +246,10 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
require.NoError(t, proxy.Close(context.Background()))
|
require.NoError(t, proxy.Close(context.Background()))
|
||||||
}
|
}
|
||||||
assert.EqualValues(t, 2, probes.Load())
|
assert.EqualValues(t, 2, probes.Load())
|
||||||
dockerProxyDirectories.Range(func(key, value any) bool {
|
workdir, err := os.Getwd()
|
||||||
directory, _ := value.(dockerProxyDirectory)
|
require.NoError(t, err)
|
||||||
directory.socket, _ = os.Stat(t.TempDir())
|
cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
|
||||||
dockerProxyDirectories.Store(key, directory)
|
cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
|
||||||
return true
|
|
||||||
})
|
|
||||||
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||||
require.NotNil(t, proxy)
|
require.NotNil(t, proxy)
|
||||||
require.NoError(t, proxy.Close(context.Background()))
|
require.NoError(t, proxy.Close(context.Background()))
|
||||||
@@ -371,6 +373,9 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
|||||||
|
|
||||||
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||||
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
||||||
|
if os.Geteuid() != 0 && errors.Is(err, os.ErrPermission) {
|
||||||
|
t.Skipf("non-root runners that cannot take the daemon socket's ownership get it directly: %v", err)
|
||||||
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
||||||
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
||||||
@@ -472,10 +477,23 @@ func BenchmarkDockerProxy(b *testing.B) {
|
|||||||
|
|
||||||
type probeClient struct {
|
type probeClient struct {
|
||||||
mobyclient.APIClient
|
mobyclient.APIClient
|
||||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
create func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||||
stat func(string) error
|
stat func(string) error
|
||||||
mounts []container.MountPoint
|
mounts []container.MountPoint
|
||||||
|
containers []container.Summary
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
|
||||||
|
return mobyclient.ContainerListResult{Items: c.containers}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||||
|
return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||||
@@ -483,11 +501,11 @@ func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.Conta
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
||||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "probe-image"}}}, nil
|
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
return c.create(opts)
|
return c.create(ctx, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
@@ -498,9 +516,193 @@ func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobycl
|
|||||||
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||||
|
type result struct {
|
||||||
|
seen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
for name, createErr := range map[string]error{"unavailable directory": nil, "daemon error": errors.New("probe failed")} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
key := [3]string{"daemon", dir, dir}
|
||||||
|
release := make(chan struct{})
|
||||||
|
var calls, removals atomic.Int32
|
||||||
|
visible := false
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
<-release
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
|
||||||
|
},
|
||||||
|
stat: func(_path string) error {
|
||||||
|
if visible {
|
||||||
|
return nil
|
||||||
|
} else {
|
||||||
|
return cerrdefs.ErrNotFound
|
||||||
|
}
|
||||||
|
},
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removals.Add(1)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
results := make(chan result, 8)
|
||||||
|
for range cap(results) {
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
results <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
synctest.Wait()
|
||||||
|
assert.EqualValues(t, 1, calls.Load())
|
||||||
|
replacement, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
replaced := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||||
|
replaced <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
close(release)
|
||||||
|
for range cap(results) {
|
||||||
|
result := <-results
|
||||||
|
require.ErrorIs(t, result.err, createErr)
|
||||||
|
assert.False(t, result.seen)
|
||||||
|
}
|
||||||
|
replacementResult := <-replaced
|
||||||
|
require.ErrorIs(t, replacementResult.err, createErr)
|
||||||
|
assert.False(t, replacementResult.seen)
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
if createErr == nil {
|
||||||
|
assert.EqualValues(t, 2, removals.Load())
|
||||||
|
} else {
|
||||||
|
assert.Zero(t, removals.Load())
|
||||||
|
}
|
||||||
|
createErr, visible = nil, true
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, probe.seen)
|
||||||
|
assert.Equal(t, dir, probe.directory.dir)
|
||||||
|
assert.Equal(t, dir, probe.directory.daemonDir)
|
||||||
|
assert.EqualValues(t, 3, calls.Load())
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, entries)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
key := [3]string{"daemon", dir, dir}
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
release := make(chan struct{})
|
||||||
|
var calls, removals atomic.Int32
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
<-release
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
|
||||||
|
},
|
||||||
|
stat: func(_path string) error { return nil },
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removals.Add(1)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||||
|
leader <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
follower <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
cancel()
|
||||||
|
leaderResult := <-leader
|
||||||
|
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||||
|
assert.False(t, leaderResult.seen)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
late <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
close(release)
|
||||||
|
for _, results := range []chan result{follower, late} {
|
||||||
|
result := <-results
|
||||||
|
require.NoError(t, result.err)
|
||||||
|
assert.True(t, result.seen)
|
||||||
|
}
|
||||||
|
assert.EqualValues(t, 1, calls.Load())
|
||||||
|
assert.EqualValues(t, 1, removals.Load())
|
||||||
|
_, cached := dockerProxyProbes.Load(key)
|
||||||
|
assert.True(t, cached)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
var calls atomic.Int32
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
<-ctx.Done()
|
||||||
|
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||||
|
}
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||||
|
},
|
||||||
|
stat: func(_path string) error { return nil },
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, probe.seen)
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveStaleProbeContainers(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
|
||||||
|
writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
|
||||||
|
var removed []string
|
||||||
|
cli := &probeClient{
|
||||||
|
containers: []container.Summary{
|
||||||
|
{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
|
||||||
|
{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
|
||||||
|
{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
|
||||||
|
},
|
||||||
|
remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removed = append(removed, id)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
removeStaleProbeContainers(cli)
|
||||||
|
assert.Equal(t, []string{"abandoned"}, removed)
|
||||||
|
assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestDaemonSeesDir(t *testing.T) {
|
func TestDaemonSeesDir(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
markers := make(map[string]bool)
|
markers := make(map[string]bool)
|
||||||
|
|
||||||
for _, testCase := range []struct {
|
for _, testCase := range []struct {
|
||||||
name string
|
name string
|
||||||
daemonDir string
|
daemonDir string
|
||||||
@@ -517,9 +719,14 @@ func TestDaemonSeesDir(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(t.Context())
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
removed := false
|
removed := false
|
||||||
|
var attemptedImages []string
|
||||||
cli := &probeClient{
|
cli := &probeClient{
|
||||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||||
|
attemptedImages = append(attemptedImages, opts.Config.Image)
|
||||||
|
if opts.Config.Image == "removed-image" {
|
||||||
|
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrNotFound
|
||||||
|
}
|
||||||
cancel()
|
cancel()
|
||||||
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
||||||
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
||||||
@@ -547,8 +754,10 @@ func TestDaemonSeesDir(t *testing.T) {
|
|||||||
}
|
}
|
||||||
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
||||||
require.ErrorIs(t, err, testCase.removeErr)
|
require.ErrorIs(t, err, testCase.removeErr)
|
||||||
|
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
|
||||||
assert.Equal(t, testCase.seen, seen)
|
assert.Equal(t, testCase.seen, seen)
|
||||||
assert.Equal(t, testCase.removed, removed)
|
assert.Equal(t, testCase.removed, removed)
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
entries, err := os.ReadDir(dir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Empty(t, entries)
|
assert.Empty(t, entries)
|
||||||
|
|||||||
@@ -114,7 +114,7 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return imagePullOptions, err
|
return imagePullOptions, err
|
||||||
}
|
}
|
||||||
if authConfig.Username == "" && authConfig.Password == "" {
|
if authConfig.Username == "" && authConfig.Password == "" && authConfig.IdentityToken == "" && authConfig.RegistryToken == "" {
|
||||||
return imagePullOptions, nil
|
return imagePullOptions, nil
|
||||||
}
|
}
|
||||||
logger.Info("using DockerAuthConfig authentication for docker pull")
|
logger.Info("using DockerAuthConfig authentication for docker pull")
|
||||||
|
|||||||
@@ -6,12 +6,17 @@ package container
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/iotest"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
|
"github.com/moby/moby/api/pkg/authconfig"
|
||||||
|
"github.com/moby/moby/api/types/registry"
|
||||||
assert "github.com/stretchr/testify/assert"
|
assert "github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -64,6 +69,21 @@ func TestGetImagePullOptions(t *testing.T) {
|
|||||||
})
|
})
|
||||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths":{"identity.example.com":{"identitytoken":"identity-token"},"registry.example.com":{"registrytoken":"registry-token"}}}`), 0o600))
|
||||||
|
t.Setenv("DOCKER_CONFIG", dir)
|
||||||
|
for host, want := range map[string]registry.AuthConfig{
|
||||||
|
"identity.example.com": {IdentityToken: "identity-token", ServerAddress: "identity.example.com"},
|
||||||
|
"registry.example.com": {RegistryToken: "registry-token", ServerAddress: "registry.example.com"},
|
||||||
|
} {
|
||||||
|
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{Image: host + "/app"})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, options.RegistryAuth, host)
|
||||||
|
got, err := authconfig.Decode(options.RegistryAuth)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, *got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A digest-pinned image is immutable, so its local copy is always current.
|
// A digest-pinned image is immutable, so its local copy is always current.
|
||||||
@@ -82,4 +102,8 @@ func TestLogDockerResponseError(t *testing.T) {
|
|||||||
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
||||||
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
||||||
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
||||||
|
require.NoError(t, streamErr(`{"stream":"`+strings.Repeat("x", 70000)+`"}`))
|
||||||
|
require.ErrorIs(t, logDockerResponse(logger, io.NopCloser(io.MultiReader(
|
||||||
|
strings.NewReader(`{"status":"Downloading"}`+"\n"), iotest.ErrReader(io.ErrUnexpectedEOF),
|
||||||
|
)), false), io.ErrUnexpectedEOF)
|
||||||
}
|
}
|
||||||
+35
-14
@@ -205,18 +205,30 @@ func (cr *containerReference) DumpLogs(ctx context.Context) error {
|
|||||||
if cr.id == "" {
|
if cr.id == "" {
|
||||||
return cr.missingContainerError("read its logs")
|
return cr.missingContainerError("read its logs")
|
||||||
}
|
}
|
||||||
|
tty, err := cr.containerTTY(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer logs.Close()
|
defer logs.Close()
|
||||||
return cr.copyOutput(logs)
|
return cr.copyOutput(logs, tty)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cr *containerReference) containerTTY(ctx context.Context) (bool, error) {
|
||||||
|
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("failed to inspect container: %w", err)
|
||||||
|
}
|
||||||
|
return inspect.Container.Config != nil && inspect.Container.Config.Tty, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// copyOutput writes a container stream to the writers the container was created with,
|
// copyOutput writes a container stream to the writers the container was created with,
|
||||||
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
|
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
|
||||||
func (cr *containerReference) copyOutput(stream io.Reader) error {
|
func (cr *containerReference) copyOutput(stream io.Reader, tty bool) error {
|
||||||
outWriter := cr.input.Stdout
|
outWriter := cr.input.Stdout
|
||||||
if outWriter == nil {
|
if outWriter == nil {
|
||||||
outWriter = os.Stdout
|
outWriter = os.Stdout
|
||||||
@@ -227,7 +239,7 @@ func (cr *containerReference) copyOutput(stream io.Reader) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if !cr.input.AllocatePTY || os.Getenv("NORAW") != "" {
|
if !tty || os.Getenv("NORAW") != "" {
|
||||||
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
|
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
|
||||||
} else {
|
} else {
|
||||||
_, err = io.Copy(outWriter, stream)
|
_, err = io.Copy(outWriter, stream)
|
||||||
@@ -296,7 +308,7 @@ func containerInfoFromInspect(inspect container.InspectResponse) *Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun)
|
return parseEnvFile(cr, srcPath, env, false).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||||
@@ -499,7 +511,7 @@ func (cr *containerReference) remove() common.Executor {
|
|||||||
if idOrName == "" && cr.input != nil {
|
if idOrName == "" && cr.input != nil {
|
||||||
idOrName = cr.input.Name
|
idOrName = cr.input.Name
|
||||||
}
|
}
|
||||||
if idOrName == "" {
|
if idOrName == "" || cr.cli == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -851,7 +863,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
cmdResponse := make(chan error, 1)
|
cmdResponse := make(chan error, 1)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
cmdResponse <- cr.copyOutput(resp.Reader)
|
cmdResponse <- cr.copyOutput(resp.Reader, cr.input.AllocatePTY)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -867,7 +879,10 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
// truncate the tail of the log. The goroutine exits once the hijacked
|
// truncate the tail of the log. The goroutine exits once the hijacked
|
||||||
// stream is closed by resp.Close() in the caller's defer.
|
// stream is closed by resp.Close() in the caller's defer.
|
||||||
select {
|
select {
|
||||||
case <-cmdResponse:
|
case err := <-cmdResponse:
|
||||||
|
if err != nil {
|
||||||
|
logger.Debugf("Failed to drain command output after cancellation: %v", err)
|
||||||
|
}
|
||||||
case <-time.After(drainGracePeriod):
|
case <-time.After(drainGracePeriod):
|
||||||
logger.Warn("Timed out draining command output after cancellation")
|
logger.Warn("Timed out draining command output after cancellation")
|
||||||
}
|
}
|
||||||
@@ -876,11 +891,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
// from executing
|
// from executing
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case err := <-cmdResponse:
|
case err := <-cmdResponse:
|
||||||
if err != nil {
|
return err
|
||||||
logger.Error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -897,7 +908,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
|||||||
}
|
}
|
||||||
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := tarFile.Close(); !errors.Is(err, os.ErrClosed) {
|
if err := tarFile.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||||
logger.Error(err)
|
logger.Error(err)
|
||||||
}
|
}
|
||||||
if err := os.Remove(tarFile.Name()); err != nil {
|
if err := os.Remove(tarFile.Name()); err != nil {
|
||||||
@@ -944,6 +955,10 @@ func (cr *containerReference) upload(ctx context.Context, dstPath string, conten
|
|||||||
|
|
||||||
func (cr *containerReference) attach() common.Executor {
|
func (cr *containerReference) attach() common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
|
tty, err := cr.containerTTY(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
||||||
Stream: true,
|
Stream: true,
|
||||||
Stdout: true,
|
Stdout: true,
|
||||||
@@ -954,9 +969,12 @@ func (cr *containerReference) attach() common.Executor {
|
|||||||
}
|
}
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
cr.attachDone = done
|
cr.attachDone = done
|
||||||
|
stopClose := context.AfterFunc(ctx, out.Close)
|
||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
if copyErr := cr.copyOutput(out.Reader); copyErr != nil {
|
defer out.Close()
|
||||||
|
defer stopClose()
|
||||||
|
if copyErr := cr.copyOutput(out.Reader, tty); copyErr != nil && ctx.Err() == nil {
|
||||||
common.Logger(ctx).Error(copyErr)
|
common.Logger(ctx).Error(copyErr)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -991,6 +1009,9 @@ func (cr *containerReference) wait() common.Executor {
|
|||||||
return fmt.Errorf("failed to wait for container: %w", err)
|
return fmt.Errorf("failed to wait for container: %w", err)
|
||||||
}
|
}
|
||||||
case status := <-waitResult.Result:
|
case status := <-waitResult.Result:
|
||||||
|
if status.Error != nil {
|
||||||
|
return fmt.Errorf("failed to wait for container: %s", status.Error.Message)
|
||||||
|
}
|
||||||
statusCode = status.StatusCode
|
statusCode = status.StatusCode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -92,6 +92,16 @@ func (m *mockDockerClient) ContainerAttach(ctx context.Context, containerID stri
|
|||||||
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
|
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockDockerClient) ContainerLogs(ctx context.Context, containerID string, opts mobyclient.ContainerLogsOptions) (mobyclient.ContainerLogsResult, error) {
|
||||||
|
args := m.Called(ctx, containerID, opts)
|
||||||
|
return args.Get(0).(mobyclient.ContainerLogsResult), args.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockDockerClient) ContainerStart(ctx context.Context, containerID string, opts mobyclient.ContainerStartOptions) (mobyclient.ContainerStartResult, error) {
|
||||||
|
args := m.Called(ctx, containerID, opts)
|
||||||
|
return args.Get(0).(mobyclient.ContainerStartResult), args.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
|
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
|
||||||
args := m.Called(ctx, containerID, opts)
|
args := m.Called(ctx, containerID, opts)
|
||||||
return args.Get(0).(mobyclient.ContainerWaitResult)
|
return args.Get(0).(mobyclient.ContainerWaitResult)
|
||||||
@@ -158,6 +168,7 @@ func (r *interruptReader) Read(_ []byte) (int, error) {
|
|||||||
type mockConn struct {
|
type mockConn struct {
|
||||||
net.Conn
|
net.Conn
|
||||||
mock.Mock
|
mock.Mock
|
||||||
|
closed atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *mockConn) Write(b []byte) (n int, err error) {
|
func (m *mockConn) Write(b []byte) (n int, err error) {
|
||||||
@@ -166,6 +177,7 @@ func (m *mockConn) Write(b []byte) (n int, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *mockConn) Close() (err error) {
|
func (m *mockConn) Close() (err error) {
|
||||||
|
m.closed.Store(true)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,6 +248,21 @@ func TestDockerExecFailure(t *testing.T) {
|
|||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDockerExecStreamFailure(t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).
|
||||||
|
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||||
|
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).
|
||||||
|
Return(mobyclient.ExecAttachResult{
|
||||||
|
Conn: &mockConn{},
|
||||||
|
Reader: bufio.NewReader(bytes.NewReader(stdcopyFrame(stdcopy.Systemerr, "unable to start exec"))),
|
||||||
|
}, nil)
|
||||||
|
require.ErrorContains(t, (&containerReference{id: "123", cli: client, input: &NewContainerInput{}}).
|
||||||
|
exec([]string{"sh"}, nil, "", "")(ctx), "unable to start exec")
|
||||||
|
client.AssertExpectations(t)
|
||||||
|
}
|
||||||
|
|
||||||
// stdcopyFrame wraps payload in a single Docker multiplexed-stream frame, the
|
// stdcopyFrame wraps payload in a single Docker multiplexed-stream frame, the
|
||||||
// format StdCopy expects: an 8-byte header (stream type + 4-byte big-endian
|
// format StdCopy expects: an 8-byte header (stream type + 4-byte big-endian
|
||||||
// length) followed by the payload.
|
// length) followed by the payload.
|
||||||
@@ -248,60 +275,88 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||||
ctx := t.Context()
|
for _, testcase := range []struct {
|
||||||
|
name string
|
||||||
|
tty bool
|
||||||
|
}{
|
||||||
|
{name: "multiplexed"},
|
||||||
|
{name: "tty from container options", tty: true},
|
||||||
|
} {
|
||||||
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
output := []byte("line one\nlast line without newline")
|
||||||
|
if !testcase.tty {
|
||||||
|
output = stdcopyFrame(stdcopy.Stdout, string(output))
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
logWriter := common.NewLineWriter(func(line string) bool {
|
||||||
|
lines = append(lines, line)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Tty: testcase.tty}}}, nil)
|
||||||
|
conn := &mockConn{}
|
||||||
|
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||||
|
Return(mobyclient.ContainerAttachResult{
|
||||||
|
Conn: conn,
|
||||||
|
Reader: bufio.NewReader(bytes.NewReader(output)),
|
||||||
|
}, nil)
|
||||||
|
client.On("ContainerLogs", ctx, "123", mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}).
|
||||||
|
Return(io.NopCloser(bytes.NewReader(output)), nil)
|
||||||
|
statusCh := make(chan container.WaitResponse, 1)
|
||||||
|
statusCh <- container.WaitResponse{StatusCode: 0}
|
||||||
|
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||||
|
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||||
|
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{Stdout: logWriter, Stderr: logWriter}}
|
||||||
|
|
||||||
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
|
require.NoError(t, cr.attach()(ctx))
|
||||||
|
require.NoError(t, cr.wait()(ctx))
|
||||||
var lines []string
|
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
||||||
logWriter := common.NewLineWriter(func(s string) bool {
|
assert.True(t, conn.closed.Load())
|
||||||
lines = append(lines, s)
|
require.NoError(t, cr.DumpLogs(ctx))
|
||||||
return true
|
assert.Equal(t, []string{"line one\n", "last line without newline", "line one\n", "last line without newline"}, lines)
|
||||||
})
|
client.AssertExpectations(t)
|
||||||
|
})
|
||||||
client := &mockDockerClient{}
|
|
||||||
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
|
||||||
Return(mobyclient.ContainerAttachResult{
|
|
||||||
Conn: &mockConn{},
|
|
||||||
Reader: bufio.NewReader(framed),
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
statusCh := make(chan container.WaitResponse, 1)
|
|
||||||
statusCh <- container.WaitResponse{StatusCode: 0}
|
|
||||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
|
||||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
|
||||||
|
|
||||||
cr := &containerReference{
|
|
||||||
id: "123",
|
|
||||||
cli: client,
|
|
||||||
input: &NewContainerInput{
|
|
||||||
Stdout: logWriter,
|
|
||||||
Stderr: logWriter,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
require.NoError(t, cr.attach()(ctx))
|
func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
|
||||||
require.NoError(t, cr.wait()(ctx))
|
conn, peer := net.Pipe()
|
||||||
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
t.Cleanup(func() { _ = peer.Close() })
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ContainerInspect", mock.Anything, "123", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{}}}, nil)
|
||||||
|
client.On("ContainerAttach", mock.Anything, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||||
|
Return(mobyclient.ContainerAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
|
||||||
|
client.On("ContainerStart", mock.Anything, "123", mobyclient.ContainerStartOptions{}).
|
||||||
|
Return(mobyclient.ContainerStartResult{}, errors.New("container start failed"))
|
||||||
|
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{}}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
require.ErrorContains(t, cr.Start(true)(ctx), "container start failed")
|
||||||
|
cancel()
|
||||||
|
<-cr.attachDone
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerWaitFailure(t *testing.T) {
|
func TestDockerWaitFailure(t *testing.T) {
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
statusCh := make(chan container.WaitResponse, 1)
|
statusCh := make(chan container.WaitResponse, 1)
|
||||||
statusCh <- container.WaitResponse{StatusCode: 2}
|
statusCh <- container.WaitResponse{StatusCode: 2}
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||||
|
cr := &containerReference{id: "123", cli: client}
|
||||||
|
|
||||||
err := (&containerReference{id: "123", cli: client}).wait()(ctx)
|
err := cr.wait()(ctx)
|
||||||
var exitErr ExitCodeError
|
var exitErr ExitCodeError
|
||||||
require.ErrorAs(t, err, &exitErr)
|
require.ErrorAs(t, err, &exitErr)
|
||||||
assert.Equal(t, ExitCodeError(2), exitErr)
|
assert.Equal(t, ExitCodeError(2), exitErr)
|
||||||
assert.Equal(t, "Process completed with exit code 2.", err.Error())
|
assert.Equal(t, "Process completed with exit code 2.", err.Error())
|
||||||
|
statusCh <- container.WaitResponse{Error: &container.WaitExitError{Message: "container wait failed"}}
|
||||||
|
require.EqualError(t, cr.wait()(ctx), "failed to wait for container: container wait failed")
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,6 +425,13 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
|
|||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemovePreservesClientCreationError(t *testing.T) {
|
||||||
|
t.Setenv("DOCKER_HOST", "://")
|
||||||
|
|
||||||
|
require.ErrorContains(t, (&containerReference{input: &NewContainerInput{Name: "job-1"}}).
|
||||||
|
Remove()(t.Context()), "failed to connect to docker daemon")
|
||||||
|
}
|
||||||
|
|
||||||
func TestFindRevalidatesStaleID(t *testing.T) {
|
func TestFindRevalidatesStaleID(t *testing.T) {
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||||
@@ -477,7 +539,8 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
|||||||
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
|
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
|
||||||
} {
|
} {
|
||||||
t.Run(testcase.name, func(t *testing.T) {
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
ctx := t.Context()
|
hook := logtest.NewWithLevel(logging.LevelError)
|
||||||
|
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
||||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
||||||
@@ -501,6 +564,7 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
|||||||
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||||
}
|
}
|
||||||
assert.Equal(t, "fresh", cr.id)
|
assert.Equal(t, "fresh", cr.id)
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -916,7 +980,7 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
|||||||
RunnerOptions: "--entrypoint=runner-entrypoint",
|
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||||
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||||
},
|
},
|
||||||
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
if len(entrypoint) > 0 {
|
if len(entrypoint) > 0 {
|
||||||
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ func TestSSHArgs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
|
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if _, err := exec.LookPath("sh"); err != nil {
|
if _, err := exec.LookPath("sh"); err != nil {
|
||||||
t.Skip("needs a POSIX shell")
|
t.Skip("needs a POSIX shell")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -225,12 +225,16 @@ func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
|
|||||||
return ppty, tty, nil
|
return ppty, tty, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeKeepAlive(ppty io.Writer) {
|
func writeKeepAlive(ctx context.Context, ppty io.Writer) {
|
||||||
c := 1
|
for ctx.Err() == nil {
|
||||||
var err error
|
if count, err := ppty.Write([]byte{4}); count != 1 || err != nil {
|
||||||
for c == 1 && err == nil {
|
return
|
||||||
c, err = ppty.Write([]byte{4})
|
}
|
||||||
<-time.After(time.Second)
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -302,7 +306,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
|||||||
defer finishLog()
|
defer finishLog()
|
||||||
_, _ = io.Copy(writer, ppty)
|
_, _ = io.Copy(writer, ppty)
|
||||||
}()
|
}()
|
||||||
go writeKeepAlive(ppty)
|
go writeKeepAlive(logctx, ppty)
|
||||||
}
|
}
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -318,21 +322,21 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
|||||||
defer k.Close()
|
defer k.Close()
|
||||||
}
|
}
|
||||||
err = treeKill.Wait(ctx, cmd)
|
err = treeKill.Wait(ctx, cmd)
|
||||||
if err != nil {
|
|
||||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
|
||||||
return ExitCodeError(exitErr.ExitCode())
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if tty != nil {
|
if tty != nil {
|
||||||
writer.AutoStop.Store(true)
|
writer.AutoStop.Store(true)
|
||||||
if _, err := tty.WriteString("\x04"); err != nil {
|
if _, err := tty.WriteString("\x04"); err != nil {
|
||||||
common.Logger(ctx).Debug("Failed to write EOT")
|
common.Logger(ctx).Debug("Failed to write EOT")
|
||||||
}
|
}
|
||||||
<-logctx.Done()
|
select {
|
||||||
|
case <-logctx.Done():
|
||||||
|
case <-ctx.Done(): // a cancelled step does not wait on output nobody consumes
|
||||||
|
}
|
||||||
ppty.Close()
|
ppty.Close()
|
||||||
ppty = nil
|
ppty = nil
|
||||||
}
|
}
|
||||||
|
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||||
|
return ExitCodeError(exitErr.ExitCode())
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -355,7 +359,7 @@ func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(e, srcPath, env)
|
return parseEnvFile(e, srcPath, env, runtime.GOOS == "windows")
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeAll is a var so tests can substitute a blocking stub.
|
// removeAll is a var so tests can substitute a blocking stub.
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
@@ -79,20 +80,48 @@ func TestGetContainerArchive(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
ctx := t.Context()
|
for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
|
||||||
e := &HostEnvironment{Path: t.TempDir(), StdOut: io.Discard}
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
reader, writer := io.Pipe()
|
||||||
|
defer reader.Close()
|
||||||
|
e := &HostEnvironment{Path: t.TempDir(), StdOut: writer, AllocatePTY: allocatePTY}
|
||||||
|
t.Cleanup(func() { assert.NoError(t, e.Remove()(t.Context())) })
|
||||||
|
|
||||||
err := e.Exec([]string{"sh", "-c", "exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(ctx)
|
result := make(chan error, 1)
|
||||||
var exitErr ExitCodeError
|
go func() {
|
||||||
require.ErrorAs(t, err, &exitErr)
|
defer writer.Close()
|
||||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
result <- e.Exec([]string{"sh", "-c", "printf 'failed\\n'; exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(t.Context())
|
||||||
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
}()
|
||||||
|
first := make([]byte, 1)
|
||||||
|
_, err := io.ReadFull(reader, first)
|
||||||
|
require.NoError(t, err)
|
||||||
|
synctest.Wait()
|
||||||
|
select {
|
||||||
|
case err := <-result:
|
||||||
|
t.Fatalf("Exec returned before its output was consumed: %v", err)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
remaining, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = <-result
|
||||||
|
var exitErr ExitCodeError
|
||||||
|
require.ErrorAs(t, err, &exitErr)
|
||||||
|
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||||
|
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
||||||
|
assert.Equal(t, "failed\n", string(first)+string(remaining))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
@@ -105,6 +134,7 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
|||||||
{name: "on", allocPTY: true, expect: "TTY"},
|
{name: "on", allocPTY: true, expect: "TTY"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
e := &HostEnvironment{
|
e := &HostEnvironment{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
@@ -146,63 +176,67 @@ func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||||
release := make(chan struct{})
|
synctest.Test(t, func(t *testing.T) {
|
||||||
stubDone := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
stubDone := make(chan struct{})
|
||||||
|
|
||||||
orig := removeAll
|
orig := removeAll
|
||||||
removeAll = func(string) error {
|
removeAll = func(string) error {
|
||||||
defer close(stubDone)
|
defer close(stubDone)
|
||||||
<-release
|
<-release
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
close(release)
|
close(release)
|
||||||
<-stubDone
|
<-stubDone
|
||||||
removeAll = orig
|
removeAll = orig
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := removeAllWithContext(ctx, t.TempDir())
|
||||||
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
})
|
})
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := removeAllWithContext(ctx, t.TempDir())
|
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||||
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
||||||
t.Run(phase, func(t *testing.T) {
|
t.Run(phase, func(t *testing.T) {
|
||||||
origTimeout := hostCleanupTimeout
|
synctest.Test(t, func(t *testing.T) {
|
||||||
hostCleanupTimeout = 100 * time.Millisecond
|
origTimeout := hostCleanupTimeout
|
||||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
hostCleanupTimeout = 100 * time.Millisecond
|
||||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||||
block := func() {
|
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||||
defer close(stubDone)
|
block := func() {
|
||||||
<-release
|
defer close(stubDone)
|
||||||
}
|
<-release
|
||||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
}
|
||||||
if phase == "CleanUp callback" {
|
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||||
e.CleanUp = block
|
if phase == "CleanUp callback" {
|
||||||
} else {
|
e.CleanUp = block
|
||||||
origRemoveAll := removeAll
|
} else {
|
||||||
removeAll = func(string) error {
|
origRemoveAll := removeAll
|
||||||
block()
|
removeAll = func(string) error {
|
||||||
return nil
|
block()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { removeAll = origRemoveAll })
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
close(release)
|
||||||
|
<-stubDone
|
||||||
|
})
|
||||||
|
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("Remove() hung on a stuck %s", phase)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { removeAll = origRemoveAll })
|
|
||||||
}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(release)
|
|
||||||
<-stubDone
|
|
||||||
})
|
})
|
||||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatalf("Remove() hung on a stuck %s", phase)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+96
-11
@@ -7,12 +7,15 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
"maps"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -23,6 +26,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
"github.com/kballard/go-shellquote"
|
"github.com/kballard/go-shellquote"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
@@ -36,6 +40,7 @@ type KubernetesOptions struct {
|
|||||||
MaxLifetime time.Duration
|
MaxLifetime time.Duration
|
||||||
ForcePull bool
|
ForcePull bool
|
||||||
RunnerUUID string
|
RunnerUUID string
|
||||||
|
Transfer *KubernetesTransfer
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubernetesPod runs a job container and its service containers as one pod.
|
// KubernetesPod runs a job container and its service containers as one pod.
|
||||||
@@ -46,6 +51,10 @@ type KubernetesPod struct {
|
|||||||
job *kubernetesContainer
|
job *kubernetesContainer
|
||||||
services []*kubernetesContainer
|
services []*kubernetesContainer
|
||||||
arch, path string
|
arch, path string
|
||||||
|
ips []net.IP
|
||||||
|
transfer *transferServer
|
||||||
|
transferImage string
|
||||||
|
transferHelper string // empty when copies go through the API server
|
||||||
}
|
}
|
||||||
|
|
||||||
type kubernetesContainer struct {
|
type kubernetesContainer struct {
|
||||||
@@ -55,6 +64,7 @@ type kubernetesContainer struct {
|
|||||||
name string
|
name string
|
||||||
input *NewContainerInput
|
input *NewContainerInput
|
||||||
probe map[string]any
|
probe map[string]any
|
||||||
|
gzip bool // the container can gunzip, so copies into it go compressed
|
||||||
}
|
}
|
||||||
|
|
||||||
type containerStatus struct {
|
type containerStatus struct {
|
||||||
@@ -74,6 +84,7 @@ type podStatus struct {
|
|||||||
Spec struct{ Containers []struct{} }
|
Spec struct{ Containers []struct{} }
|
||||||
Status struct {
|
Status struct {
|
||||||
Phase, Reason, Message string
|
Phase, Reason, Message string
|
||||||
|
PodIPs []struct{ IP string }
|
||||||
Conditions []struct{ Type, Status, Message string }
|
Conditions []struct{ Type, Status, Message string }
|
||||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||||
}
|
}
|
||||||
@@ -129,14 +140,24 @@ func (p *KubernetesPod) create(ctx context.Context) error {
|
|||||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||||
}
|
}
|
||||||
metadata := map[string]any{"name": name, "labels": labels}
|
metadata := map[string]any{"name": name, "labels": labels}
|
||||||
|
var runnerImage string
|
||||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||||
var runner struct{ Metadata struct{ Name, UID string } }
|
var runner struct {
|
||||||
|
Metadata struct{ Name, UID string }
|
||||||
|
Spec struct{ Containers []struct{ Image string } }
|
||||||
|
}
|
||||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||||
|
if len(runner.Spec.Containers) == 1 {
|
||||||
|
runnerImage = runner.Spec.Containers[0].Image
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := p.setupTransfer(runnerImage); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
secretData := map[string]string{}
|
secretData := map[string]string{}
|
||||||
manifest, err := p.manifest(name, secretData, metadata)
|
manifest, err := p.manifest(name, secretData, metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -205,6 +226,18 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||||
|
if p.transferHelper != "" {
|
||||||
|
transfer := map[string]any{
|
||||||
|
"name": "gitea-runner-transfer", "image": p.transferImage, "terminationMessagePolicy": "FallbackToLogsOnError",
|
||||||
|
"command": []string{"gitea-runner", KubernetesTransferCommand, "install", p.transferHelper},
|
||||||
|
"volumeMounts": []any{map[string]any{"name": "act", "mountPath": p.job.GetActPath()}},
|
||||||
|
"securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}},
|
||||||
|
}
|
||||||
|
if p.options.ForcePull {
|
||||||
|
transfer["imagePullPolicy"] = "Always"
|
||||||
|
}
|
||||||
|
spec["initContainers"] = []any{transfer}
|
||||||
|
}
|
||||||
if len(hostnames) > 0 {
|
if len(hostnames) > 0 {
|
||||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||||
}
|
}
|
||||||
@@ -341,7 +374,13 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if pod.Status.Phase == "Failed" {
|
if pod.Status.Phase == "Failed" {
|
||||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
reason := cmp.Or(pod.Status.Message, pod.Status.Reason)
|
||||||
|
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||||
|
if terminated := status.State.Terminated; reason == "" && terminated != nil && terminated.ExitCode != 0 {
|
||||||
|
reason = fmt.Sprintf("container %s exited with %d: %s", status.Name, terminated.ExitCode, strings.TrimSpace(terminated.Message))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pod %s failed: %s", p.name, reason)
|
||||||
}
|
}
|
||||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||||
@@ -354,6 +393,9 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if started {
|
if started {
|
||||||
|
for _, ip := range pod.Status.PodIPs {
|
||||||
|
p.ips = append(p.ips, net.ParseIP(ip.IP))
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
for _, condition := range pod.Status.Conditions {
|
for _, condition := range pod.Status.Conditions {
|
||||||
@@ -391,8 +433,11 @@ func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
machine, path, _ := strings.Cut(output, "\n")
|
probed := strings.SplitN(output, "\n", 3)
|
||||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
for len(probed) < 3 {
|
||||||
|
probed = append(probed, "")
|
||||||
|
}
|
||||||
|
c.pod.arch, c.gzip, c.pod.path = goArchToActionArch(probed[0]), probed[1] == "gzip", probed[2]
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -403,7 +448,7 @@ func (p *KubernetesPod) probe(ctx context.Context, container string) (string, er
|
|||||||
deadline := time.Now().Add(time.Minute)
|
deadline := time.Now().Add(time.Minute)
|
||||||
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
|
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard)
|
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s\n%s' "$(uname -m)" "$(command -v gzip >/dev/null && echo gzip)" "$PATH"`}, nil, &stdout, io.Discard)
|
||||||
var exitErr ExitCodeError
|
var exitErr ExitCodeError
|
||||||
var apiErr *apiError
|
var apiErr *apiError
|
||||||
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
|
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
|
||||||
@@ -452,26 +497,66 @@ func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, w
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
// compressCopyMin is the size from which Copy compresses and uses the transfer helper: below it that saves next to nothing on the wire and costs a process in the pod.
|
||||||
|
const compressCopyMin = 64 << 10
|
||||||
|
|
||||||
|
// extract streams the tar write produces into destPath, gzipped with compress, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||||
|
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, compress bool, write func(io.Writer) error) error {
|
||||||
|
if compress && c.pod.transferHelper != "" {
|
||||||
|
request, done := c.pod.transfer.offer(c.pod.ips, write)
|
||||||
|
stdin, _ := json.Marshal(request)
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{c.pod.transferHelper, KubernetesTransferCommand, "fetch", destPath}, bytes.NewReader(stdin), io.Discard, &stderr)
|
||||||
|
taken, writeErr := done()
|
||||||
|
if err == nil || ctx.Err() != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if taken || !c.pod.options.Transfer.Fallback {
|
||||||
|
return fmt.Errorf("copy to %s over the pod network: %w: %s", destPath, errors.Join(writeErr, err), strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
c.pod.transferHelper = "" // a pod that cannot reach the runner once will not later
|
||||||
|
logging.Warnf("Copying into pod %s over the pod network failed, copying through the API server instead: %v: %s", c.pod.name, err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
compress = compress && c.gzip
|
||||||
reader, writer := io.Pipe()
|
reader, writer := io.Pipe()
|
||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
go func() { writer.CloseWithError(write(writer)) }()
|
go func() {
|
||||||
|
if compress {
|
||||||
|
writer.CloseWithError(writeGzip(writer, write))
|
||||||
|
} else {
|
||||||
|
writer.CloseWithError(write(writer))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
script := `mkdir -p "$1" && tar -xf - -C "$1"`
|
||||||
|
if compress {
|
||||||
|
script = `mkdir -p "$1" && gzip -dc | tar -xf - -C "$1"` // gzip -dc rather than tar -z, which GNU tar runs gzip for anyway
|
||||||
|
}
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", script, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeGzip gzips what write produces into w, at a level fast enough to not slow copies down and nearly as small as the default.
|
||||||
|
func writeGzip(w io.Writer, write func(io.Writer) error) error {
|
||||||
|
compressed, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||||
|
return errors.Join(write(compressed), compressed.Close())
|
||||||
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
size := 0
|
||||||
|
for _, file := range files {
|
||||||
|
size += len(file.Body)
|
||||||
|
}
|
||||||
|
return c.extract(ctx, destPath, size >= compressCopyMin, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -500,7 +585,7 @@ func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath s
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(c, srcPath, env)
|
return parseEnvFile(c, srcPath, env, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func execFailure(t *testing.T, code int, status string) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
func fakeKubernetes(t *testing.T, podStatus string, env ...string) (string, *fakeCluster) {
|
||||||
cluster := &fakeCluster{requests: map[string][]string{}}
|
cluster := &fakeCluster{requests: map[string][]string{}}
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||||
@@ -109,7 +109,7 @@ func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
|||||||
if failure := cluster.nextExecFailure(); failure != nil {
|
if failure := cluster.nextExecFailure(); failure != nil {
|
||||||
failure(w, r)
|
failure(w, r)
|
||||||
} else {
|
} else {
|
||||||
serveLocalExec(t, w, r)
|
serveLocalExec(t, w, r, env)
|
||||||
}
|
}
|
||||||
case strings.HasSuffix(r.URL.Path, "/log"):
|
case strings.HasSuffix(r.URL.Path, "/log"):
|
||||||
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
||||||
@@ -132,7 +132,7 @@ users: [{name: test, user: {tokenFile: token}}]
|
|||||||
return kubeconfig, cluster
|
return kubeconfig, cluster
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request, env []string) {
|
||||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
@@ -154,7 +154,7 @@ func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
|||||||
stdin.Write(frame[1:])
|
stdin.Write(frame[1:])
|
||||||
}
|
}
|
||||||
cmd := exec.Command(command[0], command[1:]...)
|
cmd := exec.Command(command[0], command[1:]...)
|
||||||
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
|
cmd.Env = append(append(os.Environ(), "COPYFILE_DISABLE=1"), env...)
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
||||||
status := `{"status":"Success"}`
|
status := `{"status":"Success"}`
|
||||||
@@ -183,15 +183,17 @@ func TestKubernetesServiceHostnames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
|
||||||
{"name":"job","ready":true,"state":{"running":{}}},
|
|
||||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
|
|
||||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
path := workdir + ":" + os.Getenv("PATH")
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}},
|
||||||
|
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`, "PATH="+path)
|
||||||
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
||||||
{"spec": map[string]any{
|
{"spec": map[string]any{
|
||||||
"automountServiceAccountToken": true,
|
"automountServiceAccountToken": true,
|
||||||
@@ -252,13 +254,13 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
||||||
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
|
|
||||||
cluster.failExec(
|
cluster.failExec(
|
||||||
execFailure(t, http.StatusInternalServerError, `{"message":"error dialing backend: remote error: tls: internal error"}`),
|
execFailure(t, http.StatusInternalServerError, `{"message":"error dialing backend: remote error: tls: internal error"}`),
|
||||||
execFailure(t, 0, `{"status":"Failure","reason":"InternalError","message":"Internal error occurred: error sending request: remote error: tls: internal error"}`),
|
execFailure(t, 0, `{"status":"Failure","reason":"InternalError","message":"Internal error occurred: error sending request: remote error: tls: internal error"}`),
|
||||||
)
|
)
|
||||||
require.NoError(t, job.Start(false)(ctx))
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
assert.Len(t, cluster.requests["exec"], 3)
|
assert.Len(t, cluster.requests["exec"], 3)
|
||||||
|
assert.True(t, pod.job.gzip)
|
||||||
|
|
||||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||||
@@ -275,7 +277,9 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.NotContains(t, query, "hunter2")
|
assert.NotContains(t, query, "hunter2")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||||
|
assert.NotContains(t, lastExec(), "gzip")
|
||||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
reader := tar.NewReader(archive)
|
reader := tar.NewReader(archive)
|
||||||
@@ -288,6 +292,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
require.NoError(t, archive.Close())
|
require.NoError(t, archive.Close())
|
||||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
large := strings.Repeat("a", compressCopyMin)
|
||||||
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/large", Mode: 0o644, Body: large})(ctx))
|
||||||
|
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||||
|
content, err = os.ReadFile(workdir + "/act/workflow/large")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, large, string(content))
|
||||||
|
|
||||||
actionDir := filepath.Join(t.TempDir(), "action")
|
actionDir := filepath.Join(t.TempDir(), "action")
|
||||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||||
@@ -296,11 +306,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "main()", string(script))
|
assert.Equal(t, "main()", string(script))
|
||||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
assert.Contains(t, lastExec(), "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||||
|
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||||
|
|
||||||
pathEnv := map[string]string{}
|
pathEnv := map[string]string{}
|
||||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
assert.Equal(t, path, pathEnv["PATH"])
|
||||||
|
|
||||||
info, err := service.Inspect(ctx)
|
info, err := service.Inspect(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -320,6 +331,38 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKubernetesCopiesUncompressedWithoutGzip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("uses POSIX shell")
|
||||||
|
}
|
||||||
|
ctx := t.Context()
|
||||||
|
workdir, bin := t.TempDir(), t.TempDir()
|
||||||
|
for _, tool := range []string{"uname", "tar", "mkdir"} { // all the job container needs but gzip, as on a minimal image
|
||||||
|
found, err := exec.LookPath(tool)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Symlink(found, filepath.Join(bin, tool)))
|
||||||
|
}
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}}]}}`, "PATH="+bin)
|
||||||
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job := pod.JobContainer(&NewContainerInput{Image: "node", WorkingDir: workdir})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||||
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
|
assert.False(t, pod.job.gzip)
|
||||||
|
assert.Equal(t, bin, pod.path)
|
||||||
|
|
||||||
|
actionDir := filepath.Join(t.TempDir(), "action")
|
||||||
|
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||||
|
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||||
|
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "main()", string(script))
|
||||||
|
assert.NotContains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "gzip")
|
||||||
|
}
|
||||||
|
|
||||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||||
@@ -360,6 +403,13 @@ func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
|||||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||||
|
failed, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Failed","initContainerStatuses":[
|
||||||
|
{"name":"gitea-runner-transfer","state":{"terminated":{"exitCode":1,"message":"exec format error\n"}}}]}}`)
|
||||||
|
pod, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: failed})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job = pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||||
|
require.ErrorContains(t, job.Start(false)(t.Context()), "container gitea-runner-transfer exited with 1: exec format error")
|
||||||
|
|
||||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"context"
|
||||||
|
"crypto/ecdsa"
|
||||||
|
"crypto/elliptic"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/big"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KubernetesTransfer has job pods fetch large copies from the runner over the pod network instead of through the API server.
|
||||||
|
type KubernetesTransfer struct {
|
||||||
|
Host string // empty uses the outbound IP
|
||||||
|
Port uint16 // 0 picks a free port
|
||||||
|
Image string // has gitea-runner on its PATH, for an init container to copy into each job pod
|
||||||
|
Fallback bool // copy through the API server when a job pod cannot reach the runner, else fail
|
||||||
|
}
|
||||||
|
|
||||||
|
// KubernetesTransferCommand is the hidden runner subcommand job pods run the transfer helper with.
|
||||||
|
const KubernetesTransferCommand = "kubernetes-transfer"
|
||||||
|
|
||||||
|
// transferRequest goes to the helper on the exec stream's stdin, keeping the token out of the command line and the audit log.
|
||||||
|
type transferRequest struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Pin string `json:"pin"` // hex SHA-256 of the runner's certificate
|
||||||
|
}
|
||||||
|
|
||||||
|
type transferServer struct {
|
||||||
|
base string
|
||||||
|
pin string
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]transferOffer
|
||||||
|
}
|
||||||
|
|
||||||
|
type transferOffer struct {
|
||||||
|
clients []net.IP
|
||||||
|
write func(io.Writer) error
|
||||||
|
result chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
transferServersMu sync.Mutex
|
||||||
|
transferServers = map[string]*transferServer{}
|
||||||
|
)
|
||||||
|
|
||||||
|
// startTransferServer returns the runner's server for these settings, started on first use.
|
||||||
|
func startTransferServer(host string, port uint16) (*transferServer, error) {
|
||||||
|
transferServersMu.Lock()
|
||||||
|
defer transferServersMu.Unlock()
|
||||||
|
configured := net.JoinHostPort(host, strconv.Itoa(int(port)))
|
||||||
|
if server, ok := transferServers[configured]; ok {
|
||||||
|
return server, nil
|
||||||
|
}
|
||||||
|
if host == "" {
|
||||||
|
ip := common.GetOutboundIP()
|
||||||
|
if ip == nil {
|
||||||
|
return nil, errors.New("no outbound IP for job pods to reach the runner at, set kubernetes.transfer.host")
|
||||||
|
}
|
||||||
|
host = ip.String()
|
||||||
|
}
|
||||||
|
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
template := &x509.Certificate{SerialNumber: big.NewInt(1)} // trusted by its pin alone, so no CA, names or validity
|
||||||
|
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port))))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listen for job pod transfers: %w", err)
|
||||||
|
}
|
||||||
|
pin := sha256.Sum256(certificate)
|
||||||
|
server := &transferServer{base: "https://" + listener.Addr().String() + "/", pin: hex.EncodeToString(pin[:]), pending: map[string]transferOffer{}}
|
||||||
|
httpServer := &http.Server{
|
||||||
|
Handler: server,
|
||||||
|
TLSConfig: &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{certificate}, PrivateKey: privateKey}}, MinVersion: tls.VersionTLS13},
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
IdleTimeout: 30 * time.Second,
|
||||||
|
}
|
||||||
|
go func() { _ = httpServer.ServeTLS(listener, "", "") }()
|
||||||
|
transferServers[configured] = server
|
||||||
|
return server, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// offer serves write once to one of clients until done, which reports whether a client took it and what write failed with.
|
||||||
|
func (s *transferServer) offer(clients []net.IP, write func(io.Writer) error) (request transferRequest, done func() (taken bool, err error)) {
|
||||||
|
token := rand.Text()
|
||||||
|
result := make(chan error, 1)
|
||||||
|
s.mu.Lock()
|
||||||
|
s.pending[token] = transferOffer{clients: clients, write: write, result: result}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return transferRequest{URL: s.base + token, Pin: s.pin}, func() (bool, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
_, pending := s.pending[token]
|
||||||
|
delete(s.pending, token)
|
||||||
|
s.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case err := <-result:
|
||||||
|
return true, err
|
||||||
|
default:
|
||||||
|
return !pending, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *transferServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
token := strings.TrimPrefix(r.URL.Path, "/")
|
||||||
|
s.mu.Lock()
|
||||||
|
offer, ok := s.pending[token]
|
||||||
|
ok = ok && r.Method == http.MethodGet && slices.ContainsFunc(offer.clients, net.ParseIP(host).Equal)
|
||||||
|
if ok {
|
||||||
|
delete(s.pending, token)
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Encoding", "gzip")
|
||||||
|
err := writeGzip(w, offer.write)
|
||||||
|
offer.result <- err
|
||||||
|
if err != nil {
|
||||||
|
panic(http.ErrAbortHandler) // a cut-off body fails the fetch, a short tar might not
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FetchKubernetesTransfer reads a transferRequest from request and extracts the tar it points to into dest.
|
||||||
|
func FetchKubernetesTransfer(ctx context.Context, request io.Reader, dest string) error {
|
||||||
|
var transfer transferRequest
|
||||||
|
if err := json.NewDecoder(request).Decode(&transfer); err != nil {
|
||||||
|
return fmt.Errorf("read transfer request: %w", err)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: &http.Transport{
|
||||||
|
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||||
|
TLSHandshakeTimeout: 10 * time.Second,
|
||||||
|
ResponseHeaderTimeout: time.Minute,
|
||||||
|
TLSClientConfig: &tls.Config{
|
||||||
|
MinVersion: tls.VersionTLS13,
|
||||||
|
InsecureSkipVerify: true, // VerifyConnection checks the pin instead
|
||||||
|
VerifyConnection: func(state tls.ConnectionState) error {
|
||||||
|
if sum := sha256.Sum256(state.PeerCertificates[0].Raw); hex.EncodeToString(sum[:]) != transfer.Pin {
|
||||||
|
return errors.New("runner certificate does not match the pin")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, transfer.URL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
response, err := client.Do(httpRequest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
return fmt.Errorf("fetch: %s", response.Status)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dest, 0o777); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", dest)
|
||||||
|
cmd.Stdin, cmd.Stdout, cmd.Stderr = response.Body, os.Stderr, os.Stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(io.Discard, response.Body) // tar stops at the first zero blocks, a cut-off body after them still fails
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// InstallKubernetesTransferHelper copies the running binary to target.
|
||||||
|
func InstallKubernetesTransferHelper(target string) error {
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source, err := os.Open(executable)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = os.Remove(target) // a file left in a reused act volume would keep its mode
|
||||||
|
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, source)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupTransfer turns on transfers for the pod, with the runner pod's image unless one is set.
|
||||||
|
func (p *KubernetesPod) setupTransfer(runnerImage string) error {
|
||||||
|
settings := p.options.Transfer
|
||||||
|
if settings == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
image := cmp.Or(settings.Image, runnerImage)
|
||||||
|
var err error
|
||||||
|
var server *transferServer
|
||||||
|
if image == "" {
|
||||||
|
err = errors.New("set kubernetes.transfer.image, the runner is no single-container pod in kubernetes.namespace to take it from")
|
||||||
|
} else {
|
||||||
|
server, err = startTransferServer(settings.Host, settings.Port)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if !settings.Fallback {
|
||||||
|
return fmt.Errorf("kubernetes.transfer: %w", err)
|
||||||
|
}
|
||||||
|
logging.Warnf("Job pod copies go through the API server: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.transfer, p.transferImage = server, image
|
||||||
|
p.transferHelper = path.Join(p.job.GetActPath(), "bin", "gitea-runner")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/tls"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
const transferHelperEnv = "GITEA_RUNNER_TEST_TRANSFER_HELPER"
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
if os.Getenv(transferHelperEnv) == "1" && len(os.Args) == 4 && os.Args[1] == KubernetesTransferCommand && os.Args[2] == "fetch" {
|
||||||
|
if err := FetchKubernetesTransfer(context.Background(), os.Stdin, os.Args[3]); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetch(t *testing.T, request transferRequest) error {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(request)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return FetchKubernetesTransfer(t.Context(), bytes.NewReader(body), t.TempDir())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTransferServer(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("extracts with tar")
|
||||||
|
}
|
||||||
|
server, err := startTransferServer("127.0.0.1", 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
again, err := startTransferServer("127.0.0.1", 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Same(t, server, again)
|
||||||
|
loopback := []net.IP{net.ParseIP("127.0.0.1")}
|
||||||
|
nothing := func(io.Writer) error { return nil }
|
||||||
|
|
||||||
|
t.Run("serves an offer gzipped once to an IP of its job pod", func(t *testing.T) {
|
||||||
|
request, done := server.offer([]net.IP{net.ParseIP("fd00::1"), loopback[0]}, nothing)
|
||||||
|
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableCompression: true}}
|
||||||
|
response, err := client.Get(request.URL)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, response.Body.Close())
|
||||||
|
assert.Equal(t, "gzip", response.Header.Get("Content-Encoding"))
|
||||||
|
require.ErrorContains(t, fetch(t, request), "404")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.True(t, taken)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("another pod can neither take nor block an offer", func(t *testing.T) {
|
||||||
|
request, done := server.offer([]net.IP{net.ParseIP("10.1.2.3")}, nothing)
|
||||||
|
require.ErrorContains(t, fetch(t, request), "404")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.False(t, taken)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a failed write fails the fetch even past the end blocks tar stops at, and done reports it", func(t *testing.T) {
|
||||||
|
request, done := server.offer(loopback, func(w io.Writer) error {
|
||||||
|
_, _ = w.Write(make([]byte, 1024))
|
||||||
|
_, _ = io.CopyN(w, rand.Reader, 1<<20)
|
||||||
|
return errors.New("source gone")
|
||||||
|
})
|
||||||
|
require.Error(t, fetch(t, request))
|
||||||
|
taken, err := done()
|
||||||
|
assert.True(t, taken)
|
||||||
|
require.EqualError(t, err, "source gone")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the helper refuses a runner whose certificate does not match its pin", func(t *testing.T) {
|
||||||
|
request, done := server.offer(loopback, nothing)
|
||||||
|
request.Pin = strings.Repeat("0", 64)
|
||||||
|
require.ErrorContains(t, fetch(t, request), "does not match the pin")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.False(t, taken)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKubernetesPodTransferImage(t *testing.T) {
|
||||||
|
newPod := func(image string, fallback bool) *KubernetesPod {
|
||||||
|
pod := &KubernetesPod{options: KubernetesOptions{Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: image, Fallback: fallback}}}
|
||||||
|
pod.JobContainer(&NewContainerInput{})
|
||||||
|
return pod
|
||||||
|
}
|
||||||
|
|
||||||
|
pod := newPod("", false)
|
||||||
|
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||||
|
assert.Equal(t, "gitea/runner:latest", pod.transferImage)
|
||||||
|
|
||||||
|
pod = newPod("registry.example/runner:1", false)
|
||||||
|
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||||
|
assert.Equal(t, "registry.example/runner:1", pod.transferImage)
|
||||||
|
|
||||||
|
pod = newPod("", true)
|
||||||
|
require.NoError(t, pod.setupTransfer(""))
|
||||||
|
assert.Empty(t, pod.transferHelper)
|
||||||
|
|
||||||
|
pod = newPod("", false)
|
||||||
|
require.ErrorContains(t, pod.setupTransfer(""), "kubernetes.transfer.image")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKubernetesPodCopiesLargeFilesOverPodNetwork(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("uses POSIX shell")
|
||||||
|
}
|
||||||
|
ctx := t.Context()
|
||||||
|
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","podIPs":[{"ip":"127.0.0.1"}],"containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}}]}}`, transferHelperEnv+"=1")
|
||||||
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, ForcePull: true, Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: "gitea/runner:latest", Fallback: true}})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job := pod.JobContainer(&NewContainerInput{Name: "job", Image: "node", WorkingDir: workdir, Stdout: io.Discard, Stderr: io.Discard})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||||
|
|
||||||
|
var manifest struct {
|
||||||
|
Spec struct {
|
||||||
|
InitContainers []struct {
|
||||||
|
Image, ImagePullPolicy string
|
||||||
|
Command []string
|
||||||
|
VolumeMounts []struct{ Name, MountPath string }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||||
|
require.Len(t, manifest.Spec.InitContainers, 1)
|
||||||
|
initContainer := manifest.Spec.InitContainers[0]
|
||||||
|
assert.Equal(t, "gitea/runner:latest", initContainer.Image)
|
||||||
|
assert.Equal(t, "Always", initContainer.ImagePullPolicy)
|
||||||
|
assert.Equal(t, []string{"gitea-runner", KubernetesTransferCommand, "install", "/var/run/act/bin/gitea-runner"}, initContainer.Command)
|
||||||
|
assert.Equal(t, []struct{ Name, MountPath string }{{"act", "/var/run/act"}}, initContainer.VolumeMounts)
|
||||||
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
|
|
||||||
|
helper := filepath.Join(t.TempDir(), "bin", "gitea-runner")
|
||||||
|
require.NoError(t, InstallKubernetesTransferHelper(helper))
|
||||||
|
pod.transferHelper = helper
|
||||||
|
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||||
|
large := &FileEntry{Name: "large", Mode: 0o644, Body: strings.Repeat("a", compressCopyMin)}
|
||||||
|
copyLarge := func(dir string) error { return job.Copy(filepath.Join(workdir, dir)+"/", large)(ctx) }
|
||||||
|
require.NoError(t, copyLarge("transferred"))
|
||||||
|
content, err := os.ReadFile(filepath.Join(workdir, "transferred", "large"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, large.Body, string(content))
|
||||||
|
assert.Contains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||||
|
assert.NotContains(t, lastExec(), "127.0.0.1")
|
||||||
|
|
||||||
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "event.json", Mode: 0o644, Body: `{}`})(ctx))
|
||||||
|
assert.NotContains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||||
|
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workdir, "file"), nil, 0o644))
|
||||||
|
require.ErrorContains(t, copyLarge("file"), "over the pod network")
|
||||||
|
assert.Equal(t, helper, pod.transferHelper)
|
||||||
|
|
||||||
|
unreachable := filepath.Join(workdir, "unreachable")
|
||||||
|
require.NoError(t, os.WriteFile(unreachable, []byte("#!/bin/sh\necho 'connection refused' >&2\nexit 1\n"), 0o755))
|
||||||
|
pod.transferHelper = unreachable
|
||||||
|
pod.options.Transfer.Fallback = false
|
||||||
|
require.ErrorContains(t, copyLarge("direct"), "connection refused")
|
||||||
|
assert.NoFileExists(t, filepath.Join(workdir, "direct", "large"))
|
||||||
|
|
||||||
|
pod.options.Transfer.Fallback = true
|
||||||
|
require.NoError(t, copyLarge("auto"))
|
||||||
|
assert.FileExists(t, filepath.Join(workdir, "auto", "large"))
|
||||||
|
assert.Empty(t, pod.transferHelper)
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ package container
|
|||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -18,7 +19,8 @@ import (
|
|||||||
"golang.org/x/text/transform"
|
"golang.org/x/text/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor {
|
// parseEnvFile reads a GITHUB_ENV style file, splitting lines like GitHub's runner on Windows or elsewhere.
|
||||||
|
func parseEnvFile(e Container, srcPath string, env *map[string]string, windows bool) common.Executor {
|
||||||
localEnv := *env
|
localEnv := *env
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
||||||
@@ -38,8 +40,22 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
s := bufio.NewScanner(decoded)
|
s := bufio.NewScanner(decoded)
|
||||||
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
|
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
|
||||||
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||||
|
scanned := 0
|
||||||
|
s.Split(func(data []byte, atEOF bool) (int, []byte, error) {
|
||||||
|
if index := bytes.IndexByte(data[scanned:], '\n'); index >= 0 {
|
||||||
|
advance := scanned + index + 1
|
||||||
|
scanned = 0
|
||||||
|
return advance, data[:advance], nil
|
||||||
|
}
|
||||||
|
scanned = len(data)
|
||||||
|
if atEOF && len(data) > 0 {
|
||||||
|
scanned = 0
|
||||||
|
return len(data), data, nil
|
||||||
|
}
|
||||||
|
return 0, nil, nil
|
||||||
|
})
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
line := s.Text()
|
line := trimEnvLineEnding(s.Text(), windows)
|
||||||
// GitHub's runner ignores blank lines
|
// GitHub's runner ignores blank lines
|
||||||
if strings.TrimSpace(line) == "" {
|
if strings.TrimSpace(line) == "" {
|
||||||
continue
|
continue
|
||||||
@@ -52,10 +68,13 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
case multiLineEnv != -1:
|
case multiLineEnv != -1:
|
||||||
var multiLineEnvContent []string
|
var multiLineEnvContent []string
|
||||||
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
||||||
|
if multiLineEnv == 0 || multiLineEnvDelimiter == "" {
|
||||||
|
return fmt.Errorf("invalid format '%v', name and delimiter must not be empty", line)
|
||||||
|
}
|
||||||
delimiterFound := false
|
delimiterFound := false
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
content := s.Text()
|
content := s.Text()
|
||||||
if content == multiLineEnvDelimiter {
|
if trimEnvLineEnding(content, windows) == multiLineEnvDelimiter {
|
||||||
delimiterFound = true
|
delimiterFound = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -67,7 +86,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
if !delimiterFound {
|
if !delimiterFound {
|
||||||
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
||||||
}
|
}
|
||||||
localEnv[line[:multiLineEnv]] = strings.Join(multiLineEnvContent, "\n")
|
localEnv[line[:multiLineEnv]] = trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
||||||
}
|
}
|
||||||
@@ -78,3 +97,10 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func trimEnvLineEnding(line string, windows bool) string {
|
||||||
|
if windows && strings.HasSuffix(line, "\r\n") {
|
||||||
|
return line[:len(line)-2]
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(line, "\n")
|
||||||
|
}
|
||||||
@@ -22,6 +22,7 @@ func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFile(t *testing.T) {
|
func TestParseEnvFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
largeValue := strings.Repeat("x", 2*1024*1024)
|
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||||
encodeUTF16 := func(endianness unicode.Endianness) string {
|
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||||
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||||
@@ -33,41 +34,65 @@ func TestParseEnvFile(t *testing.T) {
|
|||||||
want map[string]string
|
want map[string]string
|
||||||
}{
|
}{
|
||||||
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||||
|
{"single line CRLF and final bare CR", "FOO=bar\r\nBAZ=qux\r", map[string]string{"FOO": "bar\r", "BAZ": "qux\r"}},
|
||||||
|
{"single line without final newline", "FOO=bar", map[string]string{"FOO": "bar"}},
|
||||||
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||||
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
|
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
|
||||||
|
{"multiline without final newline", "FOO<<EOF\nline1\r\nline2\r\nEOF", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||||
|
{"empty multiline without final newline", "FOO<<EOF\nEOF", map[string]string{"FOO": ""}},
|
||||||
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
|
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
|
||||||
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
|
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
|
||||||
|
{"multiline CRLF", "FOO<<EOF\r\nline1\r\nline2\r\nEOF\r\n", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||||
|
{"multiline mixed line endings", "FOO<<EOF\nline1\r\nline2\n\r\nEOF\n", map[string]string{"FOO": "line1\r\nline2\n\r"}},
|
||||||
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
|
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
|
||||||
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
|
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
|
||||||
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar", "MULTI": "line1"}},
|
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||||
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar", "MULTI": "line1"}},
|
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
e, envPath := newTestHostEnv(t)
|
e, envPath := newTestHostEnv(t)
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
require.NoError(t, parseEnvFile(e, envPath, &env, false)(t.Context()))
|
||||||
assert.Equal(t, test.want, env)
|
assert.Equal(t, test.want, env)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
t.Run("Windows line endings", func(t *testing.T) {
|
||||||
|
hostEnv, envPath := newTestHostEnv(t)
|
||||||
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||||
|
env := map[string]string{}
|
||||||
|
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
||||||
|
assert.Equal(t, map[string]string{"FOO": "bar", "MULTI": "line1\r\nline2"}, env)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
e, envPath := newTestHostEnv(t)
|
e, envPath := newTestHostEnv(t)
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||||
assert.Contains(t, err.Error(), "reading env file")
|
assert.Contains(t, err.Error(), "reading env file")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileMissingDelimiter(t *testing.T) {
|
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||||
e, envPath := newTestHostEnv(t)
|
for name, content := range map[string]string{
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
|
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
||||||
|
"empty name": "<<EOF\nline1\nEOF\n",
|
||||||
|
"empty delimiter": "FOO<<\nline1\n\n",
|
||||||
|
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
e, envPath := newTestHostEnv(t)
|
||||||
|
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "delimiter")
|
assert.Contains(t, err.Error(), "delimiter")
|
||||||
|
assert.Empty(t, env)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -28,12 +28,12 @@ var (
|
|||||||
|
|
||||||
type pinnedKey struct{}
|
type pinnedKey struct{}
|
||||||
|
|
||||||
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
|
// WithPinnedCheckout keeps a run's sha, ref and repository fixed when HEAD moves, in pins unless ctx has some.
|
||||||
func WithPinnedCheckout(ctx context.Context) context.Context {
|
func WithPinnedCheckout(ctx context.Context, pins *sync.Map) context.Context {
|
||||||
if ctx.Value(pinnedKey{}) != nil {
|
if ctx.Value(pinnedKey{}) != nil {
|
||||||
return ctx
|
return ctx
|
||||||
}
|
}
|
||||||
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
|
return context.WithValue(ctx, pinnedKey{}, pins)
|
||||||
}
|
}
|
||||||
|
|
||||||
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
||||||
@@ -57,12 +57,29 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
case "pull_request_target":
|
case "pull_request_target":
|
||||||
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||||
case "pull_request", "pull_request_review", "pull_request_review_comment":
|
case "pull_request", "pull_request_review", "pull_request_review_comment":
|
||||||
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", ghc.Event["number"])
|
number := ghc.Event["number"]
|
||||||
|
if number == nil {
|
||||||
|
number = model.NestedMapLookup(ghc.Event, "pull_request", "number")
|
||||||
|
}
|
||||||
|
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", number)
|
||||||
|
if ghc.EventName == "pull_request" && ghc.Event["action"] == "closed" && model.NestedMapLookup(ghc.Event, "pull_request", "merged") == true {
|
||||||
|
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||||
|
}
|
||||||
case "deployment", "deployment_status":
|
case "deployment", "deployment_status":
|
||||||
ghc.Ref = model.AsString(model.NestedMapLookup(ghc.Event, "deployment", "ref"))
|
ghc.Ref = model.AsString(model.NestedMapLookup(ghc.Event, "deployment", "ref"))
|
||||||
case "release":
|
case "release":
|
||||||
ghc.Ref = "refs/tags/" + model.AsString(model.NestedMapLookup(ghc.Event, "release", "tag_name"))
|
ghc.Ref = "refs/tags/" + model.AsString(model.NestedMapLookup(ghc.Event, "release", "tag_name"))
|
||||||
case "push", "create", "workflow_dispatch":
|
case "create":
|
||||||
|
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||||
|
if ghc.Ref != "" && !strings.HasPrefix(ghc.Ref, "refs/") {
|
||||||
|
switch model.AsString(ghc.Event["ref_type"]) {
|
||||||
|
case "branch":
|
||||||
|
ghc.Ref = "refs/heads/" + ghc.Ref
|
||||||
|
case "tag":
|
||||||
|
ghc.Ref = "refs/tags/" + ghc.Ref
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "push", "workflow_dispatch":
|
||||||
ghc.Ref = model.AsString(ghc.Event["ref"])
|
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||||
default:
|
default:
|
||||||
defaultBranch := model.AsString(model.NestedMapLookup(ghc.Event, "repository", "default_branch"))
|
defaultBranch := model.AsString(model.NestedMapLookup(ghc.Event, "repository", "default_branch"))
|
||||||
@@ -131,7 +148,7 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
// remote in repoPath when it is not set yet, and derives its owner.
|
// remote in repoPath when it is not set yet, and derives its owner.
|
||||||
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||||
if ghc.Repository == "" {
|
if ghc.Repository == "" {
|
||||||
repo, err := findGithubRepo(ctx, repoPath)
|
repo, err := pinned(ctx, "repo:"+repoPath, func() (string, error) { return findGithubRepo(ctx, repoPath) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
|
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package ghcontext
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
@@ -41,6 +42,30 @@ func TestSetRef(t *testing.T) {
|
|||||||
ref: "refs/pull/1234/merge",
|
ref: "refs/pull/1234/merge",
|
||||||
refName: "1234/merge",
|
refName: "1234/merge",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request_review",
|
||||||
|
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request",
|
||||||
|
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": true}},
|
||||||
|
ref: "refs/heads/master",
|
||||||
|
refName: "master",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request",
|
||||||
|
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": false}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request_review_comment",
|
||||||
|
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
eventName: "deployment",
|
eventName: "deployment",
|
||||||
event: map[string]any{
|
event: map[string]any{
|
||||||
@@ -69,6 +94,24 @@ func TestSetRef(t *testing.T) {
|
|||||||
ref: "refs/heads/somebranch",
|
ref: "refs/heads/somebranch",
|
||||||
refName: "somebranch",
|
refName: "somebranch",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "feature/new", "ref_type": "branch"},
|
||||||
|
ref: "refs/heads/feature/new",
|
||||||
|
refName: "feature/new",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "v1", "ref_type": "tag"},
|
||||||
|
ref: "refs/tags/v1",
|
||||||
|
refName: "v1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "refs/tags/v1", "ref_type": "tag"},
|
||||||
|
ref: "refs/tags/v1",
|
||||||
|
refName: "v1",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
eventName: "unknown",
|
eventName: "unknown",
|
||||||
event: map[string]any{
|
event: map[string]any{
|
||||||
@@ -122,8 +165,10 @@ func TestSetRef(t *testing.T) {
|
|||||||
func TestSetSha(t *testing.T) {
|
func TestSetSha(t *testing.T) {
|
||||||
oldFindGitRef := findGitRef
|
oldFindGitRef := findGitRef
|
||||||
oldFindGitRevision := findGitRevision
|
oldFindGitRevision := findGitRevision
|
||||||
|
oldFindGithubRepo := findGithubRepo
|
||||||
defer func() { findGitRef = oldFindGitRef }()
|
defer func() { findGitRef = oldFindGitRef }()
|
||||||
defer func() { findGitRevision = oldFindGitRevision }()
|
defer func() { findGitRevision = oldFindGitRevision }()
|
||||||
|
defer func() { findGithubRepo = oldFindGithubRepo }()
|
||||||
|
|
||||||
findGitRevision = func(_ctx context.Context, _file string) (string, error) {
|
findGitRevision = func(_ctx context.Context, _file string) (string, error) {
|
||||||
return "1234fakesha", nil
|
return "1234fakesha", nil
|
||||||
@@ -189,16 +234,19 @@ func TestSetSha(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
|
t.Run("pinned checkout keeps the first sha, ref and repository", func(t *testing.T) {
|
||||||
ctx := WithPinnedCheckout(t.Context())
|
ctx := WithPinnedCheckout(t.Context(), &sync.Map{})
|
||||||
for _, head := range []string{"first", "moved"} {
|
for _, head := range []string{"first", "moved"} {
|
||||||
findGitRevision = func(context.Context, string) (string, error) { return head, nil }
|
findGitRevision = func(context.Context, string) (string, error) { return head, nil }
|
||||||
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
||||||
|
findGithubRepo = func(context.Context, string) (string, error) { return "owner/" + head, nil }
|
||||||
ghc := &model.GithubContext{Event: map[string]any{}}
|
ghc := &model.GithubContext{Event: map[string]any{}}
|
||||||
SetSha(ctx, ghc, "/some/dir")
|
SetSha(ctx, ghc, "/some/dir")
|
||||||
SetRef(ctx, ghc, "/some/dir")
|
SetRef(ctx, ghc, "/some/dir")
|
||||||
|
SetRepositoryAndOwner(ctx, ghc, "/some/dir")
|
||||||
assert.Equal(t, "first", ghc.Sha)
|
assert.Equal(t, "first", ghc.Sha)
|
||||||
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
||||||
|
assert.Equal(t, "owner/first", ghc.Repository)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
+106
-26
@@ -19,6 +19,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
@@ -74,18 +75,24 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
|||||||
reader, closer, err = readFile("action.yaml")
|
reader, closer, err = readFile("action.yaml")
|
||||||
addError("action.yaml", err)
|
addError("action.yaml", err)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
_, closer, err := readFile("Dockerfile")
|
dockerfile := "Dockerfile"
|
||||||
addError("Dockerfile", err)
|
_, closer, err := readFile(dockerfile)
|
||||||
|
addError(dockerfile, err)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
dockerfile = "dockerfile"
|
||||||
|
_, closer, err = readFile(dockerfile)
|
||||||
|
addError(dockerfile, err)
|
||||||
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
closer.Close()
|
closer.Close()
|
||||||
action := &model.Action{
|
action := &model.Action{
|
||||||
Name: "(Synthetic)",
|
Name: "(Synthetic)",
|
||||||
Runs: model.ActionRuns{
|
Runs: model.ActionRuns{
|
||||||
Using: "docker",
|
Using: "docker",
|
||||||
Image: "Dockerfile",
|
Image: dockerfile,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
logger.Debugf("Using synthetic action %v for Dockerfile", action)
|
logger.Debugf("Using synthetic action %v for %s", action, dockerfile)
|
||||||
return action, nil
|
return action, nil
|
||||||
}
|
}
|
||||||
if val, ok := step.With["args"]; ok {
|
if val, ok := step.With["args"]; ok {
|
||||||
@@ -95,6 +102,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
|||||||
action := &model.Action{
|
action := &model.Action{
|
||||||
Name: "(Synthetic)",
|
Name: "(Synthetic)",
|
||||||
Inputs: map[string]model.Input{
|
Inputs: map[string]model.Input{
|
||||||
|
"args": {},
|
||||||
"cwd": {
|
"cwd": {
|
||||||
Description: "(Actual working directory)",
|
Description: "(Actual working directory)",
|
||||||
Required: false,
|
Required: false,
|
||||||
@@ -146,12 +154,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
|
|||||||
}
|
}
|
||||||
defer unlock()
|
defer unlock()
|
||||||
|
|
||||||
if !rc.Config.NoActionPatch {
|
if err := rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx); err != nil { // an exported commit has no .git to apply gitignore with
|
||||||
// Other jobs share the directory, so patch under the copy's lock.
|
return err
|
||||||
defer patchActions(ctx, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))()
|
|
||||||
}
|
}
|
||||||
|
if rc.Config.NoActionPatch {
|
||||||
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx) // an exported commit has no .git to apply gitignore with
|
return nil
|
||||||
|
}
|
||||||
|
patched := patchedScripts(ctx, actionDir, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))
|
||||||
|
if len(patched) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return rc.JobContainer.Copy(containerActionDirCopy, patched...)(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
|
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
|
||||||
@@ -167,11 +180,13 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
|||||||
|
|
||||||
action := step.getActionModel()
|
action := step.getActionModel()
|
||||||
|
|
||||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *step.getEnv())
|
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *step.getEnv())
|
||||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
warnUnexpectedInputs(ctx, action, stepModel.With)
|
||||||
|
printRunActionHeader(ctx, stepModel, action, *step.getEnv(), rc)
|
||||||
|
|
||||||
actionName, containerActionDir := getContainerActionPaths(step)
|
actionName, containerActionDir := getContainerActionPaths(step)
|
||||||
|
|
||||||
@@ -268,7 +283,7 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
|||||||
}
|
}
|
||||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||||
if !ok {
|
if !ok {
|
||||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
image = dockerActionImageTag(getStepGithubContext(ctx, step).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||||
job.dockerActionImages[step.getStepModel()] = image
|
job.dockerActionImages[step.getStepModel()] = image
|
||||||
}
|
}
|
||||||
@@ -301,9 +316,12 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
correctArchExists, err := ContainerImageExistsLocally(ctx, image, rc.Config.ContainerArchitecture)
|
correctArchExists := anyArchExists
|
||||||
if err != nil {
|
if anyArchExists && rc.Config.ContainerArchitecture != "" && rc.Config.ContainerArchitecture != "any" {
|
||||||
return err
|
correctArchExists, err = ContainerImageExistsLocally(ctx, image, rc.Config.ContainerArchitecture)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if anyArchExists && !correctArchExists {
|
if anyArchExists && !correctArchExists {
|
||||||
@@ -350,14 +368,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
|||||||
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cmd, err := shellquote.Split(step.getStepModel().With["args"])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ee, err := evalDockerEnv(ctx, step, action)
|
ee, err := evalDockerEnv(ctx, step, action)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
var cmd []string
|
||||||
if action.Runs.Args != nil {
|
if action.Runs.Args != nil {
|
||||||
// a fresh slice, the manifest is evaluated again for every stage
|
// a fresh slice, the manifest is evaluated again for every stage
|
||||||
cmd = make([]string, len(action.Runs.Args))
|
cmd = make([]string, len(action.Runs.Args))
|
||||||
@@ -366,6 +381,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
|||||||
return fmt.Errorf("unable to interpolate runs.args: %w", err)
|
return fmt.Errorf("unable to interpolate runs.args: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
cmd, err = shellquote.Split(stepInput(step.getStepModel(), "args"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
entrypoint, err := dockerEntrypoint(step, stage)
|
entrypoint, err := dockerEntrypoint(step, stage)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -395,7 +415,7 @@ func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
|
|||||||
default:
|
default:
|
||||||
entrypoint = runs.Entrypoint
|
entrypoint = runs.Entrypoint
|
||||||
if entrypoint == "" {
|
if entrypoint == "" {
|
||||||
if fields := strings.Fields(step.getStepModel().With["entrypoint"]); len(fields) > 0 {
|
if fields := strings.Fields(stepInput(step.getStepModel(), "entrypoint")); len(fields) > 0 {
|
||||||
return fields, nil
|
return fields, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -413,9 +433,15 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
|||||||
|
|
||||||
var err error
|
var err error
|
||||||
inputs := make(map[string]string)
|
inputs := make(map[string]string)
|
||||||
eval := rc.NewExpressionEvaluator(ctx)
|
providedInputs := make(map[string]bool, len(step.getStepModel().With))
|
||||||
// Set Defaults
|
for key := range step.getStepModel().With {
|
||||||
|
providedInputs[strings.ToLower(key)] = true
|
||||||
|
}
|
||||||
|
eval := rc.NewStepExpressionEvaluator(ctx, step)
|
||||||
for k, input := range action.Inputs {
|
for k, input := range action.Inputs {
|
||||||
|
if providedInputs[strings.ToLower(k)] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if inputs[k], err = eval.Interpolate(ctx, input.Default); err != nil {
|
if inputs[k], err = eval.Interpolate(ctx, input.Default); err != nil {
|
||||||
return nil, fmt.Errorf("unable to interpolate the default of input %s: %w", k, err)
|
return nil, fmt.Errorf("unable to interpolate the default of input %s: %w", k, err)
|
||||||
}
|
}
|
||||||
@@ -430,6 +456,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
||||||
|
aliasGiteaEnv(*step.getEnv())
|
||||||
return ee, nil
|
return ee, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -479,10 +506,15 @@ func populateEnvsFromSavedState(env *map[string]string, step actionStep, rc *Run
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *model.Action, rc *RunContext) error {
|
func inputEnvKey(inputID string) string {
|
||||||
eval := rc.NewExpressionEvaluator(ctx)
|
return "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||||
|
}
|
||||||
|
|
||||||
|
func populateEnvsFromInput(ctx context.Context, step step, action *model.Action) error {
|
||||||
|
env := step.getEnv()
|
||||||
|
eval := step.getRunContext().NewStepExpressionEvaluator(ctx, step)
|
||||||
for inputID, input := range action.Inputs {
|
for inputID, input := range action.Inputs {
|
||||||
envKey := "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
envKey := inputEnvKey(inputID)
|
||||||
if _, ok := (*env)[envKey]; !ok {
|
if _, ok := (*env)[envKey]; !ok {
|
||||||
var err error
|
var err error
|
||||||
if (*env)[envKey], err = eval.Interpolate(ctx, input.Default); err != nil {
|
if (*env)[envKey], err = eval.Interpolate(ctx, input.Default); err != nil {
|
||||||
@@ -493,6 +525,47 @@ func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||||
|
func lookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||||
|
if value, ok := values[name]; ok || !caseInsensitive {
|
||||||
|
return value, ok
|
||||||
|
}
|
||||||
|
for key, value := range values {
|
||||||
|
if strings.EqualFold(key, name) {
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var zero V
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func stepInput(step *model.Step, name string) string {
|
||||||
|
value, _ := lookupFold(step.With, name, true)
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func warnUnexpectedInputs(ctx context.Context, action *model.Action, inputs map[string]string) {
|
||||||
|
docker := action.Runs.Using.IsDocker()
|
||||||
|
var unexpectedInputs []string
|
||||||
|
for input := range inputs {
|
||||||
|
_, declared := lookupFold(action.Inputs, input, true)
|
||||||
|
containerInput := docker && (strings.EqualFold(input, "entryPoint") || strings.EqualFold(input, "args"))
|
||||||
|
if !declared && !containerInput {
|
||||||
|
unexpectedInputs = append(unexpectedInputs, input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unexpectedInputs) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
validInputs := slices.Sorted(maps.Keys(action.Inputs))
|
||||||
|
if docker {
|
||||||
|
validInputs = append([]string{"entryPoint", "args"}, validInputs...)
|
||||||
|
}
|
||||||
|
slices.Sort(unexpectedInputs)
|
||||||
|
common.Logger(ctx).With(rawOutputField, true).Warnf("##[warning]%s", EscapeCommandData(fmt.Sprintf(
|
||||||
|
"Unexpected input(s) '%s', valid inputs are ['%s']", strings.Join(unexpectedInputs, "', '"), strings.Join(validInputs, "', '"))))
|
||||||
|
}
|
||||||
|
|
||||||
func getContainerActionPaths(step actionStep) (string, string) {
|
func getContainerActionPaths(step actionStep) (string, string) {
|
||||||
rc := step.getRunContext()
|
rc := step.getRunContext()
|
||||||
if sar, ok := step.(*stepActionRemote); ok { // a copy per action, as its repository's directory is shared
|
if sar, ok := step.(*stepActionRemote); ok { // a copy per action, as its repository's directory is shared
|
||||||
@@ -581,10 +654,11 @@ func runPreStep(step actionStep) common.Executor {
|
|||||||
x := action.Runs.Using
|
x := action.Runs.Using
|
||||||
if !x.IsComposite() {
|
if !x.IsComposite() {
|
||||||
// defaults in pre steps were missing, however provided inputs are available
|
// defaults in pre steps were missing, however provided inputs are available
|
||||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
printRunActionHeader(ctx, step.getStepModel(), action, *step.getEnv(), rc)
|
||||||
switch {
|
switch {
|
||||||
case x.IsNode():
|
case x.IsNode():
|
||||||
if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil {
|
if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil {
|
||||||
@@ -633,6 +707,9 @@ func runPreStep(step actionStep) common.Executor {
|
|||||||
|
|
||||||
func shouldRunPostStep(step actionStep) common.Conditional {
|
func shouldRunPostStep(step actionStep) common.Conditional {
|
||||||
return func(ctx context.Context) bool {
|
return func(ctx context.Context) bool {
|
||||||
|
if remote, ok := step.(*stepActionRemote); ok && remote.preStepStarted && !remote.action.Runs.Using.IsComposite() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
log := common.Logger(ctx)
|
log := common.Logger(ctx)
|
||||||
stepResults := step.getRunContext().StepResults
|
stepResults := step.getRunContext().StepResults
|
||||||
stepResult := stepResults[step.getStepModel().ID]
|
stepResult := stepResults[step.getStepModel().ID]
|
||||||
@@ -680,6 +757,9 @@ func runPostStep(step actionStep) common.Executor {
|
|||||||
actionDir, actionPath, _, containerActionDir := actionStagePaths(step)
|
actionDir, actionPath, _, containerActionDir := actionStagePaths(step)
|
||||||
|
|
||||||
x := action.Runs.Using
|
x := action.Runs.Using
|
||||||
|
if !x.IsComposite() { // a composite's children print their own
|
||||||
|
logger.With(rawOutputField, true).Info("Post job cleanup.")
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case x.IsNode():
|
case x.IsNode():
|
||||||
|
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
|
|||||||
ee := parent.NewStepExpressionEvaluator(ctx, step)
|
ee := parent.NewStepExpressionEvaluator(ctx, step)
|
||||||
|
|
||||||
for inputID, input := range step.getActionModel().Inputs {
|
for inputID, input := range step.getActionModel().Inputs {
|
||||||
envKey := "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
envKey := inputEnvKey(inputID)
|
||||||
|
|
||||||
// lookup if key is defined in the step but the already
|
// lookup if key is defined in the step but the already
|
||||||
// evaluated value from the environment
|
// evaluated value from the environment
|
||||||
@@ -52,9 +52,11 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
gh := step.getGithubContext(ctx)
|
for envKey, value := range (&model.Step{With: step.getStepModel().With}).GetEnv() {
|
||||||
env["GITHUB_ACTION_REPOSITORY"] = gh.ActionRepository
|
if _, ok := env[envKey]; !ok {
|
||||||
env["GITHUB_ACTION_REF"] = gh.ActionRef
|
env[envKey] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return env, nil
|
return env, nil
|
||||||
}
|
}
|
||||||
@@ -105,6 +107,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action
|
|||||||
kubernetes: parent.kubernetes,
|
kubernetes: parent.kubernetes,
|
||||||
jobIndex: parent.jobIndex,
|
jobIndex: parent.jobIndex,
|
||||||
jobTotal: parent.jobTotal,
|
jobTotal: parent.jobTotal,
|
||||||
|
runtimeToken: parent.runtimeToken,
|
||||||
}
|
}
|
||||||
compositerc.setCompositeActionEnv(env)
|
compositerc.setCompositeActionEnv(env)
|
||||||
compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx)
|
compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx)
|
||||||
|
|||||||
@@ -7,7 +7,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/git"
|
"gitea.com/gitea/runner/act/common/git"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -16,11 +19,41 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestCompositeActionParity(t *testing.T) {
|
func TestCompositeActionParity(t *testing.T) {
|
||||||
|
t.Run("input defaults use the step identity and docker env keeps the GITEA aliases", func(t *testing.T) {
|
||||||
|
for _, uses := range []string{"./action", "own/action@v2"} {
|
||||||
|
t.Run(uses, func(t *testing.T) {
|
||||||
|
rc := &RunContext{
|
||||||
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
|
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||||
|
}
|
||||||
|
var currentStep step = &stepActionLocal{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}}
|
||||||
|
want := "|"
|
||||||
|
if uses == "own/action@v2" {
|
||||||
|
currentStep = &stepActionRemote{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}, remoteAction: mustNewRemoteAction(t, uses)}
|
||||||
|
want = "own/action|v2"
|
||||||
|
}
|
||||||
|
action := &model.Action{
|
||||||
|
Inputs: map[string]model.Input{"identity": {Default: "${{ github.action_repository }}|${{ github.action_ref }}"}},
|
||||||
|
Runs: model.ActionRuns{Env: map[string]string{"GITHUB_WORKFLOW_SHA": "runs-sha"}},
|
||||||
|
}
|
||||||
|
require.NoError(t, setupEnv(t.Context(), currentStep))
|
||||||
|
require.NoError(t, populateEnvsFromInput(t.Context(), currentStep, action))
|
||||||
|
assert.Equal(t, want, (*currentStep.getEnv())["INPUT_IDENTITY"])
|
||||||
|
_, err := evalDockerEnv(t.Context(), currentStep, action)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, (*currentStep.getEnv())["identity"])
|
||||||
|
assert.Equal(t, "runs-sha", (*currentStep.getEnv())["GITEA_WORKFLOW_SHA"])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("inherits contexts without leaking inputs", func(t *testing.T) {
|
t.Run("inherits contexts without leaking inputs", func(t *testing.T) {
|
||||||
ctx := t.Context()
|
hook := logtest.NewWithLevel(logging.LevelWarn)
|
||||||
|
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||||
strategy := &model.Strategy{MaxParallelString: "3", RawMatrix: yaml.Node{Kind: yaml.MappingNode}}
|
strategy := &model.Strategy{MaxParallelString: "3", RawMatrix: yaml.Node{Kind: yaml.MappingNode}}
|
||||||
parent := &RunContext{
|
parent := &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Matrix: map[string]any{"os": "linux"},
|
Matrix: map[string]any{"os": "linux"},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Name: "workflow", Jobs: map[string]*model.Job{"job": {Strategy: strategy}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Name: "workflow", Jobs: map[string]*model.Job{"job": {Strategy: strategy}}}},
|
||||||
JobContainer: &jobContainerMock{},
|
JobContainer: &jobContainerMock{},
|
||||||
@@ -28,24 +61,29 @@ func TestCompositeActionParity(t *testing.T) {
|
|||||||
jobIndex: 1,
|
jobIndex: 1,
|
||||||
jobTotal: 4,
|
jobTotal: 4,
|
||||||
}
|
}
|
||||||
composite, err := newCompositeRunContext(ctx, parent, &stepActionRemote{
|
step := &stepActionLocal{
|
||||||
Step: &model.Step{With: map[string]string{"SHARED": "outer"}},
|
Step: &model.Step{Uses: "./action", With: map[string]string{"SHARED": "outer", "extra": "literal${{ github.job }}"}},
|
||||||
RunContext: parent,
|
RunContext: parent,
|
||||||
action: &model.Action{Inputs: map[string]model.Input{"shared": {Default: "outer-default"}}},
|
action: &model.Action{Inputs: map[string]model.Input{"shared": {Default: "outer-default"}}, Runs: model.ActionRuns{Using: "composite"}},
|
||||||
env: map[string]string{"INPUT_SHARED": "outer"},
|
env: map[string]string{"INPUT_SHARED": "outer"},
|
||||||
}, "/action")
|
}
|
||||||
|
require.NoError(t, runActionImpl(step, "/action", nil)(ctx))
|
||||||
|
require.Len(t, hook.AllRecords(), 1)
|
||||||
|
assert.Equal(t, "##[warning]Unexpected input(s) 'extra', valid inputs are ['shared']", hook.LastRecord().Message)
|
||||||
|
composite, err := step.getCompositeRunContext(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Same(t, strategy, composite.Run.Job().Strategy)
|
assert.Same(t, strategy, composite.Run.Job().Strategy)
|
||||||
assert.True(t, composite.IsHostEnv())
|
assert.True(t, composite.IsHostEnv())
|
||||||
interpolated, err := composite.NewExpressionEvaluator(ctx).Interpolate(ctx,
|
interpolated, err := composite.NewExpressionEvaluator(ctx).Interpolate(ctx,
|
||||||
"${{ matrix.os }}|${{ strategy.max-parallel }}|${{ strategy.job-index }}|${{ inputs.shared }}")
|
"${{ matrix.os }}|${{ strategy.max-parallel }}|${{ strategy.job-index }}|${{ inputs.shared }}|${{ inputs.extra }}")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "linux|3|1|outer", interpolated)
|
assert.Equal(t, "linux|3|1|outer|literal${{ github.job }}", interpolated)
|
||||||
assert.NotContains(t, composite.Env, "INPUT_SHARED")
|
assert.NotContains(t, composite.Env, "INPUT_SHARED")
|
||||||
|
assert.NotContains(t, composite.Env, "INPUT_EXTRA")
|
||||||
|
|
||||||
nestedEnv := composite.GetEnv()
|
nestedEnv := composite.GetEnv()
|
||||||
require.NoError(t, populateEnvsFromInput(ctx, &nestedEnv, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}, composite))
|
require.NoError(t, populateEnvsFromInput(ctx, &stepActionLocal{RunContext: composite, env: nestedEnv}, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}))
|
||||||
assert.Equal(t, "inner-default", nestedEnv["INPUT_SHARED"])
|
assert.Equal(t, "inner-default", nestedEnv["INPUT_SHARED"])
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -54,7 +92,7 @@ func TestCompositeActionParity(t *testing.T) {
|
|||||||
return "", "", assert.AnError
|
return "", "", assert.AnError
|
||||||
})
|
})
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{GitHubInstance: "github.com", ActionCacheDir: t.TempDir()},
|
Config: &Config{GitHubInstance: "github.com", ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
JobContainer: &jobContainerMock{},
|
JobContainer: &jobContainerMock{},
|
||||||
}
|
}
|
||||||
|
|||||||
+165
-14
@@ -8,6 +8,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -19,6 +20,8 @@ import (
|
|||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/git"
|
"gitea.com/gitea/runner/act/common/git"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
"gitea.com/gitea/runner/act/container"
|
"gitea.com/gitea/runner/act/container"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
@@ -85,6 +88,19 @@ runs:
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "readLowercaseDockerfile",
|
||||||
|
step: &model.Step{},
|
||||||
|
filename: "dockerfile",
|
||||||
|
fileContent: "FROM ubuntu:20.04",
|
||||||
|
expected: &model.Action{
|
||||||
|
Name: "(Synthetic)",
|
||||||
|
Runs: model.ActionRuns{
|
||||||
|
Using: "docker",
|
||||||
|
Image: "dockerfile",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "readWithArgs",
|
name: "readWithArgs",
|
||||||
step: &model.Step{
|
step: &model.Step{
|
||||||
@@ -95,6 +111,7 @@ runs:
|
|||||||
expected: &model.Action{
|
expected: &model.Action{
|
||||||
Name: "(Synthetic)",
|
Name: "(Synthetic)",
|
||||||
Inputs: map[string]model.Input{
|
Inputs: map[string]model.Input{
|
||||||
|
"args": {},
|
||||||
"cwd": {
|
"cwd": {
|
||||||
Description: "(Actual working directory)",
|
Description: "(Actual working directory)",
|
||||||
Required: false,
|
Required: false,
|
||||||
@@ -147,14 +164,33 @@ func TestActionRunner(t *testing.T) {
|
|||||||
for _, tt := range []struct {
|
for _, tt := range []struct {
|
||||||
name string
|
name string
|
||||||
inputs map[string]model.Input
|
inputs map[string]model.Input
|
||||||
|
with map[string]string
|
||||||
state map[string]string
|
state map[string]string
|
||||||
expectedEnv map[string]string
|
expectedEnv map[string]string
|
||||||
|
warning string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "with-input",
|
name: "with-input",
|
||||||
inputs: map[string]model.Input{"key": {Default: "default value"}},
|
inputs: map[string]model.Input{"key": {Default: "default value"}},
|
||||||
expectedEnv: map[string]string{"INPUT_KEY": "default value"},
|
expectedEnv: map[string]string{"INPUT_KEY": "default value"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "declared input ignores case",
|
||||||
|
inputs: map[string]model.Input{"key": {Default: "default value"}},
|
||||||
|
with: map[string]string{"KEY": "provided value"},
|
||||||
|
expectedEnv: map[string]string{"INPUT_KEY": "provided value"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unexpected inputs warn once across stages",
|
||||||
|
inputs: map[string]model.Input{"key": {}},
|
||||||
|
with: map[string]string{"KEY": "provided value", "UNKNOWN": "value", "other%0A\r\n": "value"},
|
||||||
|
warning: "##[warning]Unexpected input(s) 'UNKNOWN', 'other%250A%0D%0A', valid inputs are ['key']",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no declared inputs",
|
||||||
|
with: map[string]string{"unknown": "value"},
|
||||||
|
warning: "##[warning]Unexpected input(s) 'unknown', valid inputs are ['']",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "restore-saved-state",
|
name: "restore-saved-state",
|
||||||
state: map[string]string{"name": "state value"},
|
state: map[string]string{"name": "state value"},
|
||||||
@@ -162,6 +198,8 @@ func TestActionRunner(t *testing.T) {
|
|||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
hook := logtest.NewWithLevel(logging.LevelWarn)
|
||||||
|
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
containerActionDir := "/var/run/act/actions/" + model.UsesHash("/org/repo/path@ref") + "/"
|
containerActionDir := "/var/run/act/actions/" + model.UsesHash("/org/repo/path@ref") + "/"
|
||||||
cm.On("CopyDir", containerActionDir, "dir/", false, true).Return(func(_ctx context.Context) error { return nil })
|
cm.On("CopyDir", containerActionDir, "dir/", false, true).Return(func(_ctx context.Context) error { return nil })
|
||||||
@@ -175,21 +213,36 @@ func TestActionRunner(t *testing.T) {
|
|||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
cm.On("Exec", []string{"node", "--preserve-symlinks-main", containerActionDir + "path"}, envMatcher, "", "").Return(func(_ctx context.Context) error { return nil })
|
cm.On("Exec", []string{"node", "--preserve-symlinks-main", containerActionDir + "path"}, envMatcher, "", "").Return(func(_ctx context.Context) error { return nil })
|
||||||
|
cm.On("Exec", nodeActionCommand(containerActionDir+"path/pre.js"), mock.Anything, "", "").Return(func(_ctx context.Context) error { return nil })
|
||||||
|
cm.On("Exec", nodeActionCommand(containerActionDir+"path/post.js"), envMatcher, "", "").Return(func(_ctx context.Context) error { return nil })
|
||||||
|
|
||||||
remoteAction := mustNewRemoteAction(t, "org/repo/path@ref")
|
remoteAction := mustNewRemoteAction(t, "org/repo/path@ref")
|
||||||
step := &stepActionRemote{
|
step := &stepActionRemote{
|
||||||
Step: &model.Step{ID: "step", Uses: "org/repo/path@ref"},
|
Step: &model.Step{ID: "step", Uses: "org/repo/path@ref", With: tt.with},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
IntraActionState: map[string]map[string]string{"step": tt.state},
|
IntraActionState: map[string]map[string]string{"step": tt.state},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
},
|
},
|
||||||
action: &model.Action{Inputs: tt.inputs, Runs: model.ActionRuns{Using: "node16"}},
|
action: &model.Action{Inputs: tt.inputs, Runs: model.ActionRuns{Using: "node16", Pre: "pre.js", Post: "post.js"}},
|
||||||
env: map[string]string{},
|
actionDir: "dir",
|
||||||
|
env: (&model.Step{With: tt.with}).GetEnv(),
|
||||||
remoteAction: remoteAction,
|
remoteAction: remoteAction,
|
||||||
}
|
}
|
||||||
require.NoError(t, runActionImpl(step, "dir", remoteAction)(t.Context()))
|
require.NoError(t, runPreStep(step)(ctx))
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
require.NoError(t, runActionImpl(step, "dir", remoteAction)(ctx))
|
||||||
|
require.NoError(t, runPostStep(step)(ctx))
|
||||||
|
if tt.warning == "" {
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
} else {
|
||||||
|
require.Len(t, hook.AllRecords(), 1)
|
||||||
|
assert.Equal(t, tt.warning, hook.LastRecord().Message)
|
||||||
|
rawOutput, ok := hook.LastRecord().Attr(rawOutputField)
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, true, rawOutput)
|
||||||
|
}
|
||||||
cm.AssertExpectations(t)
|
cm.AssertExpectations(t)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -376,7 +429,7 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
|
|||||||
step := &stepActionRemote{
|
step := &stepActionRemote{
|
||||||
Step: &model.Step{ID: "1", Uses: "remote/action@v1", With: map[string]string{}},
|
Step: &model.Step{ID: "1", Uses: "remote/action@v1", With: map[string]string{}},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{Env: testCheckoutEnv()},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
Workflow: &model.Workflow{
|
Workflow: &model.Workflow{
|
||||||
@@ -414,7 +467,68 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
|
|||||||
|
|
||||||
cancel()
|
cancel()
|
||||||
releaseOnce()
|
releaseOnce()
|
||||||
<-done
|
assert.ErrorIs(t, <-done, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecAsDockerImageChecks(t *testing.T) {
|
||||||
|
originalExists, originalBuild, originalContainer := ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer
|
||||||
|
t.Cleanup(func() {
|
||||||
|
ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer = originalExists, originalBuild, originalContainer
|
||||||
|
})
|
||||||
|
ContainerNewContainer = func(*container.NewContainerInput) container.ExecutionsEnvironment { return fakeContainer{} }
|
||||||
|
|
||||||
|
for _, testcase := range []struct {
|
||||||
|
name, platform string
|
||||||
|
exists, matches, rebuild bool
|
||||||
|
wantBuild, wantRemove bool
|
||||||
|
wantChecks []string
|
||||||
|
}{
|
||||||
|
{name: "missing image", platform: "linux/arm64", wantBuild: true, wantChecks: []string{"any"}},
|
||||||
|
{name: "unspecified platform", exists: true, matches: true, wantChecks: []string{"any"}},
|
||||||
|
{name: "any platform", platform: "any", exists: true, matches: true, wantChecks: []string{"any"}},
|
||||||
|
{name: "matching platform", platform: "linux/arm64", exists: true, matches: true, wantChecks: []string{"any", "linux/arm64"}},
|
||||||
|
{name: "different platform", platform: "linux/arm64", exists: true, wantBuild: true, wantRemove: true, wantChecks: []string{"any", "linux/arm64"}},
|
||||||
|
{name: "forced rebuild", exists: true, matches: true, rebuild: true, wantBuild: true, wantChecks: []string{"any"}},
|
||||||
|
} {
|
||||||
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
|
var checks []string
|
||||||
|
ContainerImageExistsLocally = func(_ctx context.Context, _image, platform string) (bool, error) {
|
||||||
|
checks = append(checks, platform)
|
||||||
|
return testcase.exists && (platform == "any" || testcase.matches), nil
|
||||||
|
}
|
||||||
|
var built, removed bool
|
||||||
|
ContainerNewDockerBuildExecutor = func(container.NewDockerBuildExecutorInput) common.Executor {
|
||||||
|
return func(context.Context) error { built = true; return nil }
|
||||||
|
}
|
||||||
|
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Method == http.MethodDelete {
|
||||||
|
removed = true
|
||||||
|
_, _ = io.WriteString(writer, `[]`)
|
||||||
|
} else {
|
||||||
|
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
step := &stepActionRemote{
|
||||||
|
Step: &model.Step{ID: "action"},
|
||||||
|
RunContext: &RunContext{
|
||||||
|
Config: &Config{
|
||||||
|
PresetGitHubContext: &model.GithubContext{},
|
||||||
|
ContainerArchitecture: testcase.platform,
|
||||||
|
ForceRebuild: testcase.rebuild,
|
||||||
|
},
|
||||||
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
|
JobContainer: &jobContainerMock{},
|
||||||
|
},
|
||||||
|
action: &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "Dockerfile"}},
|
||||||
|
env: map[string]string{},
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, execAsDocker(t.Context(), step, "action", dir, dir, false, stepStageMain))
|
||||||
|
assert.Equal(t, testcase.wantChecks, checks)
|
||||||
|
assert.Equal(t, testcase.wantBuild, built)
|
||||||
|
assert.Equal(t, testcase.wantRemove, removed)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerActionImageTag(t *testing.T) {
|
func TestDockerActionImageTag(t *testing.T) {
|
||||||
@@ -468,15 +582,20 @@ func TestDockerActionImageTag(t *testing.T) {
|
|||||||
func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
||||||
orig := ContainerNewContainer
|
orig := ContainerNewContainer
|
||||||
defer func() { ContainerNewContainer = orig }()
|
defer func() { ContainerNewContainer = orig }()
|
||||||
|
fakeDockerDaemon(t, func(writer http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.WriteString(writer, `{"Architecture":"x86_64"}`)
|
||||||
|
})
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
stage stepStage
|
stage stepStage
|
||||||
with map[string]string
|
with map[string]string
|
||||||
|
inputs map[string]model.Input
|
||||||
runs model.ActionRuns
|
runs model.ActionRuns
|
||||||
env map[string]string
|
env map[string]string
|
||||||
wantCmd []string
|
wantCmd []string
|
||||||
wantEntrypoint []string
|
wantEntrypoint []string
|
||||||
|
warning string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "main stage prefers manifest values",
|
name: "main stage prefers manifest values",
|
||||||
@@ -491,17 +610,36 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
|||||||
wantEntrypoint: []string{"main.sh"},
|
wantEntrypoint: []string{"main.sh"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "main stage uses caller fallbacks",
|
name: "main stage uses caller fallbacks in any case and warns about undeclared inputs",
|
||||||
stage: stepStageMain,
|
stage: stepStageMain,
|
||||||
with: map[string]string{"args": "caller --flag", "entrypoint": "input.sh --verbose"},
|
with: map[string]string{"ARGS": "caller --flag", "entryPoint": "input.sh --verbose", "unknown": "value"},
|
||||||
|
inputs: map[string]model.Input{"message": {}},
|
||||||
runs: model.ActionRuns{Env: map[string]string{"ACTION_ONLY": "manifest"}},
|
runs: model.ActionRuns{Env: map[string]string{"ACTION_ONLY": "manifest"}},
|
||||||
wantCmd: []string{"caller", "--flag"},
|
wantCmd: []string{"caller", "--flag"},
|
||||||
wantEntrypoint: []string{"input.sh", "--verbose"},
|
wantEntrypoint: []string{"input.sh", "--verbose"},
|
||||||
|
warning: "##[warning]Unexpected input(s) 'unknown', valid inputs are ['entryPoint', 'args', 'message']",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "manifest args keep caller input quotes literal",
|
||||||
|
stage: stepStageMain,
|
||||||
|
with: map[string]string{"args": "it's literal"},
|
||||||
|
runs: model.ActionRuns{Args: []string{"${{ inputs.args }}"}},
|
||||||
|
env: map[string]string{"INPUT_ARGS": "it's literal"},
|
||||||
|
wantCmd: []string{"it's literal"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "caller input skips its default expression",
|
||||||
|
stage: stepStageMain,
|
||||||
|
with: map[string]string{"MESSAGE": "provided"},
|
||||||
|
inputs: map[string]model.Input{"message": {Default: "${{ fromJSON('') }}"}},
|
||||||
|
runs: model.ActionRuns{Args: []string{"${{ inputs.message }}"}},
|
||||||
|
env: map[string]string{"INPUT_MESSAGE": "provided"},
|
||||||
|
wantCmd: []string{"provided"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "explicit empty manifest args suppress caller args",
|
name: "explicit empty manifest args suppress caller args",
|
||||||
stage: stepStageMain,
|
stage: stepStageMain,
|
||||||
with: map[string]string{"args": "caller"},
|
with: map[string]string{"args": "it's ignored"},
|
||||||
runs: model.ActionRuns{Args: []string{}},
|
runs: model.ActionRuns{Args: []string{}},
|
||||||
wantCmd: []string{},
|
wantCmd: []string{},
|
||||||
},
|
},
|
||||||
@@ -527,6 +665,8 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
|||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
hook := logtest.NewWithLevel(logging.LevelWarn)
|
||||||
|
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||||
tc.runs.Using, tc.runs.Image = "docker", "docker://node:14"
|
tc.runs.Using, tc.runs.Image = "docker", "docker://node:14"
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
var input *container.NewContainerInput
|
var input *container.NewContainerInput
|
||||||
@@ -538,12 +678,13 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
|||||||
step := &stepActionRemote{
|
step := &stepActionRemote{
|
||||||
Step: &model.Step{ID: "1", Uses: "org/action@v1", With: tc.with},
|
Step: &model.Step{ID: "1", Uses: "org/action@v1", With: tc.with},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
},
|
},
|
||||||
action: &model.Action{Runs: tc.runs},
|
action: &model.Action{Inputs: tc.inputs, Runs: tc.runs},
|
||||||
env: mergeMaps(tc.env),
|
env: mergeMaps(tc.env),
|
||||||
|
remoteAction: mustNewRemoteAction(t, "org/action@v1"),
|
||||||
}
|
}
|
||||||
|
|
||||||
cm.On("Pull", false).Return(func(context.Context) error { return nil })
|
cm.On("Pull", false).Return(func(context.Context) error { return nil })
|
||||||
@@ -552,7 +693,17 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
|||||||
cm.On("Start", true).Return(func(context.Context) error { return nil })
|
cm.On("Start", true).Return(func(context.Context) error { return nil })
|
||||||
cm.On("Close").Return(func(context.Context) error { return nil })
|
cm.On("Close").Return(func(context.Context) error { return nil })
|
||||||
|
|
||||||
require.NoError(t, execAsDocker(t.Context(), step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
|
if tc.stage == stepStageMain {
|
||||||
|
require.NoError(t, runActionImpl(step, t.TempDir(), step.remoteAction)(ctx))
|
||||||
|
} else {
|
||||||
|
require.NoError(t, execAsDocker(ctx, step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
|
||||||
|
}
|
||||||
|
if tc.warning == "" {
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
} else {
|
||||||
|
require.Len(t, hook.AllRecords(), 1)
|
||||||
|
assert.Equal(t, tc.warning, hook.LastRecord().Message)
|
||||||
|
}
|
||||||
require.NotNil(t, input)
|
require.NotNil(t, input)
|
||||||
assert.Equal(t, tc.wantCmd, input.Cmd)
|
assert.Equal(t, tc.wantCmd, input.Cmd)
|
||||||
assert.Equal(t, tc.wantEntrypoint, input.Entrypoint)
|
assert.Equal(t, tc.wantEntrypoint, input.Entrypoint)
|
||||||
|
|||||||
@@ -101,9 +101,13 @@ func TestMainStepsExecutorRunsAlwaysStepsAfterInterruption(t *testing.T) {
|
|||||||
assert.Equal(t, []string{"step1", "always-step"}, ran)
|
assert.Equal(t, []string{"step1", "always-step"}, ran)
|
||||||
require.NoError(t, laterStepCtxErr)
|
require.NoError(t, laterStepCtxErr)
|
||||||
assert.Equal(t, wantError == context.Canceled, rc.jobCancelled)
|
assert.Equal(t, wantError == context.Canceled, rc.jobCancelled)
|
||||||
|
setJobResult(t.Context(), rc, rc, true)
|
||||||
if wantError == context.DeadlineExceeded {
|
if wantError == context.DeadlineExceeded {
|
||||||
assert.True(t, rc.jobFailed)
|
assert.True(t, rc.jobFailed)
|
||||||
assert.Equal(t, "failure", rc.getJobContext().Status)
|
assert.Equal(t, "failure", rc.getJobContext().Status)
|
||||||
|
assert.Equal(t, "failure", rc.Run.Job().Result)
|
||||||
|
} else {
|
||||||
|
assert.Equal(t, "cancelled", rc.Run.Job().Result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+24
-3
@@ -8,6 +8,8 @@ import (
|
|||||||
"cmp"
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
@@ -158,11 +160,18 @@ const allowUnsecureCommandsVar = "ACTIONS_ALLOW_UNSECURE_COMMANDS"
|
|||||||
// not run, recording the error that fails the step. GitHub disabled both because a step that
|
// not run, recording the error that fails the step. GitHub disabled both because a step that
|
||||||
// echoes untrusted content can use them to set NODE_OPTIONS or PATH for every later step.
|
// echoes untrusted content can use them to set NODE_OPTIONS or PATH for every later step.
|
||||||
func (rc *RunContext) refuseUnsecureCommand(ctx context.Context, command string) bool {
|
func (rc *RunContext) refuseUnsecureCommand(ctx context.Context, command string) bool {
|
||||||
if isTruthyEnv(rc.currentStepEnv()[allowUnsecureCommandsVar]) ||
|
if isTruthyEnv(os.Getenv(allowUnsecureCommandsVar)) { // the runner's own opt-in wins, as on GitHub
|
||||||
isTruthyEnv(rc.Env[allowUnsecureCommandsVar]) ||
|
|
||||||
isTruthyEnv(rc.GlobalEnv[allowUnsecureCommandsVar]) {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
caseInsensitive := rc.JobContainer != nil && rc.JobContainer.IsEnvironmentCaseInsensitive()
|
||||||
|
for _, env := range []map[string]string{rc.currentStepEnv(), rc.Env, rc.GlobalEnv} {
|
||||||
|
if value, ok := lookupFold(env, allowUnsecureCommandsVar, caseInsensitive); ok {
|
||||||
|
if isTruthyEnv(value) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The step executor logs the failure itself, so keep this line's wording distinct.
|
// The step executor logs the failure itself, so keep this line's wording distinct.
|
||||||
common.Logger(ctx).With(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(fmt.Sprintf(
|
common.Logger(ctx).With(rawOutputField, true).Errorf("##[error]%s", EscapeCommandData(fmt.Sprintf(
|
||||||
@@ -192,6 +201,18 @@ func (rc *RunContext) takeUnsecureCommandError() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runWithCommands runs executor judging its workflow commands by env, failing it on a refused one.
|
||||||
|
func (rc *RunContext) runWithCommands(ctx context.Context, env map[string]string, executor common.Executor) error {
|
||||||
|
rc.setCurrentStepEnv(maps.Clone(env)) // the executor keeps writing to env while the command handler reads it
|
||||||
|
defer rc.setCurrentStepEnv(nil)
|
||||||
|
_ = rc.takeUnsecureCommandError() // a refusal from before belongs to no step
|
||||||
|
err := executor(ctx)
|
||||||
|
if refusal := rc.takeUnsecureCommandError(); err == nil {
|
||||||
|
return refusal
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg string, fromCommand bool) {
|
func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg string, fromCommand bool) {
|
||||||
name := kvPairs["name"]
|
name := kvPairs["name"]
|
||||||
if strings.EqualFold(name, "NODE_OPTIONS") {
|
if strings.EqualFold(name, "NODE_OPTIONS") {
|
||||||
|
|||||||
@@ -136,21 +136,28 @@ func TestEscapeCommandData(t *testing.T) {
|
|||||||
|
|
||||||
func TestUnsecureCommands(t *testing.T) {
|
func TestUnsecureCommands(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
jobEnv map[string]string
|
jobEnv map[string]string
|
||||||
stepEnv map[string]string
|
globalEnv map[string]string
|
||||||
optedIn bool
|
stepEnv map[string]string
|
||||||
|
processEnv string
|
||||||
|
caseInsensitive bool
|
||||||
|
optedIn bool
|
||||||
}{
|
}{
|
||||||
{name: "refused with no opt-in"},
|
{name: "refused with no opt-in"},
|
||||||
// GitHub reads the opt-in with bool.TryParse, so "1" is not one.
|
|
||||||
{name: "refused for a value bool.TryParse rejects", jobEnv: map[string]string{allowUnsecureCommandsVar: "1"}},
|
{name: "refused for a value bool.TryParse rejects", jobEnv: map[string]string{allowUnsecureCommandsVar: "1"}},
|
||||||
{name: "opted in through the step environment", stepEnv: map[string]string{allowUnsecureCommandsVar: "true"}, optedIn: true},
|
{name: "opted in through the step environment", stepEnv: map[string]string{allowUnsecureCommandsVar: "true"}, optedIn: true},
|
||||||
{name: "opted in through the job environment", jobEnv: map[string]string{allowUnsecureCommandsVar: "TRUE"}, optedIn: true},
|
{name: "opted in through the job environment", jobEnv: map[string]string{allowUnsecureCommandsVar: "TRUE"}, optedIn: true},
|
||||||
|
{name: "step opt-out overrides the job and global environment", jobEnv: map[string]string{allowUnsecureCommandsVar: "true"}, globalEnv: map[string]string{allowUnsecureCommandsVar: "true"}, stepEnv: map[string]string{allowUnsecureCommandsVar: "false"}},
|
||||||
|
{name: "the runner's own opt-in overrides a step opt-out", stepEnv: map[string]string{allowUnsecureCommandsVar: "false"}, processEnv: "true", optedIn: true},
|
||||||
|
{name: "a differently cased opt-in counts where env names are case-insensitive", jobEnv: map[string]string{"Actions_Allow_Unsecure_Commands": "true"}, caseInsensitive: true, optedIn: true},
|
||||||
|
{name: "a differently cased opt-in is ignored where env names are case-sensitive", jobEnv: map[string]string{"Actions_Allow_Unsecure_Commands": "true"}},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Setenv(allowUnsecureCommandsVar, tt.processEnv)
|
||||||
a := assert.New(t)
|
a := assert.New(t)
|
||||||
rc := &RunContext{Config: &Config{}, Env: tt.jobEnv}
|
rc := &RunContext{Config: &Config{}, Env: tt.jobEnv, GlobalEnv: tt.globalEnv, JobContainer: fakeContainer{caseInsensitive: tt.caseInsensitive}}
|
||||||
rc.setCurrentStepEnv(tt.stepEnv)
|
rc.setCurrentStepEnv(tt.stepEnv)
|
||||||
handler := rc.commandHandler(t.Context())
|
handler := rc.commandHandler(t.Context())
|
||||||
|
|
||||||
@@ -160,7 +167,6 @@ func TestUnsecureCommands(t *testing.T) {
|
|||||||
if !tt.optedIn {
|
if !tt.optedIn {
|
||||||
a.Empty(rc.Env["x"])
|
a.Empty(rc.Env["x"])
|
||||||
a.Empty(rc.ExtraPath)
|
a.Empty(rc.ExtraPath)
|
||||||
// The refusal fails the step that produced it, once.
|
|
||||||
require.ErrorContains(t, rc.takeUnsecureCommandError(), "set-env")
|
require.ErrorContains(t, rc.takeUnsecureCommandError(), "set-env")
|
||||||
a.NoError(rc.takeUnsecureCommandError())
|
a.NoError(rc.takeUnsecureCommandError())
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -16,10 +16,18 @@ import (
|
|||||||
|
|
||||||
var noopExecutor = func(context.Context) error { return nil }
|
var noopExecutor = func(context.Context) error { return nil }
|
||||||
|
|
||||||
|
type linuxEnvironment struct {
|
||||||
|
container.LinuxContainerEnvironmentExtensions
|
||||||
|
}
|
||||||
|
|
||||||
|
func (linuxEnvironment) GetRunnerContext(context.Context) map[string]any {
|
||||||
|
return map[string]any{"os": "Linux", "arch": "X64", "temp": "/tmp", "tool_cache": container.DefaultToolCache}
|
||||||
|
}
|
||||||
|
|
||||||
type containerMock struct {
|
type containerMock struct {
|
||||||
mock.Mock
|
mock.Mock
|
||||||
container.Container
|
container.Container
|
||||||
container.LinuxContainerEnvironmentExtensions
|
linuxEnvironment
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cm *containerMock) Create(capAdd, capDrop []string) common.Executor {
|
func (cm *containerMock) Create(capAdd, capDrop []string) common.Executor {
|
||||||
|
|||||||
+39
-18
@@ -7,6 +7,7 @@ package runner
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
"path"
|
"path"
|
||||||
@@ -89,18 +90,18 @@ var hashfiles string
|
|||||||
|
|
||||||
// NewStepExpressionEvaluator creates a new evaluator with the `inputs` of the enclosing workflow or composite action
|
// NewStepExpressionEvaluator creates a new evaluator with the `inputs` of the enclosing workflow or composite action
|
||||||
func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||||
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext())
|
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewActionInputsExpressionEvaluator creates a new evaluator with the step's own with: values as `inputs`
|
// NewActionInputsExpressionEvaluator creates a new evaluator with the step's own with: values as `inputs`
|
||||||
func (rc *RunContext) NewActionInputsExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
func (rc *RunContext) NewActionInputsExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||||
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext())
|
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext) *ExpressionEvaluator {
|
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext, github *model.GithubContext) *ExpressionEvaluator {
|
||||||
// todo: cleanup EvaluationEnvironment creation
|
// todo: cleanup EvaluationEnvironment creation
|
||||||
ee := &exprparser.EvaluationEnvironment{
|
ee := &exprparser.EvaluationEnvironment{
|
||||||
Github: step.getGithubContext(ctx),
|
Github: github,
|
||||||
Env: *step.getEnv(),
|
Env: *step.getEnv(),
|
||||||
Job: jobContext,
|
Job: jobContext,
|
||||||
Steps: rc.StepResults,
|
Steps: rc.StepResults,
|
||||||
@@ -111,7 +112,7 @@ func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step,
|
|||||||
Needs: exprparser.NeedsContext(rc.Run),
|
Needs: exprparser.NeedsContext(rc.Run),
|
||||||
// todo: should be unavailable
|
// todo: should be unavailable
|
||||||
// but required to interpolate/evaluate the inputs in actions/composite
|
// but required to interpolate/evaluate the inputs in actions/composite
|
||||||
Inputs: getEvaluatorInputs(rc, stepInputs, rc.getGithubContext(ctx)),
|
Inputs: getEvaluatorInputs(rc, stepInputs, github),
|
||||||
HashFiles: getHashFilesFunction(ctx, rc),
|
HashFiles: getHashFilesFunction(ctx, rc),
|
||||||
}
|
}
|
||||||
ee.Runner = rc.getRunnerContext(ctx)
|
ee.Runner = rc.getRunnerContext(ctx)
|
||||||
@@ -127,7 +128,7 @@ func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step,
|
|||||||
func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.Value) (any, error) {
|
func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.Value) (any, error) {
|
||||||
return func(v []reflect.Value) (any, error) {
|
return func(v []reflect.Value) (any, error) {
|
||||||
if rc.JobContainer != nil {
|
if rc.JobContainer != nil {
|
||||||
timeed, cancel := context.WithTimeout(ctx, time.Minute)
|
timeoutCtx, cancel := context.WithTimeout(ctx, 120*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
name := "workflow/hashfiles/index.js"
|
name := "workflow/hashfiles/index.js"
|
||||||
hout := &bytes.Buffer{}
|
hout := &bytes.Buffer{}
|
||||||
@@ -154,20 +155,30 @@ func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.
|
|||||||
}
|
}
|
||||||
|
|
||||||
stdout, stderr := rc.JobContainer.ReplaceLogWriter(hout, herr)
|
stdout, stderr := rc.JobContainer.ReplaceLogWriter(hout, herr)
|
||||||
_ = rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{
|
defer rc.JobContainer.ReplaceLogWriter(stdout, stderr)
|
||||||
|
err := rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{
|
||||||
Name: name,
|
Name: name,
|
||||||
Mode: 0o644,
|
Mode: 0o644,
|
||||||
Body: hashfiles,
|
Body: hashfiles,
|
||||||
}).
|
}).Then(rc.JobContainer.Exec([]string{"node", path.Join(rc.JobContainer.GetActPath(), name)}, env, "", ""))(timeoutCtx)
|
||||||
Then(rc.JobContainer.Exec([]string{"node", path.Join(rc.JobContainer.GetActPath(), name)},
|
quotedPatterns := expressionStringEscape(strings.Join(patterns, ", "))
|
||||||
env, "", "")).
|
switch {
|
||||||
Finally(func(context.Context) error {
|
case err == nil:
|
||||||
rc.JobContainer.ReplaceLogWriter(stdout, stderr)
|
case ctx.Err() != nil:
|
||||||
return nil
|
return "", ctx.Err()
|
||||||
})(timeed)
|
case timeoutCtx.Err() != nil || errors.Is(err, context.DeadlineExceeded):
|
||||||
if _, output, found := strings.Cut(hout.String()+"\n"+herr.String(), "__OUTPUT__"); found {
|
return "", fmt.Errorf("hashFiles('%s') couldn't finish within 120 seconds", quotedPatterns)
|
||||||
if hash, _, found := strings.Cut(output, "__OUTPUT__"); found {
|
case hashFilesNodeMissing(err):
|
||||||
return hash, nil
|
common.Logger(ctx).With(rawOutputField, true).Warnf("##[warning]hashFiles could not run because node is unavailable in the job environment. Returning an empty string.")
|
||||||
|
return "", nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("hashFiles('%s') failed. Fail to hash files under directory '%s'", quotedPatterns, rc.githubWorkspace())
|
||||||
|
}
|
||||||
|
for line := range strings.Lines(hout.String() + "\n" + herr.String()) {
|
||||||
|
if hash, ok := strings.CutPrefix(strings.TrimRight(line, "\r\n"), "__OUTPUT__"); ok {
|
||||||
|
if hash, ok = strings.CutSuffix(hash, "__OUTPUT__"); ok {
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -175,6 +186,11 @@ func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func hashFilesNodeMissing(err error) bool {
|
||||||
|
exitErr, ok := errors.AsType[container.ExitCodeError](err)
|
||||||
|
return ok && (exitErr == 126 || exitErr == 127) || err.Error() == "Cannot find: node in PATH"
|
||||||
|
}
|
||||||
|
|
||||||
type expressionEvaluator struct {
|
type expressionEvaluator struct {
|
||||||
interpreter exprparser.Interpreter
|
interpreter exprparser.Interpreter
|
||||||
}
|
}
|
||||||
@@ -316,5 +332,10 @@ func getWorkflowSecrets(rc *RunContext) map[string]string {
|
|||||||
if rc.caller != nil {
|
if rc.caller != nil {
|
||||||
return rc.workflowCallSecrets
|
return rc.workflowCallSecrets
|
||||||
}
|
}
|
||||||
return rc.Config.Secrets
|
if rc.runtimeToken == "" {
|
||||||
|
return rc.Config.Secrets
|
||||||
|
}
|
||||||
|
secrets := map[string]string{"ACTIONS_RUNTIME_TOKEN": rc.runtimeToken} // as the daemon exposes its token
|
||||||
|
maps.Copy(secrets, rc.Config.Secrets)
|
||||||
|
return secrets
|
||||||
}
|
}
|
||||||
@@ -5,9 +5,20 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
"gitea.com/gitea/runner/act/container"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/exprparser"
|
"gitea.dev/actionslib/pkg/exprparser"
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
@@ -22,10 +33,11 @@ func createRunContext(t *testing.T) *RunContext {
|
|||||||
"os": {"Linux", "Windows"},
|
"os": {"Linux", "Windows"},
|
||||||
"foo": {"bar", "baz"},
|
"foo": {"bar", "baz"},
|
||||||
})
|
})
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return &RunContext{
|
return &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
Workdir: ".",
|
Workdir: ".",
|
||||||
Secrets: map[string]string{
|
Secrets: map[string]string{
|
||||||
"CASE_INSENSITIVE_SECRET": "value",
|
"CASE_INSENSITIVE_SECRET": "value",
|
||||||
@@ -144,6 +156,92 @@ func TestEvaluateRunContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type hashFilesContainer struct {
|
||||||
|
fakeContainer
|
||||||
|
stdout, stderr io.Writer
|
||||||
|
copyErr error
|
||||||
|
execute func([]string, map[string]string, string, string) common.Executor
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *hashFilesContainer) Copy(_destPath string, _files ...*container.FileEntry) common.Executor {
|
||||||
|
return common.NewErrorExecutor(c.copyErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *hashFilesContainer) Exec(command []string, env map[string]string, user, workdir string) common.Executor {
|
||||||
|
return c.execute(command, env, user, workdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *hashFilesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
|
||||||
|
previousOut, previousErr := c.stdout, c.stderr
|
||||||
|
c.stdout, c.stderr = stdout, stderr
|
||||||
|
return previousOut, previousErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetHashFilesFunction(t *testing.T) {
|
||||||
|
const failureMessage = "hashFiles('dir''file, *.sum') failed. Fail to hash files under directory '/workspace'"
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
output string
|
||||||
|
execErr error
|
||||||
|
copyErr error
|
||||||
|
hash string
|
||||||
|
err string
|
||||||
|
warning bool
|
||||||
|
expired bool
|
||||||
|
}{
|
||||||
|
{name: "success", output: "patterns __OUTPUT__wrong__OUTPUT__ echoed\n__OUTPUT__digest__OUTPUT__", hash: "digest"},
|
||||||
|
{name: "no matches", output: "__OUTPUT____OUTPUT__"},
|
||||||
|
{name: "script failure", output: "Cannot find: node in PATH\n__OUTPUT__digest__OUTPUT__", execErr: container.ExitCodeError(1), err: failureMessage},
|
||||||
|
{name: "timeout", execErr: context.DeadlineExceeded, err: "hashFiles('dir''file, *.sum') couldn't finish within 120 seconds"},
|
||||||
|
{name: "job deadline", execErr: context.DeadlineExceeded, expired: true, err: context.DeadlineExceeded.Error()},
|
||||||
|
{name: "node missing", execErr: fmt.Errorf("wrapped: %w", container.ExitCodeError(127)), warning: true},
|
||||||
|
{name: "node not executable", execErr: container.ExitCodeError(126), warning: true},
|
||||||
|
{name: "host node missing", execErr: errors.New("Cannot find: node in PATH"), warning: true},
|
||||||
|
{name: "copy failure", copyErr: errors.New("open /runner/cache: permission denied"), err: failureMessage},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
logger, hook := logtest.NewLogger()
|
||||||
|
stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
|
||||||
|
jobContainer := &hashFilesContainer{stdout: stdout, stderr: stderr, copyErr: test.copyErr}
|
||||||
|
jobContainer.execute = func(command []string, env map[string]string, user, workdir string) common.Executor {
|
||||||
|
assert.Equal(t, []string{"node", "/var/run/act/workflow/hashfiles/index.js"}, command)
|
||||||
|
assert.Equal(t, map[string]string{"patterns": "dir'file\n*.sum", "followSymbolicLinks": "true"}, env)
|
||||||
|
assert.Empty(t, user)
|
||||||
|
assert.Empty(t, workdir)
|
||||||
|
return func(_ctx context.Context) error {
|
||||||
|
_, err := io.WriteString(jobContainer.stderr, test.output)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return test.execErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
|
if test.expired {
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
ctx, cancel = context.WithDeadline(ctx, time.Now())
|
||||||
|
defer cancel()
|
||||||
|
}
|
||||||
|
result, err := getHashFilesFunction(ctx, &RunContext{
|
||||||
|
Config: &Config{Workdir: "/workspace"},
|
||||||
|
JobContainer: jobContainer,
|
||||||
|
})([]reflect.Value{reflect.ValueOf("--follow-symbolic-links"), reflect.ValueOf("dir'file"), reflect.ValueOf("*.sum")})
|
||||||
|
if test.err == "" {
|
||||||
|
require.NoError(t, err)
|
||||||
|
} else {
|
||||||
|
require.EqualError(t, err, test.err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, test.hash, result)
|
||||||
|
assert.Same(t, stdout, jobContainer.stdout)
|
||||||
|
assert.Same(t, stderr, jobContainer.stderr)
|
||||||
|
if test.warning {
|
||||||
|
require.Len(t, hook.AllRecords(), 1)
|
||||||
|
assert.Equal(t, "##[warning]hashFiles could not run because node is unavailable in the job environment. Returning an empty string.", hook.LastRecord().Message)
|
||||||
|
} else {
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEvaluateStep(t *testing.T) {
|
func TestEvaluateStep(t *testing.T) {
|
||||||
rc := createRunContext(t)
|
rc := createRunContext(t)
|
||||||
rc.Env["INPUT_FORGED"] = "leaked"
|
rc.Env["INPUT_FORGED"] = "leaked"
|
||||||
@@ -182,6 +280,7 @@ func TestEvaluateStep(t *testing.T) {
|
|||||||
func TestInterpolate(t *testing.T) {
|
func TestInterpolate(t *testing.T) {
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
Workdir: ".",
|
Workdir: ".",
|
||||||
Secrets: map[string]string{
|
Secrets: map[string]string{
|
||||||
"CASE_INSENSITIVE_SECRET": "value",
|
"CASE_INSENSITIVE_SECRET": "value",
|
||||||
@@ -347,9 +446,9 @@ jobs:
|
|||||||
for _, value := range []string{"true", "false"} {
|
for _, value := range []string{"true", "false"} {
|
||||||
t.Run(value, func(t *testing.T) {
|
t.Run(value, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
parent, err := (&runnerImpl{config: &Config{}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "call"}, map[string]any{"args": `{"flag":` + value + `,"name":"${{ 'runner' }}"}`})
|
parent, err := (&runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "call"}, map[string]any{"args": `{"flag":` + value + `,"name":"${{ 'runner' }}"}`})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
child, err := (&runnerImpl{config: &Config{}, caller: &caller{runContext: parent}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
|
child, err := (&runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}, caller: &caller{runContext: parent}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, map[string]any{"flag": value == "true", "name": "${{ 'runner' }}", "unset": false}, child.workflowCallInputs)
|
assert.Equal(t, map[string]any{"flag": value == "true", "name": "${{ 'runner' }}", "unset": false}, child.workflowCallInputs)
|
||||||
assert.Equal(t, rawWith, job.RawWith)
|
assert.Equal(t, rawWith, job.RawWith)
|
||||||
@@ -405,7 +504,7 @@ jobs:
|
|||||||
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
|
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) {
|
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsAndRuntimeToken(t *testing.T) {
|
||||||
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
|
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
|
||||||
jobs:
|
jobs:
|
||||||
call:
|
call:
|
||||||
@@ -428,7 +527,11 @@ jobs:
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n"))
|
notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
config := &Config{Secrets: map[string]string{"token": "${{ github.token }}"}}
|
config := &Config{
|
||||||
|
Secrets: map[string]string{"token": "${{ github.token }}"},
|
||||||
|
PresetGitHubContext: &model.GithubContext{},
|
||||||
|
RuntimeTokens: map[*model.Workflow]string{callerWorkflow: "caller-run"},
|
||||||
|
}
|
||||||
resolve := func(callerJobID string, workflow *model.Workflow) *RunContext {
|
resolve := func(callerJobID string, workflow *model.Workflow) *RunContext {
|
||||||
parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil)
|
parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -440,9 +543,10 @@ jobs:
|
|||||||
called := resolve("call", callee)
|
called := resolve("call", callee)
|
||||||
assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs)
|
assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs)
|
||||||
assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets)
|
assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets)
|
||||||
|
assert.Equal(t, "caller-run", called.GetEnv()["ACTIONS_RUNTIME_TOKEN"])
|
||||||
inherited := resolve("inherit", notCallable)
|
inherited := resolve("inherit", notCallable)
|
||||||
assert.Empty(t, inherited.workflowCallInputs)
|
assert.Empty(t, inherited.workflowCallInputs)
|
||||||
assert.Equal(t, config.Secrets, inherited.workflowCallSecrets)
|
assert.Equal(t, map[string]string{"token": "${{ github.token }}", "ACTIONS_RUNTIME_TOKEN": "caller-run"}, inherited.workflowCallSecrets)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestJobNameMasksSecrets(t *testing.T) {
|
func TestJobNameMasksSecrets(t *testing.T) {
|
||||||
@@ -455,7 +559,7 @@ jobs:
|
|||||||
`))
|
`))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
runner := &runnerImpl{config: &Config{Secrets: map[string]string{"A": "s3cr3t-a", "B": "s3cr3t-b"}}}
|
runner := &runnerImpl{config: &Config{Secrets: map[string]string{"A": "s3cr3t-a", "B": "s3cr3t-b"}, PresetGitHubContext: &model.GithubContext{}}}
|
||||||
containerName := func(jobID string) string {
|
containerName := func(jobID string) string {
|
||||||
rc, err := runner.newRunContext(t.Context(), &model.Run{JobID: jobID, Workflow: workflow}, nil)
|
rc, err := runner.newRunContext(t.Context(), &model.Run{JobID: jobID, Workflow: workflow}, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -60,3 +60,7 @@ func requireHostTools(t *testing.T, tools ...string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testCheckoutEnv() map[string]string {
|
||||||
|
return map[string]string{"GITHUB_REPOSITORY": "owner/repo", "GITHUB_REF": "refs/heads/main", "SHA_REF": "1234567890"}
|
||||||
|
}
|
||||||
@@ -111,7 +111,7 @@ func printPrepareActions(rc *RunContext, preparers []actionPreparer) common.Exec
|
|||||||
if !ok || strings.Contains(action.Step.Uses, "${{") || action.action != nil {
|
if !ok || strings.Contains(action.Step.Uses, "${{") || action.action != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
github := action.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
github := action.RunContext.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||||
remote, err := newRemoteAction(action.Step.Uses, github, action.RunContext.Config.GitHubInstance)
|
remote, err := newRemoteAction(action.Step.Uses, github, action.RunContext.Config.GitHubInstance)
|
||||||
if err != nil || action.skipsLocalCheckout(remote, github) {
|
if err != nil || action.skipsLocalCheckout(remote, github) {
|
||||||
continue
|
continue
|
||||||
@@ -285,7 +285,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
|
|||||||
jobError := common.JobError(ctx)
|
jobError := common.JobError(ctx)
|
||||||
var err error
|
var err error
|
||||||
// always allow 1 min for stopping and removing the runner, even if we were cancelled
|
// always allow 1 min for stopping and removing the runner, even if we were cancelled
|
||||||
ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), jobCleanupTimeout)
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
@@ -400,6 +400,7 @@ func runMainStepsAfterInterrupt(ctx context.Context, rc *RunContext, steps []com
|
|||||||
|
|
||||||
func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) {
|
func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
|
success = success && !rc.jobFailed
|
||||||
|
|
||||||
// Matrix combinations share one *model.Job and run in parallel; serialize the
|
// Matrix combinations share one *model.Job and run in parallel; serialize the
|
||||||
// read-modify-write of the job result so a failing combination is not lost-updated by a
|
// read-modify-write of the job result so a failing combination is not lost-updated by a
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ func (jim *jobInfoMock) result(result string) {
|
|||||||
|
|
||||||
type jobContainerMock struct {
|
type jobContainerMock struct {
|
||||||
container.Container
|
container.Container
|
||||||
container.LinuxContainerEnvironmentExtensions
|
linuxEnvironment
|
||||||
}
|
}
|
||||||
|
|
||||||
func (jcm *jobContainerMock) ReplaceLogWriter(_, _ io.Writer) (io.Writer, io.Writer) {
|
func (jcm *jobContainerMock) ReplaceLogWriter(_, _ io.Writer) (io.Writer, io.Writer) {
|
||||||
@@ -290,7 +290,7 @@ func TestNewJobExecutorDownloadsAllActionsBeforeTheFirstStep(t *testing.T) {
|
|||||||
Jobs: map[string]*model.Job{"test": {}},
|
Jobs: map[string]*model.Job{"test": {}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
}
|
}
|
||||||
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
||||||
|
|
||||||
@@ -550,7 +550,7 @@ func TestNewJobExecutor(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
}
|
}
|
||||||
if tt.output != "" {
|
if tt.output != "" {
|
||||||
require.NoError(t, rc.Run.Job().RawOutputs.Encode(map[string]string{"bad": tt.output}))
|
require.NoError(t, rc.Run.Job().RawOutputs.Encode(map[string]string{"bad": tt.output}))
|
||||||
@@ -695,6 +695,7 @@ func TestSetJobResultMatrixContinueOnError(t *testing.T) {
|
|||||||
workflow := &model.Workflow{Name: "workflow1", Jobs: map[string]*model.Job{"job1": job}}
|
workflow := &model.Workflow{Name: "workflow1", Jobs: map[string]*model.Job{"job1": job}}
|
||||||
for _, experimental := range []bool{true, tolerated} {
|
for _, experimental := range []bool{true, tolerated} {
|
||||||
rc := newTestRC(workflow, map[string]any{"experimental": experimental})
|
rc := newTestRC(workflow, map[string]any{"experimental": experimental})
|
||||||
|
rc.Config.PresetGitHubContext = &model.GithubContext{}
|
||||||
setJobResult(ctx, rc, rc, false)
|
setJobResult(ctx, rc, rc, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1023,7 +1024,7 @@ func tarArchive(t *testing.T, entries ...tarEntry) []byte {
|
|||||||
|
|
||||||
func newTestRC(wf *model.Workflow, matrix map[string]any) *RunContext {
|
func newTestRC(wf *model.Workflow, matrix map[string]any) *RunContext {
|
||||||
return &RunContext{
|
return &RunContext{
|
||||||
Config: &Config{Workdir: ".", PlatformPicker: func([]string) string { return "ubuntu-latest" }},
|
Config: &Config{Workdir: ".", Env: testCheckoutEnv(), PlatformPicker: func([]string) string { return "ubuntu-latest" }},
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
Env: map[string]string{},
|
Env: map[string]string{},
|
||||||
Matrix: matrix,
|
Matrix: matrix,
|
||||||
@@ -1039,29 +1040,32 @@ func makeTestRC(t *testing.T, jobYAML string) *RunContext {
|
|||||||
Name: "workflow1",
|
Name: "workflow1",
|
||||||
Jobs: map[string]*model.Job{"job1": job},
|
Jobs: map[string]*model.Job{"job1": job},
|
||||||
}, nil)
|
}, nil)
|
||||||
|
rc.Config.PresetGitHubContext = &model.GithubContext{}
|
||||||
rc.ExprEval = rc.NewExpressionEvaluator(t.Context())
|
rc.ExprEval = rc.NewExpressionEvaluator(t.Context())
|
||||||
return rc
|
return rc
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyJobTimeout(t *testing.T) {
|
func TestApplyJobTimeout(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
yaml string
|
yaml string
|
||||||
wantTimeout bool
|
want time.Duration
|
||||||
}{
|
}{
|
||||||
{"empty", "runs-on: ubuntu-latest", false},
|
{"empty", "runs-on: ubuntu-latest", 0},
|
||||||
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", true},
|
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", 5 * time.Minute},
|
||||||
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", false},
|
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", 0},
|
||||||
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", false},
|
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", 0},
|
||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
rc := makeTestRC(t, tc.yaml)
|
rc := makeTestRC(t, tc.yaml)
|
||||||
ctx := t.Context()
|
newCtx, cancel := applyJobTimeout(t.Context(), rc, rc.Run.Job())
|
||||||
newCtx, cancel := applyJobTimeout(ctx, rc, rc.Run.Job())
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
_, hasDeadline := newCtx.Deadline()
|
deadline, hasDeadline := newCtx.Deadline()
|
||||||
assert.Equal(t, tc.wantTimeout, hasDeadline)
|
assert.Equal(t, tc.want != 0, hasDeadline)
|
||||||
|
if hasDeadline {
|
||||||
|
assert.WithinDuration(t, time.Now().Add(tc.want), deadline, time.Minute)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-10
@@ -40,15 +40,18 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
|
|||||||
if hookPath == "" {
|
if hookPath == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
previousStep := rc.CurrentStep
|
||||||
|
rc.CurrentStep = ""
|
||||||
|
defer func() { rc.CurrentStep = previousStep }()
|
||||||
|
|
||||||
cmd, shell := hookCommand(hookPath)
|
cmd, shell := hookCommand(hookPath)
|
||||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
||||||
defer rawLogger.Infof("::endgroup::")
|
|
||||||
rawLogger.Infof("::group::Run '%s'", EscapeCommandData(hookPath))
|
|
||||||
rawLogger.Infof("A %s hook has been configured by the runner administrator", name)
|
rawLogger.Infof("A %s hook has been configured by the runner administrator", name)
|
||||||
|
rawLogger.Infof("::group::Run '%s'", EscapeCommandData(hookPath))
|
||||||
if shell != "" {
|
if shell != "" {
|
||||||
rawLogger.Infof("shell: %s", shell)
|
rawLogger.Infof("shell: %s", shell)
|
||||||
}
|
}
|
||||||
|
rawLogger.Infof("::endgroup::")
|
||||||
|
|
||||||
env := mergeMaps(rc.containerSpec.Env, rc.GetEnv())
|
env := mergeMaps(rc.containerSpec.Env, rc.GetEnv())
|
||||||
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
||||||
@@ -56,12 +59,12 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
|
|||||||
|
|
||||||
err := rc.setupHookFileCommands(ctx, env)
|
err := rc.setupHookFileCommands(ctx, env)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = rc.withCommandLogWriter(ctx, rc.JobContainer.Exec(cmd, env, "", ""))
|
err = rc.runWithCommands(ctx, env, func(ctx context.Context) error {
|
||||||
}
|
return rc.withCommandLogWriter(ctx, rc.JobContainer.Exec(cmd, env, "", ""))
|
||||||
// Processed even on failure, so a hook that exports what it managed to set up before
|
})
|
||||||
// failing still hands it to the job.
|
if processErr := rc.processHookFileCommands(ctx); err == nil { // a failed hook still exports what it set up
|
||||||
if processErr := rc.processHookFileCommands(ctx); err == nil {
|
err = processErr
|
||||||
err = processErr
|
}
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -81,8 +84,7 @@ func (rc *RunContext) setupHookFileCommands(ctx context.Context, env map[string]
|
|||||||
actPath := rc.JobContainer.GetActPath()
|
actPath := rc.JobContainer.GetActPath()
|
||||||
env["GITHUB_ENV"] = path.Join(actPath, hookEnvFileCommand)
|
env["GITHUB_ENV"] = path.Join(actPath, hookEnvFileCommand)
|
||||||
env["GITHUB_PATH"] = path.Join(actPath, hookPathFileCommand)
|
env["GITHUB_PATH"] = path.Join(actPath, hookPathFileCommand)
|
||||||
env["GITEA_ENV"] = env["GITHUB_ENV"]
|
aliasGiteaEnv(env)
|
||||||
env["GITEA_PATH"] = env["GITHUB_PATH"]
|
|
||||||
|
|
||||||
return rc.JobContainer.Copy(actPath,
|
return rc.JobContainer.Copy(actPath,
|
||||||
&container.FileEntry{Name: hookEnvFileCommand, Mode: 0o666},
|
&container.FileEntry{Name: hookEnvFileCommand, Mode: 0o666},
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
"gitea.com/gitea/runner/act/container"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -27,6 +28,7 @@ type hookContainer struct {
|
|||||||
cmd []string
|
cmd []string
|
||||||
env map[string]string
|
env map[string]string
|
||||||
err error
|
err error
|
||||||
|
copyErr error
|
||||||
envFile map[string]string
|
envFile map[string]string
|
||||||
pathTar []byte
|
pathTar []byte
|
||||||
output string
|
output string
|
||||||
@@ -58,6 +60,10 @@ func (c *hookContainer) ReplaceLogWriter(stdout, _ io.Writer) (io.Writer, io.Wri
|
|||||||
return old, nil
|
return old, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *hookContainer) Copy(_destPath string, _files ...*container.FileEntry) common.Executor {
|
||||||
|
return common.NewErrorExecutor(c.copyErr)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.Executor {
|
func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.Executor {
|
||||||
return func(context.Context) error {
|
return func(context.Context) error {
|
||||||
maps.Copy(*env, c.envFile)
|
maps.Copy(*env, c.envFile)
|
||||||
@@ -72,7 +78,7 @@ func (c *hookContainer) GetContainerArchive(context.Context, string) (io.ReadClo
|
|||||||
// newHookRunContext returns a RunContext and the context to run a hook with, whose logger is
|
// newHookRunContext returns a RunContext and the context to run a hook with, whose logger is
|
||||||
// silenced so the hook's job-log output does not reach the test output.
|
// silenced so the hook's job-log output does not reach the test output.
|
||||||
func newHookRunContext(jobContainer *hookContainer, config *Config) (*RunContext, context.Context) {
|
func newHookRunContext(jobContainer *hookContainer, config *Config) (*RunContext, context.Context) {
|
||||||
// Env is left nil so that it is built from the config, as it is for a real job.
|
config.PresetGitHubContext = &model.GithubContext{}
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: config,
|
Config: config,
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
@@ -85,15 +91,29 @@ func newHookRunContext(jobContainer *hookContainer, config *Config) (*RunContext
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunJobHook(t *testing.T) {
|
func TestRunJobHook(t *testing.T) {
|
||||||
t.Run("runs the hook with the job environment", func(t *testing.T) {
|
t.Run("runs the hook with the job environment after its closed header group", func(t *testing.T) {
|
||||||
jobContainer := &hookContainer{}
|
jobContainer := &hookContainer{output: "from the hook\n"}
|
||||||
rc, ctx := newHookRunContext(jobContainer, &Config{
|
rc, ctx := newHookRunContext(jobContainer, &Config{
|
||||||
JobStartedHook: "/hooks/started.sh",
|
JobStartedHook: "/hooks/started.sh",
|
||||||
Env: map[string]string{"A_VAR": "value", jobStartedHookEnv: "/from/env.sh"},
|
Env: map[string]string{"A_VAR": "value", jobStartedHookEnv: "/from/env.sh"},
|
||||||
})
|
})
|
||||||
|
records := logtest.New()
|
||||||
|
ctx = common.WithLogger(ctx, logging.New(records))
|
||||||
|
|
||||||
require.NoError(t, rc.runJobStartedHook(ctx))
|
require.NoError(t, rc.runJobStartedHook(ctx))
|
||||||
|
|
||||||
|
var lines []string
|
||||||
|
for _, record := range records.AllRecords() {
|
||||||
|
lines = append(lines, record.Message)
|
||||||
|
}
|
||||||
|
assert.Equal(t, []string{
|
||||||
|
"A job started hook has been configured by the runner administrator",
|
||||||
|
"::group::Run '/hooks/started.sh'",
|
||||||
|
"shell: bash -e {0}",
|
||||||
|
"::endgroup::",
|
||||||
|
"from the hook\n",
|
||||||
|
}, lines)
|
||||||
|
|
||||||
// The setting wins over the environment variable.
|
// The setting wins over the environment variable.
|
||||||
assert.Equal(t, []string{"bash", "-e", "/hooks/started.sh"}, jobContainer.cmd)
|
assert.Equal(t, []string{"bash", "-e", "/hooks/started.sh"}, jobContainer.cmd)
|
||||||
assert.Equal(t, "value", jobContainer.env["A_VAR"])
|
assert.Equal(t, "value", jobContainer.env["A_VAR"])
|
||||||
@@ -101,6 +121,7 @@ func TestRunJobHook(t *testing.T) {
|
|||||||
assert.Equal(t, "job", jobContainer.env["GITHUB_JOB"])
|
assert.Equal(t, "job", jobContainer.env["GITHUB_JOB"])
|
||||||
assert.Equal(t, "/var/run/act/workflow/hook-envs.txt", jobContainer.env["GITHUB_ENV"])
|
assert.Equal(t, "/var/run/act/workflow/hook-envs.txt", jobContainer.env["GITHUB_ENV"])
|
||||||
assert.Equal(t, "/var/run/act/workflow/hook-path.txt", jobContainer.env["GITHUB_PATH"])
|
assert.Equal(t, "/var/run/act/workflow/hook-path.txt", jobContainer.env["GITHUB_PATH"])
|
||||||
|
assert.Equal(t, jobContainer.env["GITHUB_PATH"], jobContainer.env["GITEA_PATH"])
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("each hook has its own command state", func(t *testing.T) {
|
t.Run("each hook has its own command state", func(t *testing.T) {
|
||||||
@@ -116,6 +137,44 @@ func TestRunJobHook(t *testing.T) {
|
|||||||
assert.Same(t, jobWriter, jobContainer.stdout)
|
assert.Same(t, jobWriter, jobContainer.stdout)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("fails on refused commands and honors the container opt-in", func(t *testing.T) {
|
||||||
|
for _, allow := range []bool{false, true} {
|
||||||
|
jobContainer := &hookContainer{output: "::set-env name=FROM_HOOK::1\n::add-path::/opt/hook/bin"}
|
||||||
|
rc, ctx := newHookRunContext(jobContainer, &Config{JobStartedHook: "/started.sh"})
|
||||||
|
if allow {
|
||||||
|
rc.containerSpec.Env = map[string]string{allowUnsecureCommandsVar: "true"}
|
||||||
|
}
|
||||||
|
|
||||||
|
err := rc.runJobStartedHook(ctx)
|
||||||
|
if allow {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "1", rc.Env["FROM_HOOK"])
|
||||||
|
assert.Equal(t, []string{"/opt/hook/bin"}, rc.ExtraPath)
|
||||||
|
} else {
|
||||||
|
require.ErrorContains(t, err, "workflow command is disabled")
|
||||||
|
assert.Equal(t, "failure", rc.getJobContext().Status)
|
||||||
|
require.ErrorContains(t, common.JobError(ctx), "workflow command is disabled")
|
||||||
|
assert.NotContains(t, rc.Env, "FROM_HOOK")
|
||||||
|
assert.Empty(t, rc.ExtraPath)
|
||||||
|
}
|
||||||
|
assert.Nil(t, rc.currentStepEnv())
|
||||||
|
require.NoError(t, rc.takeUnsecureCommandError())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the completed hook has no preceding step identity", func(t *testing.T) {
|
||||||
|
jobContainer := &hookContainer{output: "::set-output name=value::after\n::save-state name=value::after"}
|
||||||
|
rc, ctx := newHookRunContext(jobContainer, &Config{JobCompletedHook: "/completed.sh"})
|
||||||
|
rc.CurrentStep = "last-step"
|
||||||
|
rc.StepResults = map[string]*model.StepResult{"last-step": {Outputs: map[string]string{"value": "before"}}}
|
||||||
|
|
||||||
|
require.NoError(t, rc.runJobCompletedHook(ctx))
|
||||||
|
assert.Empty(t, jobContainer.env["GITHUB_ACTION"])
|
||||||
|
assert.Equal(t, "before", rc.StepResults["last-step"].Outputs["value"])
|
||||||
|
assert.Empty(t, rc.IntraActionState)
|
||||||
|
assert.Equal(t, "last-step", rc.CurrentStep)
|
||||||
|
})
|
||||||
|
|
||||||
// Each hook reads its own variable, so a swapped constant cannot pass.
|
// Each hook reads its own variable, so a swapped constant cannot pass.
|
||||||
t.Run("falls back to the GitHub environment variables", func(t *testing.T) {
|
t.Run("falls back to the GitHub environment variables", func(t *testing.T) {
|
||||||
for name, hook := range map[string]struct {
|
for name, hook := range map[string]struct {
|
||||||
@@ -161,6 +220,15 @@ func TestRunJobHook(t *testing.T) {
|
|||||||
require.ErrorContains(t, common.JobError(ctx), "boom")
|
require.ErrorContains(t, common.JobError(ctx), "boom")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("a hook whose file commands cannot be reset neither runs nor exports stale ones", func(t *testing.T) {
|
||||||
|
jobContainer := &hookContainer{copyErr: errors.New("boom"), envFile: map[string]string{"STALE": "1"}}
|
||||||
|
rc, ctx := newHookRunContext(jobContainer, &Config{JobStartedHook: "/hooks/started.sh"})
|
||||||
|
|
||||||
|
require.ErrorContains(t, rc.runJobStartedHook(ctx), "boom")
|
||||||
|
assert.Nil(t, jobContainer.cmd)
|
||||||
|
assert.NotContains(t, rc.Env, "STALE")
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("is a no-op without a hook", func(t *testing.T) {
|
t.Run("is a no-op without a hook", func(t *testing.T) {
|
||||||
jobContainer := &hookContainer{}
|
jobContainer := &hookContainer{}
|
||||||
rc, ctx := newHookRunContext(jobContainer, &Config{})
|
rc, ctx := newHookRunContext(jobContainer, &Config{})
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ type masksContextKey string
|
|||||||
|
|
||||||
const masksContextKeyVal = masksContextKey("logrus.FieldLogger")
|
const masksContextKeyVal = masksContextKey("logrus.FieldLogger")
|
||||||
|
|
||||||
|
type valueMaskerContextKey struct{}
|
||||||
|
|
||||||
// Logger returns the appropriate logger for current context
|
// Logger returns the appropriate logger for current context
|
||||||
func Masks(ctx context.Context) *[]string {
|
func Masks(ctx context.Context) *[]string {
|
||||||
if masks, ok := ctx.Value(masksContextKeyVal).(*[]string); ok {
|
if masks, ok := ctx.Value(masksContextKeyVal).(*[]string); ok {
|
||||||
@@ -88,6 +90,8 @@ func WithJobLoggerFactory(ctx context.Context, factory JobLoggerFactory) context
|
|||||||
|
|
||||||
// WithJobLogger attaches a new logger to context that is aware of steps
|
// WithJobLogger attaches a new logger to context that is aware of steps
|
||||||
func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, masks *[]string, matrix map[string]any) context.Context {
|
func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, masks *[]string, matrix map[string]any) context.Context {
|
||||||
|
masker := valueMasker(config.InsecureSecrets, config.maskers())
|
||||||
|
ctx = context.WithValue(ctx, valueMaskerContextKey{}, masker)
|
||||||
ctx = WithMasks(ctx, masks)
|
ctx = WithMasks(ctx, masks)
|
||||||
|
|
||||||
var output logging.Handler
|
var output logging.Handler
|
||||||
@@ -109,7 +113,7 @@ func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, m
|
|||||||
}
|
}
|
||||||
handlers = append(handlers, &maskedHandler{
|
handlers = append(handlers, &maskedHandler{
|
||||||
Handler: output,
|
Handler: output,
|
||||||
masker: valueMasker(config.InsecureSecrets, config.maskers()),
|
masker: masker,
|
||||||
})
|
})
|
||||||
|
|
||||||
level := logging.LevelTrace
|
level := logging.LevelTrace
|
||||||
@@ -140,6 +144,9 @@ func WithCompositeStepLogger(ctx context.Context, stepID string) context.Context
|
|||||||
}
|
}
|
||||||
|
|
||||||
func withStepLogger(ctx context.Context, stepNumber int, stepID, stepName, stageName string) context.Context {
|
func withStepLogger(ctx context.Context, stepNumber int, stepID, stepName, stageName string) context.Context {
|
||||||
|
if masker, ok := ctx.Value(valueMaskerContextKey{}).(entryProcessor); ok {
|
||||||
|
stepName = masker(ctx, logging.Record{Message: stepName}).Message
|
||||||
|
}
|
||||||
stepIDs := []string{stepID}
|
stepIDs := []string{stepID}
|
||||||
ctx = common.WithStepIDs(ctx, stepIDs)
|
ctx = common.WithStepIDs(ctx, stepIDs)
|
||||||
|
|
||||||
|
|||||||
@@ -255,7 +255,11 @@ func TestWithJobLoggerFansOutAndMasksOnlyOutput(t *testing.T) {
|
|||||||
|
|
||||||
func TestJobLogJSONHandler(t *testing.T) {
|
func TestJobLogJSONHandler(t *testing.T) {
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
logger := logging.New(newJobLogJSONHandler(buf)).With("job", "build", "msg", "clash").WithError(errors.New("boom"))
|
rc := &RunContext{Config: &Config{Secrets: map[string]string{"TOKEN": "s3cr3t-value"}}}
|
||||||
|
ctx := WithJobLoggerFactory(t.Context(), factoryFunc(func() logging.Handler { return newJobLogJSONHandler(buf) }))
|
||||||
|
ctx = WithJobLogger(ctx, "j1", "build", rc.Config, &rc.Masks, nil)
|
||||||
|
logger := common.Logger(withStepLogger(ctx, 1, "s1", "deploy s3cr3t-value", StageMain)).
|
||||||
|
With("msg", "clash").WithError(errors.New("boom"))
|
||||||
logger.Warnf("careful %d", 1)
|
logger.Warnf("careful %d", 1)
|
||||||
|
|
||||||
var got map[string]any
|
var got map[string]any
|
||||||
@@ -263,7 +267,15 @@ func TestJobLogJSONHandler(t *testing.T) {
|
|||||||
assert.Equal(t, "careful 1", got["msg"])
|
assert.Equal(t, "careful 1", got["msg"])
|
||||||
assert.Equal(t, "warning", got["level"])
|
assert.Equal(t, "warning", got["level"])
|
||||||
assert.Equal(t, "build", got["job"])
|
assert.Equal(t, "build", got["job"])
|
||||||
|
assert.Equal(t, "deploy ***", got["step"])
|
||||||
assert.Equal(t, "boom", got["error"])
|
assert.Equal(t, "boom", got["error"])
|
||||||
assert.Equal(t, "clash", got["fields.msg"])
|
assert.Equal(t, "clash", got["fields.msg"])
|
||||||
assert.NotEmpty(t, got["time"])
|
assert.NotEmpty(t, got["time"])
|
||||||
|
|
||||||
|
rc.AddMask(ctx, "runtime-value")
|
||||||
|
buf.Reset()
|
||||||
|
common.Logger(withStepLogger(ctx, 2, "s2", "deploy s3cr3t-value runtime-value", StageMain)).Info("done")
|
||||||
|
|
||||||
|
require.NoError(t, json.Unmarshal(buf.Bytes(), &got))
|
||||||
|
assert.Equal(t, "deploy *** ***", got["step"])
|
||||||
}
|
}
|
||||||
@@ -47,7 +47,7 @@ jobs:
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
job := workflow.Jobs["test"]
|
job := workflow.Jobs["test"]
|
||||||
rawStrategy := model.CloneYamlNode(job.RawStrategy)
|
rawStrategy := model.CloneYamlNode(job.RawStrategy)
|
||||||
require.NoError(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "test"}}}}})(t.Context()))
|
require.NoError(t, (&runnerImpl{config: &Config{Env: testCheckoutEnv()}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "test"}}}}})(t.Context()))
|
||||||
require.NotNil(t, job.Strategy)
|
require.NotNil(t, job.Strategy)
|
||||||
assert.False(t, job.Strategy.GetFailFast())
|
assert.False(t, job.Strategy.GetFailFast())
|
||||||
assert.Equal(t, 2, maxParallelFor(job.Strategy, 5))
|
assert.Equal(t, 2, maxParallelFor(job.Strategy, 5))
|
||||||
@@ -70,7 +70,7 @@ func TestNewPlanExecutorInvalidMatrix(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
JobID: "test",
|
JobID: "test",
|
||||||
}}}}}
|
}}}}}
|
||||||
runner := &runnerImpl{config: &Config{}}
|
runner := &runnerImpl{config: &Config{Env: testCheckoutEnv()}}
|
||||||
|
|
||||||
require.ErrorContains(t, runner.NewPlanExecutor(plan)(t.Context()), "could not get job matrix:")
|
require.ErrorContains(t, runner.NewPlanExecutor(plan)(t.Context()), "could not get job matrix:")
|
||||||
|
|
||||||
|
|||||||
+24
-29
@@ -12,12 +12,13 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/container"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Actions bundle the @actions toolkit into their own JavaScript, and two of its lines keep it
|
// Actions bundle the @actions toolkit into their own JavaScript, and two of its lines keep it
|
||||||
// from working against Gitea. Both are edited out of the bundle the runner downloaded.
|
// from working against Gitea. Both are edited out of the copy each job container gets.
|
||||||
//
|
//
|
||||||
// isGhes() takes any host that is not github.com, *.ghe.com or *.localhost for GitHub
|
// isGhes() takes any host that is not github.com, *.ghe.com or *.localhost for GitHub
|
||||||
// Enterprise. @actions/cache then forces the v1 API, and @actions/artifact refuses outright,
|
// Enterprise. @actions/cache then forces the v1 API, and @actions/artifact refuses outright,
|
||||||
@@ -86,9 +87,10 @@ func quoted(pattern string) string {
|
|||||||
return "(?:'" + pattern + "'|\"" + pattern + "\"|`" + pattern + "`)"
|
return "(?:'" + pattern + "'|\"" + pattern + "\"|`" + pattern + "`)"
|
||||||
}
|
}
|
||||||
|
|
||||||
// actionScriptPaths returns a node action's entrypoints and the chunks bundled beside them. Only
|
// actionScriptPaths returns a node action's entrypoints and the chunks bundled beside them,
|
||||||
// remote actions get here: a local one lives in the user's checkout, which the runner does not
|
// relative to checkout. Only remote actions get here: a local one lives in the user's checkout,
|
||||||
// rewrite. Scripts may leave dir for the rest of the checkout, as setup-gradle's ../dist does.
|
// which the runner does not rewrite. Scripts may leave dir for the rest of the checkout, as
|
||||||
|
// setup-gradle's ../dist does.
|
||||||
func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
||||||
if action == nil || !action.Runs.Using.IsNode() {
|
if action == nil || !action.Runs.Using.IsNode() {
|
||||||
return nil
|
return nil
|
||||||
@@ -107,12 +109,12 @@ func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// `runs` is the action's own yaml, and a key resolving outside its checkout is not ours.
|
// `runs` is the action's own yaml, and a key resolving outside its checkout is not ours.
|
||||||
bundleDir := filepath.Dir(path)
|
bundleDir, err := filepath.Rel(root, filepath.Dir(path))
|
||||||
if rel, err := filepath.Rel(root, bundleDir); err != nil || !filepath.IsLocal(rel) {
|
if err != nil || !filepath.IsLocal(bundleDir) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
paths = append(paths, path)
|
paths = append(paths, filepath.Join(bundleDir, filepath.Base(path)))
|
||||||
entries, _ := os.ReadDir(bundleDir)
|
entries, _ := os.ReadDir(filepath.Dir(path))
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if entry.Type().IsRegular() && slices.Contains([]string{".js", ".cjs", ".mjs"}, filepath.Ext(entry.Name())) {
|
if entry.Type().IsRegular() && slices.Contains([]string{".js", ".cjs", ".mjs"}, filepath.Ext(entry.Name())) {
|
||||||
paths = append(paths, filepath.Join(bundleDir, entry.Name()))
|
paths = append(paths, filepath.Join(bundleDir, entry.Name()))
|
||||||
@@ -123,45 +125,38 @@ func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
|||||||
return slices.Compact(paths)
|
return slices.Compact(paths)
|
||||||
}
|
}
|
||||||
|
|
||||||
// patchActions returns a restore: the tree is shared, so a job with patching off gets no edits.
|
// patchedScripts returns the scripts needing the edit, patched, to copy over the job container's copy.
|
||||||
func patchActions(ctx context.Context, scripts []string) func() {
|
func patchedScripts(ctx context.Context, checkout string, scripts []string) []*container.FileEntry {
|
||||||
restore := map[string][]byte{}
|
var patched []*container.FileEntry
|
||||||
for _, script := range scripts {
|
for _, script := range scripts {
|
||||||
original, err := patchBundle(script)
|
body, mode, err := patchedScript(filepath.Join(checkout, script))
|
||||||
if original != nil {
|
|
||||||
restore[script] = original // also when the write failed part way through
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Warnf("actions toolkit: %s left unpatched: %v", script, err)
|
common.Logger(ctx).Warnf("actions toolkit: %s left unpatched: %v", script, err)
|
||||||
|
} else if body != nil {
|
||||||
|
patched = append(patched, &container.FileEntry{Name: filepath.ToSlash(script), Mode: int64(mode), Body: string(body)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return func() {
|
return patched
|
||||||
for script, original := range restore {
|
|
||||||
if err := os.WriteFile(script, original, 0o644); err != nil { //nolint:gosec // as the checkout wrote it
|
|
||||||
common.Logger(ctx).Warnf("actions toolkit: %s left patched in the shared copy: %v", script, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// patchBundle returns the bytes it replaced, or nil when it left the bundle alone.
|
// patchedScript returns the bundle patched, or nil when it is left alone.
|
||||||
func patchBundle(script string) ([]byte, error) {
|
func patchedScript(script string) ([]byte, os.FileMode, error) {
|
||||||
info, err := os.Stat(script)
|
info, err := os.Stat(script)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
if info.Size() > maxBundleSize {
|
if info.Size() > maxBundleSize {
|
||||||
return nil, nil
|
return nil, 0, nil
|
||||||
}
|
}
|
||||||
data, err := os.ReadFile(script)
|
data, err := os.ReadFile(script)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
patched, ok := patchedBundle(data)
|
patched, ok := patchedBundle(data)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, nil
|
return nil, 0, nil
|
||||||
}
|
}
|
||||||
return data, os.WriteFile(script, patched, info.Mode().Perm())
|
return patched, info.Mode().Perm(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// patchedBundle opens the GHES gate, and where the cache toolkit is present, points the cache
|
// patchedBundle opens the GHES gate, and where the cache toolkit is present, points the cache
|
||||||
|
|||||||
@@ -122,9 +122,11 @@ func patchedAction(t *testing.T, repo, ref, entrypoint string) string {
|
|||||||
|
|
||||||
body, err := os.ReadFile(bundleFromGitHub(t, repo, ref, entrypoint))
|
body, err := os.ReadFile(bundleFromGitHub(t, repo, ref, entrypoint))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
if patched, ok := patchedBundle(body); ok {
|
||||||
|
body = patched
|
||||||
|
}
|
||||||
script := filepath.Join(tempDirPath(t), filepath.Base(entrypoint))
|
script := filepath.Join(tempDirPath(t), filepath.Base(entrypoint))
|
||||||
require.NoError(t, os.WriteFile(script, body, 0o600))
|
require.NoError(t, os.WriteFile(script, body, 0o600))
|
||||||
patchActions(t.Context(), []string{script})
|
|
||||||
return script
|
return script
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,6 +139,7 @@ func tempDirPath(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2EndToEnd(t *testing.T) {
|
func TestCacheServiceV2EndToEnd(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireHostTools(t, "node")
|
requireHostTools(t, "node")
|
||||||
|
|
||||||
restore := patchedAction(t, "actions/cache", actionsCacheRef, "dist/restore/index.js")
|
restore := patchedAction(t, "actions/cache", actionsCacheRef, "dist/restore/index.js")
|
||||||
@@ -220,6 +223,7 @@ func TestPatchedBundleAcrossActions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestUploadArtifactThroughTheResultsService(t *testing.T) {
|
func TestUploadArtifactThroughTheResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireHostTools(t, "node")
|
requireHostTools(t, "node")
|
||||||
|
|
||||||
var called []string
|
var called []string
|
||||||
@@ -288,6 +292,7 @@ func TestUploadArtifactThroughTheResultsService(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSetupActionFindsTheCacheService(t *testing.T) {
|
func TestSetupActionFindsTheCacheService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireHostTools(t, "node", "npm")
|
requireHostTools(t, "node", "npm")
|
||||||
|
|
||||||
setup := patchedAction(t, "actions/setup-node", "v7.1.0", "dist/setup/index.js")
|
setup := patchedAction(t, "actions/setup-node", "v7.1.0", "dist/setup/index.js")
|
||||||
@@ -327,7 +332,9 @@ func TestCodeSplitActionFindsTheCacheService(t *testing.T) {
|
|||||||
}
|
}
|
||||||
t.Setenv("CARGO_HOME", tempDirPath(t))
|
t.Setenv("CARGO_HOME", tempDirPath(t))
|
||||||
|
|
||||||
defer patchActions(t.Context(), actionScriptPaths(dir, dir, action))()
|
for _, entry := range patchedScripts(t.Context(), dir, actionScriptPaths(dir, dir, action)) {
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, entry.Name), []byte(entry.Body), 0o600))
|
||||||
|
}
|
||||||
out := runActionEntrypoint(t, filepath.Join(dir, action.Runs.Main),
|
out := runActionEntrypoint(t, filepath.Join(dir, action.Runs.Main),
|
||||||
newActionJobEnv(t, "split-runtime-token", artifactcache.JobCredential{Repo: "testuser/testrepo"}), inputs)
|
newActionJobEnv(t, "split-runtime-token", artifactcache.JobCredential{Repo: "testuser/testrepo"}), inputs)
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/container"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/mock"
|
"github.com/stretchr/testify/mock"
|
||||||
@@ -133,39 +135,36 @@ func gateOpened(body string) bool {
|
|||||||
// an ordinary Gitea host, the cache server for the service URL, and the results URL when there is
|
// an ordinary Gitea host, the cache server for the service URL, and the results URL when there is
|
||||||
// no cache server. Unpatched, the same bundle must still choose v1, or the patch proves nothing.
|
// no cache server. Unpatched, the same bundle must still choose v1, or the patch proves nothing.
|
||||||
func TestPatchedBundleBehavesInNode(t *testing.T) {
|
func TestPatchedBundleBehavesInNode(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireHostTools(t, "node")
|
requireHostTools(t, "node")
|
||||||
|
|
||||||
eval := func(t *testing.T, bundle, prelude, cacheURL string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
script := prelude + bundle + "\nprocess.stdout.write(getCacheServiceVersion()+' '+getCacheServiceURL())"
|
|
||||||
cmd := exec.CommandContext(t.Context(), "node", "-e", script)
|
|
||||||
cmd.Env = append(os.Environ(),
|
|
||||||
"ACTIONS_CACHE_SERVICE_V2=true",
|
|
||||||
"ACTIONS_CACHE_URL="+cacheURL,
|
|
||||||
"ACTIONS_RESULTS_URL=https://gitea.example",
|
|
||||||
"GITHUB_SERVER_URL=https://gitea.example",
|
|
||||||
)
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
require.NoError(t, err, "%s", out)
|
|
||||||
return string(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct{ name, bundle, prelude string }{
|
for _, tc := range []struct{ name, bundle, prelude string }{
|
||||||
{"tsc", gateTSC, ""},
|
{"tsc", gateTSC, ""},
|
||||||
{"webpack", gateWebpack, "const getCacheServiceVersion=()=>config_getCacheServiceVersion();"},
|
{"webpack", gateWebpack, "const getCacheServiceVersion=()=>config_getCacheServiceVersion();"},
|
||||||
{"esbuild", gateEsbuild, "var vu={};const getCacheServiceVersion=()=>XK(),getCacheServiceURL=()=>YK();"},
|
{"esbuild", gateEsbuild, "var vu={};const getCacheServiceVersion=()=>XK(),getCacheServiceURL=()=>YK();"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
// Unpatched, a Gitea host is taken for GHES: v1, whose branch already reads the
|
|
||||||
// cache URL. The patch has to move the version without moving that.
|
|
||||||
assert.Equal(t, "v1 http://cache:8088/", eval(t, tc.bundle, tc.prelude, "http://cache:8088/"))
|
|
||||||
|
|
||||||
patched, ok := patchedBundle([]byte(tc.bundle))
|
patched, ok := patchedBundle([]byte(tc.bundle))
|
||||||
require.True(t, ok)
|
require.True(t, ok)
|
||||||
|
|
||||||
assert.Equal(t, "v2 http://cache:8088/", eval(t, string(patched), tc.prelude, "http://cache:8088/"))
|
evaluate := func(bundle string) string {
|
||||||
assert.Equal(t, "v2 https://gitea.example", eval(t, string(patched), tc.prelude, ""),
|
return "(() => {" + tc.prelude + bundle + "\nconsole.log(getCacheServiceVersion()+' '+getCacheServiceURL());})();\n"
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(t.Context(), "node", "-e", evaluate(tc.bundle)+evaluate(string(patched))+
|
||||||
|
"process.env.ACTIONS_CACHE_URL='';\n"+evaluate(string(patched)))
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"ACTIONS_CACHE_SERVICE_V2=true",
|
||||||
|
"ACTIONS_CACHE_URL=http://cache:8088/",
|
||||||
|
"ACTIONS_RESULTS_URL=https://gitea.example",
|
||||||
|
"GITHUB_SERVER_URL=https://gitea.example",
|
||||||
|
)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, "%s", out)
|
||||||
|
results := strings.Split(strings.TrimSuffix(string(out), "\n"), "\n")
|
||||||
|
require.Len(t, results, 3)
|
||||||
|
assert.Equal(t, "v1 http://cache:8088/", results[0])
|
||||||
|
assert.Equal(t, "v2 http://cache:8088/", results[1])
|
||||||
|
assert.Equal(t, "v2 https://gitea.example", results[2],
|
||||||
"with no cache server the results URL is still the fallback")
|
"with no cache server the results URL is still the fallback")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -175,6 +174,7 @@ func TestPatchedBundleBehavesInNode(t *testing.T) {
|
|||||||
// the gate inside a double-quoted literal. Rewriting the call rather than emptying its argument
|
// the gate inside a double-quoted literal. Rewriting the call rather than emptying its argument
|
||||||
// would end that string early and leave the bundle unparseable.
|
// would end that string early and leave the bundle unparseable.
|
||||||
func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireHostTools(t, "node")
|
requireHostTools(t, "node")
|
||||||
|
|
||||||
escaped := strings.ReplaceAll(gateTSC, `"`, `\"`)
|
escaped := strings.ReplaceAll(gateTSC, `"`, `\"`)
|
||||||
@@ -188,34 +188,24 @@ func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
|||||||
require.NoError(t, err, "%s", checked)
|
require.NoError(t, err, "%s", checked)
|
||||||
}
|
}
|
||||||
|
|
||||||
// An action with a pre step is copied, and so patched, twice.
|
// A bundle an older runner left patched in the shared copy is copied as it is.
|
||||||
func TestPatchBundleIsIdempotent(t *testing.T) {
|
func TestPatchedScriptIsIdempotent(t *testing.T) {
|
||||||
script := bundleFile(t, gateTSC)
|
script := bundleFile(t, gateTSC)
|
||||||
|
|
||||||
original, err := patchBundle(script)
|
patched, _, err := patchedScript(script)
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, original)
|
|
||||||
patched, err := os.ReadFile(script)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(t, gateOpened(string(patched)))
|
require.True(t, gateOpened(string(patched)))
|
||||||
|
|
||||||
original, err = patchBundle(script)
|
require.NoError(t, os.WriteFile(script, patched, 0o600))
|
||||||
|
again, _, err := patchedScript(script)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Nil(t, original, "a patched bundle is not patched again")
|
assert.Nil(t, again, "a patched bundle is not patched again")
|
||||||
again, err := os.ReadFile(script)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, string(patched), string(again))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPatchBundleLeavesOtherActionsAlone(t *testing.T) {
|
func TestPatchedScriptLeavesOtherActionsAlone(t *testing.T) {
|
||||||
script := bundleFile(t, `console.log("checkout")`)
|
patched, _, err := patchedScript(bundleFile(t, `console.log("checkout")`))
|
||||||
|
|
||||||
original, err := patchBundle(script)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Nil(t, original)
|
assert.Nil(t, patched)
|
||||||
body, err := os.ReadFile(script)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, `console.log("checkout")`, string(body))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func bundleFile(t *testing.T, body string) string {
|
func bundleFile(t *testing.T, body string) string {
|
||||||
@@ -242,10 +232,10 @@ func TestActionScriptPaths(t *testing.T) {
|
|||||||
|
|
||||||
var chunks []string
|
var chunks []string
|
||||||
for _, name := range []string{"295.index.js", "cache-CTfQoCTL.js", "chunk.cjs", "chunk.mjs", "restore.js", "save.js"} {
|
for _, name := range []string{"295.index.js", "cache-CTfQoCTL.js", "chunk.cjs", "chunk.mjs", "restore.js", "save.js"} {
|
||||||
chunks = append(chunks, filepath.Join(dist, name))
|
chunks = append(chunks, filepath.Join("dist", name))
|
||||||
}
|
}
|
||||||
node := &model.Action{Runs: model.ActionRuns{Using: "node20", Pre: "pre.js", Main: "dist/restore.js", Post: "dist/save.js"}}
|
node := &model.Action{Runs: model.ActionRuns{Using: "node20", Pre: "pre.js", Main: "dist/restore.js", Post: "dist/save.js"}}
|
||||||
assert.Equal(t, append(slices.Clone(chunks), filepath.Join(dir, "pre.js")), actionScriptPaths(dir, dir, node))
|
assert.Equal(t, append(slices.Clone(chunks), "pre.js"), actionScriptPaths(dir, dir, node))
|
||||||
|
|
||||||
// Only a node action has a bundle to patch.
|
// Only a node action has a bundle to patch.
|
||||||
assert.Nil(t, actionScriptPaths(dir, dir, &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "alpine"}}))
|
assert.Nil(t, actionScriptPaths(dir, dir, &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "alpine"}}))
|
||||||
@@ -266,7 +256,7 @@ func TestActionScriptPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||||
copiedBundle := func(t *testing.T, noPatch bool) string {
|
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
@@ -285,27 +275,39 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
|||||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||||
|
|
||||||
var copied string
|
var calls []string
|
||||||
cm.On("CopyDir", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(func(context.Context) error {
|
var copied []*container.FileEntry
|
||||||
body, err := os.ReadFile(script)
|
cm.On("CopyDir", "/var/run/act/actions/repo/", sar.actionDir+"/", false, true).Return(func(context.Context) error {
|
||||||
require.NoError(t, err)
|
calls = append(calls, "CopyDir")
|
||||||
copied = string(body)
|
return nil
|
||||||
|
})
|
||||||
|
cm.On("Copy", "/var/run/act/actions/repo/", mock.Anything).Run(func(args mock.Arguments) {
|
||||||
|
copied = args.Get(1).([]*container.FileEntry)
|
||||||
|
}).Return(func(context.Context) error {
|
||||||
|
calls = append(calls, "Copy")
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
require.NoError(t, maybeCopyToActionDir(t.Context(), sar, sar.actionDir, "sub", "/var/run/act/actions/repo/sub"))
|
require.NoError(t, maybeCopyToActionDir(t.Context(), sar, sar.actionDir, "sub", "/var/run/act/actions/repo/sub"))
|
||||||
source, err := os.ReadFile(script)
|
source, err := os.ReadFile(script)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gateTSC, string(source), "the shared copy is left as it was found")
|
assert.Equal(t, gateTSC, string(source), "the shared copy is never written")
|
||||||
return copied
|
return calls, copied
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("patched on its way in", func(t *testing.T) {
|
t.Run("patched over the container's copy", func(t *testing.T) {
|
||||||
assert.True(t, gateOpened(copiedBundle(t, false)))
|
calls, copied := copiedBundles(t, false)
|
||||||
|
assert.Equal(t, []string{"CopyDir", "Copy"}, calls)
|
||||||
|
require.Len(t, copied, 1)
|
||||||
|
assert.Equal(t, "dist/cache.js", copied[0].Name)
|
||||||
|
assert.Equal(t, int64(0o600), copied[0].Mode)
|
||||||
|
assert.True(t, gateOpened(copied[0].Body))
|
||||||
})
|
})
|
||||||
|
|
||||||
// The escape hatch, for an action the edit breaks: the artifact actions refuse again, and the
|
// The escape hatch, for an action the edit breaks: the artifact actions refuse again, and the
|
||||||
// cache client keeps to v1.
|
// cache client keeps to v1.
|
||||||
t.Run("as shipped when the runner is told not to patch", func(t *testing.T) {
|
t.Run("as shipped when the runner is told not to patch", func(t *testing.T) {
|
||||||
assert.Equal(t, gateTSC, copiedBundle(t, true))
|
calls, copied := copiedBundles(t, true)
|
||||||
|
assert.Equal(t, []string{"CopyDir"}, calls)
|
||||||
|
assert.Empty(t, copied)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -23,13 +23,14 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
|||||||
return common.NewErrorExecutor(err)
|
return common.NewErrorExecutor(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rc.Config.NoSkipCheckout {
|
preset := rc.Config.PresetGitHubContext
|
||||||
|
if preset == nil {
|
||||||
// resolve the local workflow against the workspace root, not the process
|
// resolve the local workflow against the workspace root, not the process
|
||||||
// working directory, so it is found regardless of where the runner is invoked
|
// working directory, so it is found regardless of where the runner is invoked
|
||||||
return newReusableWorkflowExecutor(rc, rc.Config.Workdir, localWorkflow.Path, false)
|
return newReusableWorkflowExecutor(rc, rc.Config.Workdir, localWorkflow.Path, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
uses := fmt.Sprintf("%s/%s@%s", rc.Config.PresetGitHubContext.Repository, localWorkflow.Path, rc.Config.PresetGitHubContext.Sha)
|
uses := fmt.Sprintf("%s/%s@%s", preset.Repository, localWorkflow.Path, preset.Sha)
|
||||||
|
|
||||||
remoteReusableWorkflow, err := newRemoteReusableWorkflow(rc.Config.GitHubInstance, uses)
|
remoteReusableWorkflow, err := newRemoteReusableWorkflow(rc.Config.GitHubInstance, uses)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import (
|
|||||||
// ref (branch/tag) must reflect the new tip on every invocation, not stay pinned
|
// ref (branch/tag) must reflect the new tip on every invocation, not stay pinned
|
||||||
// to the cache populated on the first run.
|
// to the cache populated on the first run.
|
||||||
func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if _, err := exec.LookPath("git"); err != nil {
|
if _, err := exec.LookPath("git"); err != nil {
|
||||||
t.Skip("git not available in PATH")
|
t.Skip("git not available in PATH")
|
||||||
}
|
}
|
||||||
@@ -52,7 +53,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
|||||||
gitMust(t, workDir, "push", "-u", "origin", "master")
|
gitMust(t, workDir, "push", "-u", "origin", "master")
|
||||||
|
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{Env: testCheckoutEnv()},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "j1",
|
JobID: "j1",
|
||||||
Workflow: &model.Workflow{
|
Workflow: &model.Workflow{
|
||||||
@@ -121,7 +122,7 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
|||||||
|
|
||||||
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{Workdir: workdir},
|
Config: &Config{Workdir: workdir, NoSkipCheckout: true},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {Uses: uses}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {Uses: uses}}}},
|
||||||
}
|
}
|
||||||
err := newLocalReusableWorkflowExecutor(rc)(t.Context())
|
err := newLocalReusableWorkflowExecutor(rc)(t.Context())
|
||||||
@@ -149,6 +150,7 @@ func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T
|
|||||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
GitHubInstance: server.URL,
|
GitHubInstance: server.URL,
|
||||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||||
InsecureSkipTLS: true,
|
InsecureSkipTLS: true,
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ type RunContext struct {
|
|||||||
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
||||||
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
||||||
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
||||||
|
runtimeToken string // ACTIONS_RUNTIME_TOKEN of the workflow run, shared with its called workflows and composite actions
|
||||||
|
|
||||||
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
||||||
dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends
|
dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends
|
||||||
@@ -114,6 +115,7 @@ type RunContext struct {
|
|||||||
stepEnv map[string]string
|
stepEnv map[string]string
|
||||||
unsecureCommandErr error // refused ::set-env::/::add-path::, turned into a step failure
|
unsecureCommandErr error // refused ::set-env::/::add-path::, turned into a step failure
|
||||||
unsecureCommandMu sync.Mutex
|
unsecureCommandMu sync.Mutex
|
||||||
|
checkoutPins sync.Map // git lookups of contexts no run has pinned
|
||||||
}
|
}
|
||||||
|
|
||||||
// serviceContainer pairs a service container with the workflow id that keys job.services.
|
// serviceContainer pairs a service container with the workflow id that keys job.services.
|
||||||
@@ -192,6 +194,9 @@ func (rc *RunContext) GetEnv() map[string]string {
|
|||||||
rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if rc.runtimeToken != "" {
|
||||||
|
rc.Env["ACTIONS_RUNTIME_TOKEN"] = rc.runtimeToken // as the daemon's Config.Env carries its token
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !rc.Config.DisableActEnv {
|
if !rc.Config.DisableActEnv {
|
||||||
rc.Env["ACT"] = "true"
|
rc.Env["ACT"] = "true"
|
||||||
@@ -333,9 +338,12 @@ func splitVolumes(specs []string) ([]string, map[string]string, map[string]bool)
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
targets[parsed.Target] = true
|
targets[parsed.Target] = true
|
||||||
if parsed.Type == mount.TypeVolume && parsed.Source != "" && !parsed.ReadOnly {
|
switch {
|
||||||
|
case parsed.Type != mount.TypeVolume || parsed.Source == "" || parsed.ReadOnly:
|
||||||
|
binds = append(binds, spec)
|
||||||
|
case mounts[parsed.Source] == "":
|
||||||
mounts[parsed.Source] = parsed.Target
|
mounts[parsed.Source] = parsed.Target
|
||||||
} else {
|
case mounts[parsed.Source] != parsed.Target: // a repeat of the same mount is dropped
|
||||||
binds = append(binds, spec)
|
binds = append(binds, spec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -393,6 +401,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
|||||||
randBytes := make([]byte, 8)
|
randBytes := make([]byte, 8)
|
||||||
_, _ = rand.Read(randBytes)
|
_, _ = rand.Read(randBytes)
|
||||||
miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes))
|
miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes))
|
||||||
|
rc.cleanUpJobContainer = func(_ctx context.Context) error { return os.RemoveAll(miscpath) }
|
||||||
actPath := filepath.Join(miscpath, "act")
|
actPath := filepath.Join(miscpath, "act")
|
||||||
if err := os.MkdirAll(actPath, 0o777); err != nil {
|
if err := os.MkdirAll(actPath, 0o777); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1535,6 +1544,7 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext
|
|||||||
|
|
||||||
ghc.SetBaseAndHeadRef()
|
ghc.SetBaseAndHeadRef()
|
||||||
repoPath := rc.Config.Workdir
|
repoPath := rc.Config.Workdir
|
||||||
|
ctx = ghcontext.WithPinnedCheckout(ctx, &rc.checkoutPins)
|
||||||
ghcontext.SetRepositoryAndOwner(ctx, ghc, repoPath)
|
ghcontext.SetRepositoryAndOwner(ctx, ghc, repoPath)
|
||||||
if ghc.Ref == "" {
|
if ghc.Ref == "" {
|
||||||
ghcontext.SetRef(ctx, ghc, repoPath)
|
ghcontext.SetRef(ctx, ghc, repoPath)
|
||||||
@@ -1592,8 +1602,27 @@ func isLocalCheckout(ghc *model.GithubContext, step *model.Step) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// aliasGiteaEnv mirrors the runner's GITHUB_* variables as GITEA_*, call it after the last write to them.
|
||||||
|
func aliasGiteaEnv(env map[string]string) {
|
||||||
|
for _, suffix := range []string{
|
||||||
|
"ACTION", "ACTION_PATH", "ACTION_REF", "ACTION_REPOSITORY", "ACTIONS", "ACTOR", "ACTOR_ID", "API_URL", "BASE_REF",
|
||||||
|
"ENV", "EVENT_NAME", "EVENT_PATH", "GRAPHQL_URL", "HEAD_REF", "JOB", "OUTPUT", "PATH", "REF", "REF_NAME",
|
||||||
|
"REF_PROTECTED", "REF_TYPE", "REPOSITORY", "REPOSITORY_ID", "REPOSITORY_OWNER", "REPOSITORY_OWNER_ID",
|
||||||
|
"RETENTION_DAYS", "RUN_ATTEMPT", "RUN_ID", "RUN_NUMBER", "SERVER_URL", "SHA", "STATE", "STEP_SUMMARY",
|
||||||
|
"TRIGGERING_ACTOR", "WORKFLOW", "WORKFLOW_REF", "WORKFLOW_SHA", "WORKSPACE",
|
||||||
|
} {
|
||||||
|
if value, ok := env["GITHUB_"+suffix]; ok {
|
||||||
|
env["GITEA_"+suffix] = value
|
||||||
|
} else {
|
||||||
|
delete(env, "GITEA_"+suffix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) {
|
func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) {
|
||||||
env["CI"] = "true"
|
if _, ok := env["CI"]; !ok {
|
||||||
|
env["CI"] = "true"
|
||||||
|
}
|
||||||
env["GITHUB_WORKFLOW"] = github.Workflow
|
env["GITHUB_WORKFLOW"] = github.Workflow
|
||||||
env["GITHUB_RUN_ID"] = github.RunID
|
env["GITHUB_RUN_ID"] = github.RunID
|
||||||
env["GITHUB_RUN_NUMBER"] = github.RunNumber
|
env["GITHUB_RUN_NUMBER"] = github.RunNumber
|
||||||
@@ -1611,6 +1640,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
|||||||
env["GITHUB_REF"] = github.Ref
|
env["GITHUB_REF"] = github.Ref
|
||||||
env["GITHUB_REF_NAME"] = github.RefName
|
env["GITHUB_REF_NAME"] = github.RefName
|
||||||
env["GITHUB_REF_TYPE"] = github.RefType
|
env["GITHUB_REF_TYPE"] = github.RefType
|
||||||
|
env["GITHUB_REF_PROTECTED"] = strconv.FormatBool(github.RefProtected)
|
||||||
env["GITHUB_JOB"] = github.Job
|
env["GITHUB_JOB"] = github.Job
|
||||||
env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner
|
env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner
|
||||||
env["GITHUB_RETENTION_DAYS"] = github.RetentionDays
|
env["GITHUB_RETENTION_DAYS"] = github.RetentionDays
|
||||||
@@ -1620,6 +1650,18 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
|||||||
env["GITHUB_HEAD_REF"] = github.HeadRef
|
env["GITHUB_HEAD_REF"] = github.HeadRef
|
||||||
env["GITHUB_SERVER_URL"] = github.ServerURL
|
env["GITHUB_SERVER_URL"] = github.ServerURL
|
||||||
env["GITHUB_API_URL"] = github.APIURL
|
env["GITHUB_API_URL"] = github.APIURL
|
||||||
|
for name, value := range map[string]string{
|
||||||
|
"GITHUB_WORKFLOW_REF": github.WorkflowRef,
|
||||||
|
"GITHUB_WORKFLOW_SHA": github.WorkflowSha,
|
||||||
|
"GITHUB_ACTOR_ID": github.ActorID,
|
||||||
|
"GITHUB_TRIGGERING_ACTOR": github.TriggeringActor,
|
||||||
|
"GITHUB_REPOSITORY_ID": github.RepositoryID,
|
||||||
|
"GITHUB_REPOSITORY_OWNER_ID": github.RepositoryOwnerID,
|
||||||
|
} {
|
||||||
|
if value != "" { // a context without them keeps what the environment set
|
||||||
|
env[name] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, name := range []string{"GITHUB_SERVER_URL", "GITHUB_API_URL"} {
|
for _, name := range []string{"GITHUB_SERVER_URL", "GITHUB_API_URL"} {
|
||||||
if env[name] == "" {
|
if env[name] == "" {
|
||||||
@@ -1629,6 +1671,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
|||||||
|
|
||||||
env["GITHUB_GRAPHQL_URL"] = ""
|
env["GITHUB_GRAPHQL_URL"] = ""
|
||||||
env["GITHUB_RUN_ATTEMPT"] = github.RunAttempt
|
env["GITHUB_RUN_ATTEMPT"] = github.RunAttempt
|
||||||
|
aliasGiteaEnv(env)
|
||||||
|
|
||||||
env["RUNNER_NAME"] = rc.Config.RunnerName
|
env["RUNNER_NAME"] = rc.Config.RunnerName
|
||||||
env["RUNNER_ENVIRONMENT"] = "self-hosted"
|
env["RUNNER_ENVIRONMENT"] = "self-hosted"
|
||||||
@@ -1641,7 +1684,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
|||||||
|
|
||||||
if rc.Config.ArtifactServerPath != "" {
|
if rc.Config.ArtifactServerPath != "" {
|
||||||
env["ACTIONS_RUNTIME_URL"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_URL"), "http://"+net.JoinHostPort(rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort)+"/")
|
env["ACTIONS_RUNTIME_URL"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_URL"), "http://"+net.JoinHostPort(rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort)+"/")
|
||||||
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(rc.runtimeToken, os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
||||||
}
|
}
|
||||||
|
|
||||||
if imageOS := rc.imageOS(ctx); imageOS != "" {
|
if imageOS := rc.imageOS(ctx); imageOS != "" {
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ func TestRunContext_EvalBool(t *testing.T) {
|
|||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
Workdir: ".",
|
Workdir: ".",
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
},
|
},
|
||||||
Env: map[string]string{
|
Env: map[string]string{
|
||||||
"SOMETHING_TRUE": "true",
|
"SOMETHING_TRUE": "true",
|
||||||
@@ -143,8 +144,11 @@ func TestRunContext_EvalBool(t *testing.T) {
|
|||||||
|
|
||||||
type fakeContainer struct {
|
type fakeContainer struct {
|
||||||
container.ExecutionsEnvironment
|
container.ExecutionsEnvironment
|
||||||
|
caseInsensitive bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c fakeContainer) IsEnvironmentCaseInsensitive() bool { return c.caseInsensitive }
|
||||||
|
|
||||||
func (fakeContainer) Pull(bool) common.Executor { return func(context.Context) error { return nil } }
|
func (fakeContainer) Pull(bool) common.Executor { return func(context.Context) error { return nil } }
|
||||||
|
|
||||||
func (fakeContainer) Start(bool) common.Executor { return func(context.Context) error { return nil } }
|
func (fakeContainer) Start(bool) common.Executor { return func(context.Context) error { return nil } }
|
||||||
@@ -197,7 +201,7 @@ func startJobContainerInputs(t *testing.T, workflowYAML string, cfg *Config) []*
|
|||||||
|
|
||||||
cfg.Workdir = "/tmp"
|
cfg.Workdir = "/tmp"
|
||||||
cfg.ContainerNetworkMode = "host"
|
cfg.ContainerNetworkMode = "host"
|
||||||
cfg.Env = map[string]string{}
|
cfg.Env = testCheckoutEnv()
|
||||||
cfg.Secrets = map[string]string{}
|
cfg.Secrets = map[string]string{}
|
||||||
|
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
@@ -281,12 +285,13 @@ jobs:
|
|||||||
image: redis:latest
|
image: redis:latest
|
||||||
volumes:
|
volumes:
|
||||||
- data:/data
|
- data:/data
|
||||||
|
- data:/backup
|
||||||
`, &Config{ValidVolumes: []string{"data"}})[0]
|
`, &Config{ValidVolumes: []string{"data"}})[0]
|
||||||
|
|
||||||
require.Equal(t, "redis:latest", redis.Image)
|
require.Equal(t, "redis:latest", redis.Image)
|
||||||
require.Equal(t, []string{"data"}, redis.ValidVolumes)
|
require.Equal(t, []string{"data"}, redis.ValidVolumes)
|
||||||
require.Equal(t, map[string]string{"data": "/data"}, redis.Mounts)
|
require.Equal(t, map[string]string{"data": "/data"}, redis.Mounts)
|
||||||
require.Empty(t, redis.Binds)
|
require.Equal(t, []string{"data:/backup"}, redis.Binds)
|
||||||
require.Empty(t, redis.WorkingDir)
|
require.Empty(t, redis.WorkingDir)
|
||||||
require.Nil(t, redis.Entrypoint)
|
require.Nil(t, redis.Entrypoint)
|
||||||
require.Nil(t, redis.Cmd)
|
require.Nil(t, redis.Cmd)
|
||||||
@@ -461,6 +466,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
|||||||
{"BindAnonymousVolume", []string{"/volume"}, "/volume", map[string]string{}},
|
{"BindAnonymousVolume", []string{"/volume"}, "/volume", map[string]string{}},
|
||||||
{"BindHostFile", []string{"/path/to/file/on/host:/volume"}, "/path/to/file/on/host:/volume", map[string]string{}},
|
{"BindHostFile", []string{"/path/to/file/on/host:/volume"}, "/path/to/file/on/host:/volume", map[string]string{}},
|
||||||
{"MountExistingVolume", []string{"volume-id:/volume"}, "", map[string]string{"volume-id": "/volume"}},
|
{"MountExistingVolume", []string{"volume-id:/volume"}, "", map[string]string{"volume-id": "/volume"}},
|
||||||
|
{"MountExistingVolumeTwice", []string{"volume-id:/volume", "volume-id:/other"}, "volume-id:/other", map[string]string{"volume-id": "/volume"}},
|
||||||
{"MountExistingVolumeReadOnly", []string{"volume-id:/volume:ro"}, "volume-id:/volume:ro", map[string]string{}},
|
{"MountExistingVolumeReadOnly", []string{"volume-id:/volume:ro"}, "volume-id:/volume:ro", map[string]string{}},
|
||||||
{"BindRelativeHostPath", []string{"./relative:/volume"}, "./relative:/volume", map[string]string{}},
|
{"BindRelativeHostPath", []string{"./relative:/volume"}, "./relative:/volume", map[string]string{}},
|
||||||
{"OverridesToolCache", []string{"/host/tools:/opt/hostedtoolcache"}, "/host/tools:/opt/hostedtoolcache", map[string]string{}},
|
{"OverridesToolCache", []string{"/host/tools:/opt/hostedtoolcache"}, "/host/tools:/opt/hostedtoolcache", map[string]string{}},
|
||||||
@@ -476,6 +482,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
rc := newRunContext(&Config{
|
rc := newRunContext(&Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
ValidVolumes: []string{"**"},
|
ValidVolumes: []string{"**"},
|
||||||
Secrets: map[string]string{
|
Secrets: map[string]string{
|
||||||
"MAME": "/host/mame/roms",
|
"MAME": "/host/mame/roms",
|
||||||
@@ -772,7 +779,7 @@ func TestInterpolateOutputsIsPerMatrixComboKeepingNeedsOutputs(t *testing.T) {
|
|||||||
require.NoError(t, job.RawOutputs.Encode(job.Outputs))
|
require.NoError(t, job.RawOutputs.Encode(job.Outputs))
|
||||||
needed := &model.Job{Outputs: map[string]string{"o": "from-gitea"}}
|
needed := &model.Job{Outputs: map[string]string{"o": "from-gitea"}}
|
||||||
run := &model.Run{JobID: "j", Workflow: &model.Workflow{Name: "w", Jobs: map[string]*model.Job{"j": job, "needed": needed}}}
|
run := &model.Run{JobID: "j", Workflow: &model.Workflow{Name: "w", Jobs: map[string]*model.Job{"j": job, "needed": needed}}}
|
||||||
r := &runnerImpl{config: &Config{}}
|
r := &runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}}
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
_, err := r.newRunContext(ctx, &model.Run{JobID: "needed", Workflow: run.Workflow}, nil)
|
_, err := r.newRunContext(ctx, &model.Run{JobID: "needed", Workflow: run.Workflow}, nil)
|
||||||
@@ -799,7 +806,7 @@ func TestInterpolateOutputsFromExpression(t *testing.T) {
|
|||||||
|
|
||||||
job := &model.Job{RawOutputs: rawOutputs}
|
job := &model.Job{RawOutputs: rawOutputs}
|
||||||
run := &model.Run{JobID: "j", Workflow: &model.Workflow{Name: "w", Jobs: map[string]*model.Job{"j": job}}}
|
run := &model.Run{JobID: "j", Workflow: &model.Workflow{Name: "w", Jobs: map[string]*model.Job{"j": job}}}
|
||||||
rc, err := (&runnerImpl{config: &Config{}}).newRunContext(t.Context(), run, nil)
|
rc, err := (&runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}}).newRunContext(t.Context(), run, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, rc.interpolateOutputs()(t.Context()))
|
require.NoError(t, rc.interpolateOutputs()(t.Context()))
|
||||||
@@ -862,11 +869,37 @@ func TestGetGitHubContext(t *testing.T) {
|
|||||||
assert.Equal(t, rc.Config.Secrets["GITHUB_TOKEN"], ghc.Token)
|
assert.Equal(t, rc.Config.Secrets["GITHUB_TOKEN"], ghc.Token)
|
||||||
assert.Equal(t, "job1", ghc.Job)
|
assert.Equal(t, "job1", ghc.Job)
|
||||||
|
|
||||||
rc.Config.PresetGitHubContext = &model.GithubContext{Actor: "preset-actor", TriggeringActor: "triggerer", Job: "preset-job"}
|
rc.Config.PresetGitHubContext = &model.GithubContext{
|
||||||
|
Actor: "preset-actor",
|
||||||
|
ActorID: "12",
|
||||||
|
TriggeringActor: "triggerer",
|
||||||
|
Job: "preset-job",
|
||||||
|
RepositoryID: "34",
|
||||||
|
RepositoryOwnerID: "56",
|
||||||
|
WorkflowRef: "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||||
|
RefProtected: true,
|
||||||
|
}
|
||||||
ghc = rc.getGithubContext(t.Context())
|
ghc = rc.getGithubContext(t.Context())
|
||||||
assert.Equal(t, "preset-actor", ghc.Actor)
|
assert.Equal(t, "preset-actor", ghc.Actor)
|
||||||
assert.Equal(t, "triggerer", ghc.TriggeringActor)
|
assert.Equal(t, "triggerer", ghc.TriggeringActor)
|
||||||
assert.Equal(t, "job1", ghc.Job)
|
assert.Equal(t, "job1", ghc.Job)
|
||||||
|
env := map[string]string{"GITHUB_WORKFLOW_SHA": "workflow-sha"}
|
||||||
|
rc.withGithubEnv(t.Context(), ghc, env)
|
||||||
|
for suffix, value := range map[string]string{
|
||||||
|
"ACTOR_ID": "12",
|
||||||
|
"TRIGGERING_ACTOR": "triggerer",
|
||||||
|
"REPOSITORY_ID": "34",
|
||||||
|
"REPOSITORY_OWNER_ID": "56",
|
||||||
|
"WORKFLOW_REF": "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||||
|
"WORKFLOW_SHA": "workflow-sha",
|
||||||
|
"REF_PROTECTED": "true",
|
||||||
|
} {
|
||||||
|
assert.Equal(t, value, env["GITHUB_"+suffix], suffix)
|
||||||
|
assert.Equal(t, value, env["GITEA_"+suffix], suffix)
|
||||||
|
}
|
||||||
|
ghc.RefProtected = false
|
||||||
|
rc.withGithubEnv(t.Context(), ghc, env)
|
||||||
|
assert.Equal(t, "false", env["GITHUB_REF_PROTECTED"])
|
||||||
|
|
||||||
preset := &model.GithubContext{ServerURL: "https://preset.test", APIURL: "https://preset.test/api/v1"}
|
preset := &model.GithubContext{ServerURL: "https://preset.test", APIURL: "https://preset.test/api/v1"}
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
@@ -889,12 +922,12 @@ func TestGetGitHubContext(t *testing.T) {
|
|||||||
github := rc.getGithubContext(t.Context())
|
github := rc.getGithubContext(t.Context())
|
||||||
assert.Equal(t, testcase.serverURL, github.ServerURL)
|
assert.Equal(t, testcase.serverURL, github.ServerURL)
|
||||||
assert.Equal(t, testcase.apiURL, github.APIURL)
|
assert.Equal(t, testcase.apiURL, github.APIURL)
|
||||||
env := map[string]string{}
|
env := map[string]string{"GITEA_SERVER_URL": "stale", "GITEA_API_URL": "stale"}
|
||||||
rc.withGithubEnv(t.Context(), github, env)
|
rc.withGithubEnv(t.Context(), github, env)
|
||||||
_, exported := env["GITHUB_SERVER_URL"]
|
_, exported := env["GITHUB_SERVER_URL"]
|
||||||
assert.Equal(t, testcase.serverURL != "", exported)
|
assert.Equal(t, testcase.serverURL != "", exported)
|
||||||
assert.Equal(t, testcase.serverURL, env["GITHUB_SERVER_URL"])
|
assert.Equal(t, testcase.serverURL, env["GITEA_SERVER_URL"])
|
||||||
assert.Equal(t, testcase.apiURL, env["GITHUB_API_URL"])
|
assert.Equal(t, testcase.apiURL, env["GITEA_API_URL"])
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -925,6 +958,7 @@ func TestGetGithubContextRef(t *testing.T) {
|
|||||||
Config: &Config{
|
Config: &Config{
|
||||||
EventName: data.event,
|
EventName: data.event,
|
||||||
Workdir: "",
|
Workdir: "",
|
||||||
|
Env: map[string]string{"GITHUB_REPOSITORY": "owner/repo", "SHA_REF": "1234567890"},
|
||||||
},
|
},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
Workflow: &model.Workflow{
|
Workflow: &model.Workflow{
|
||||||
@@ -942,7 +976,7 @@ func TestGetGithubContextRef(t *testing.T) {
|
|||||||
|
|
||||||
func createIfTestRunContext(jobs map[string]*model.Job) *RunContext {
|
func createIfTestRunContext(jobs map[string]*model.Job) *RunContext {
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{Workdir: ".", PlatformPicker: func([]string) string { return "ubuntu-latest" }},
|
Config: &Config{Workdir: ".", PresetGitHubContext: &model.GithubContext{}, PlatformPicker: func([]string) string { return "ubuntu-latest" }},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "job1",
|
JobID: "job1",
|
||||||
Workflow: &model.Workflow{
|
Workflow: &model.Workflow{
|
||||||
@@ -1092,6 +1126,22 @@ func TestRunContext_cleanupFailedStart(t *testing.T) {
|
|||||||
type ctxKey string
|
type ctxKey string
|
||||||
const sentinel = ctxKey("sentinel")
|
const sentinel = ctxKey("sentinel")
|
||||||
|
|
||||||
|
t.Run("removes host directories when tool cache creation fails", func(t *testing.T) {
|
||||||
|
cacheDir := t.TempDir()
|
||||||
|
toolCache := filepath.Join(t.TempDir(), "file")
|
||||||
|
require.NoError(t, os.WriteFile(toolCache, nil, 0o600))
|
||||||
|
rc := &RunContext{
|
||||||
|
Config: &Config{ActionCacheDir: cacheDir},
|
||||||
|
Env: map[string]string{"RUNNER_TOOL_CACHE": toolCache},
|
||||||
|
platformImage: "-self-hosted",
|
||||||
|
}
|
||||||
|
require.Error(t, rc.startContainer()(t.Context()))
|
||||||
|
entries, err := os.ReadDir(cacheDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, entries)
|
||||||
|
assert.FileExists(t, toolCache)
|
||||||
|
})
|
||||||
|
|
||||||
for name, canceled := range map[string]bool{"cancellation during cleanup": false, "already canceled": true} {
|
for name, canceled := range map[string]bool{"cancellation during cleanup": false, "already canceled": true} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.WithValue(t.Context(), sentinel, "v"))
|
ctx, cancel := context.WithCancel(context.WithValue(t.Context(), sentinel, "v"))
|
||||||
@@ -1163,7 +1213,7 @@ func TestRunContext_cleanupFailedStart(t *testing.T) {
|
|||||||
workflow, err := model.ReadWorkflow(strings.NewReader("jobs: {job: {services: {postgres: {image: postgres:latest}}}}"))
|
workflow, err := model.ReadWorkflow(strings.NewReader("jobs: {job: {services: {postgres: {image: postgres:latest}}}}"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{ForcePull: true, ContainerNetworkMode: "host", Workdir: "/workspace"},
|
Config: &Config{ForcePull: true, ContainerNetworkMode: "host", Workdir: "/workspace", Env: testCheckoutEnv()},
|
||||||
Run: &model.Run{JobID: "job", Workflow: workflow},
|
Run: &model.Run{JobID: "job", Workflow: workflow},
|
||||||
platformImage: "node:20",
|
platformImage: "node:20",
|
||||||
}
|
}
|
||||||
@@ -1515,6 +1565,7 @@ func TestRunContextGetRunnerContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunContextUpdateExtraPath(t *testing.T) {
|
func TestRunContextUpdateExtraPath(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
longPath := strings.Repeat("x", 64*1024+1)
|
longPath := strings.Repeat("x", 64*1024+1)
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -1573,4 +1624,15 @@ func TestRunContextWithGithubEnvRunnerValues(t *testing.T) {
|
|||||||
assert.Equal(t, "/workspace/owner", env["RUNNER_WORKSPACE"])
|
assert.Equal(t, "/workspace/owner", env["RUNNER_WORKSPACE"])
|
||||||
assert.Equal(t, "1", env["RUNNER_DEBUG"])
|
assert.Equal(t, "1", env["RUNNER_DEBUG"])
|
||||||
assert.Equal(t, "http://[2001:db8::1]:8080/", env["ACTIONS_RUNTIME_URL"])
|
assert.Equal(t, "http://[2001:db8::1]:8080/", env["ACTIONS_RUNTIME_URL"])
|
||||||
|
assert.Equal(t, "true", env["CI"])
|
||||||
|
|
||||||
|
for _, ci := range []string{"false", ""} {
|
||||||
|
env["CI"] = ci
|
||||||
|
rc.withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||||
|
assert.Equal(t, ci, env["CI"])
|
||||||
|
}
|
||||||
|
|
||||||
|
rc.runtimeToken = "run-token"
|
||||||
|
newTestCompositeRunContext(t, rc).withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||||
|
assert.Equal(t, "run-token", env["ACTIONS_RUNTIME_TOKEN"])
|
||||||
}
|
}
|
||||||
@@ -41,6 +41,7 @@ type Config struct {
|
|||||||
Env map[string]string // env for containers
|
Env map[string]string // env for containers
|
||||||
Secrets map[string]string // list of secrets
|
Secrets map[string]string // list of secrets
|
||||||
ExtraMasks []string // values to hide that are not in Secrets, such as the proxy password
|
ExtraMasks []string // values to hide that are not in Secrets, such as the proxy password
|
||||||
|
RuntimeTokens map[*model.Workflow]string // ACTIONS_RUNTIME_TOKEN per workflow run, as each is its own artifact run
|
||||||
Vars map[string]string // list of vars
|
Vars map[string]string // list of vars
|
||||||
Token string // GitHub token
|
Token string // GitHub token
|
||||||
InsecureSecrets bool // switch hiding output when printing to terminal
|
InsecureSecrets bool // switch hiding output when printing to terminal
|
||||||
@@ -324,7 +325,7 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
|
|||||||
|
|
||||||
executor := common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan))
|
executor := common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan))
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return executor(container.WithRunnerArch(ghcontext.WithPinnedCheckout(ctx)))
|
return executor(container.WithRunnerArch(ghcontext.WithPinnedCheckout(ctx, &sync.Map{})))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -355,6 +356,11 @@ func (runner *runnerImpl) newCombinationRunContext(ctx context.Context, run *mod
|
|||||||
caller: runner.caller,
|
caller: runner.caller,
|
||||||
jobIndex: index,
|
jobIndex: index,
|
||||||
jobTotal: total,
|
jobTotal: total,
|
||||||
|
|
||||||
|
runtimeToken: runner.config.RuntimeTokens[run.Workflow],
|
||||||
|
}
|
||||||
|
if runner.caller != nil {
|
||||||
|
rc.runtimeToken = runner.caller.runContext.runtimeToken
|
||||||
}
|
}
|
||||||
if err := rc.resolveWorkflowCall(ctx); err != nil {
|
if err := rc.resolveWorkflowCall(ctx); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -13,11 +13,13 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/ghcontext"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
@@ -26,12 +28,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
baseImage = "node:24-bookworm-slim"
|
baseImage = "node:24-bookworm-slim"
|
||||||
platforms map[string]string
|
platforms map[string]string
|
||||||
logLevel = logging.LevelDebug
|
logLevel = logging.LevelDebug
|
||||||
workdir = "testdata"
|
workdir = "testdata"
|
||||||
secrets map[string]string
|
secrets map[string]string
|
||||||
testRunID = strings.ToLower(rand.Text()[:8]) // keeps docker names apart from concurrent test processes on one daemon
|
testRunID = strings.ToLower(rand.Text()[:8]) // keeps docker names apart from concurrent test processes on one daemon
|
||||||
|
testCheckoutPins sync.Map
|
||||||
)
|
)
|
||||||
|
|
||||||
func mapPlatformPicker(platforms map[string]string) func([]string) string {
|
func mapPlatformPicker(platforms map[string]string) func([]string) string {
|
||||||
@@ -158,12 +161,12 @@ func TestNewPlanExecutorRequiresRunsOnBeforeRunningAnyJob(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
plan, err := model.CombineWorkflowPlanner(workflow).PlanJob("test")
|
plan, err := model.CombineWorkflowPlanner(workflow).PlanJob("test")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.ErrorContains(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(plan)(t.Context()), `job "test" requires a non-empty 'runs-on'`, runsOn)
|
require.ErrorContains(t, (&runnerImpl{config: &Config{Env: testCheckoutEnv()}}).NewPlanExecutor(plan)(t.Context()), `job "test" requires a non-empty 'runs-on'`, runsOn)
|
||||||
assert.Empty(t, workflow.Jobs["first"].Result, runsOn)
|
assert.Empty(t, workflow.Jobs["first"].Result, runsOn)
|
||||||
}
|
}
|
||||||
|
|
||||||
workflow := &model.Workflow{Jobs: map[string]*model.Job{"build": {Result: "success"}}}
|
workflow := &model.Workflow{Jobs: map[string]*model.Job{"build": {Result: "success"}}}
|
||||||
require.NoError(t, (&runnerImpl{config: &Config{}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "build"}}}}})(t.Context()))
|
require.NoError(t, (&runnerImpl{config: &Config{Env: testCheckoutEnv()}}).NewPlanExecutor(&model.Plan{Stages: []*model.Stage{{Runs: []*model.Run{{Workflow: workflow, JobID: "build"}}}}})(t.Context()))
|
||||||
assert.Equal(t, "success", workflow.Jobs["build"].Result)
|
assert.Equal(t, "success", workflow.Jobs["build"].Result)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,6 +222,7 @@ type TestJobFileInfo struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config) {
|
func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config) {
|
||||||
|
ctx = ghcontext.WithPinnedCheckout(ctx, &testCheckoutPins)
|
||||||
fmt.Printf("::group::%s\n", j.workflowPath) //nolint:forbidigo // pre-existing issue from nektos/act
|
fmt.Printf("::group::%s\n", j.workflowPath) //nolint:forbidigo // pre-existing issue from nektos/act
|
||||||
|
|
||||||
logging.SetLevel(logLevel)
|
logging.SetLevel(logLevel)
|
||||||
@@ -456,6 +460,7 @@ func TestRunEventHostEnvironment(t *testing.T) {
|
|||||||
{workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets},
|
{workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets},
|
||||||
{workdir, "set-env-step-env-override", "push", "", platforms, secrets},
|
{workdir, "set-env-step-env-override", "push", "", platforms, secrets},
|
||||||
{workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets},
|
{workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets},
|
||||||
|
{workdir, "github-parity", "push", "", platforms, secrets},
|
||||||
{workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets},
|
{workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets},
|
||||||
}...)
|
}...)
|
||||||
}
|
}
|
||||||
@@ -480,7 +485,7 @@ func TestRunEventHostEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDryrunEvent(t *testing.T) {
|
func TestDryrunEvent(t *testing.T) {
|
||||||
t.Parallel()
|
t.Setenv("DOCKER_HOST", "://")
|
||||||
// Dryrun plans without containers or network (shells and local actions only).
|
// Dryrun plans without containers or network (shells and local actions only).
|
||||||
ctx := common.WithDryrun(t.Context(), true)
|
ctx := common.WithDryrun(t.Context(), true)
|
||||||
|
|
||||||
|
|||||||
+51
-43
@@ -27,12 +27,24 @@ type step interface {
|
|||||||
post() common.Executor
|
post() common.Executor
|
||||||
|
|
||||||
getRunContext() *RunContext
|
getRunContext() *RunContext
|
||||||
getGithubContext(ctx context.Context) *model.GithubContext
|
|
||||||
getStepModel() *model.Step
|
getStepModel() *model.Step
|
||||||
getEnv() *map[string]string
|
getEnv() *map[string]string
|
||||||
getIfExpression(context context.Context, stage stepStage) string
|
getIfExpression(context context.Context, stage stepStage) string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stepActionIdentity(step step) (repository, ref string) {
|
||||||
|
if action, ok := step.(interface{ actionIdentity() (string, string) }); ok {
|
||||||
|
return action.actionIdentity()
|
||||||
|
}
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func getStepGithubContext(ctx context.Context, step step) *model.GithubContext {
|
||||||
|
github := step.getRunContext().getGithubContext(ctx)
|
||||||
|
github.ActionRepository, github.ActionRef = stepActionIdentity(step)
|
||||||
|
return github
|
||||||
|
}
|
||||||
|
|
||||||
type stepStage int
|
type stepStage int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -92,6 +104,8 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
|||||||
runStep, err = isStepEnabled(ctx, ifExpression, step, stage)
|
runStep, err = isStepEnabled(ctx, ifExpression, step, stage)
|
||||||
}
|
}
|
||||||
if err == nil && runStep {
|
if err == nil && runStep {
|
||||||
|
env := *step.getEnv()
|
||||||
|
env["GITHUB_ACTION_REPOSITORY"], env["GITHUB_ACTION_REF"] = stepActionIdentity(step)
|
||||||
err = setupInputs(ctx, step)
|
err = setupInputs(ctx, step)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -143,15 +157,7 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
|||||||
stepSummaryIndex := topRC.CurrentStepIndex
|
stepSummaryIndex := topRC.CurrentStepIndex
|
||||||
summaryFileCommand := path.Join("workflow", "step-summary-"+strconv.Itoa(stepSummaryIndex)+".md")
|
summaryFileCommand := path.Join("workflow", "step-summary-"+strconv.Itoa(stepSummaryIndex)+".md")
|
||||||
(*step.getEnv())["GITHUB_STEP_SUMMARY"] = path.Join(actPath, summaryFileCommand)
|
(*step.getEnv())["GITHUB_STEP_SUMMARY"] = path.Join(actPath, summaryFileCommand)
|
||||||
|
aliasGiteaEnv(*step.getEnv())
|
||||||
{
|
|
||||||
// For Gitea
|
|
||||||
(*step.getEnv())["GITEA_OUTPUT"] = (*step.getEnv())["GITHUB_OUTPUT"]
|
|
||||||
(*step.getEnv())["GITEA_STATE"] = (*step.getEnv())["GITHUB_STATE"]
|
|
||||||
(*step.getEnv())["GITEA_PATH"] = (*step.getEnv())["GITHUB_PATH"]
|
|
||||||
(*step.getEnv())["GITEA_ENV"] = (*step.getEnv())["GITHUB_ENV"]
|
|
||||||
(*step.getEnv())["GITEA_STEP_SUMMARY"] = (*step.getEnv())["GITHUB_STEP_SUMMARY"]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset the per-phase file-command files. GITHUB_STEP_SUMMARY is intentionally
|
// Reset the per-phase file-command files. GITHUB_STEP_SUMMARY is intentionally
|
||||||
// excluded here and initialized below at most once per step so writes from later
|
// excluded here and initialized below at most once per step so writes from later
|
||||||
@@ -170,25 +176,16 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
|||||||
initialFiles = append([]*container.FileEntry{{Name: summaryFileCommand, Mode: 0o666}}, files...)
|
initialFiles = append([]*container.FileEntry{{Name: summaryFileCommand, Mode: 0o666}}, files...)
|
||||||
topRC.summaryFileInitialized[stepSummaryIndex] = true
|
topRC.summaryFileInitialized[stepSummaryIndex] = true
|
||||||
}
|
}
|
||||||
if err := rc.JobContainer.Copy(actPath, initialFiles...)(ctx); err != nil && len(initialFiles) > len(files) {
|
fileCommandInitErr := rc.JobContainer.Copy(actPath, initialFiles...)(ctx)
|
||||||
_ = rc.JobContainer.Copy(actPath, files...)(ctx) // the summary goes first and the resets retry alone, so neither failure blocks the other
|
if fileCommandInitErr != nil && len(initialFiles) > len(files) {
|
||||||
|
fileCommandInitErr = rc.JobContainer.Copy(actPath, files...)(ctx) // a broken summary must not block the resets
|
||||||
}
|
}
|
||||||
|
|
||||||
// The command handler needs the step's env to judge ACTIONS_ALLOW_UNSECURE_COMMANDS.
|
timeoutctx, cancelTimeOut := evaluateStepTimeout(ctx, step)
|
||||||
// Cloned: the step executor keeps writing to its own env map after this point, on a
|
|
||||||
// different goroutine from the command handler that reads it.
|
|
||||||
rc.setCurrentStepEnv(maps0.Clone(*step.getEnv()))
|
|
||||||
defer rc.setCurrentStepEnv(nil)
|
|
||||||
_ = rc.takeUnsecureCommandError() // a refusal from before any step belongs to no step
|
|
||||||
|
|
||||||
timeoutctx, cancelTimeOut := evaluateStepTimeout(ctx, rc.ExprEval, stepModel)
|
|
||||||
defer cancelTimeOut()
|
defer cancelTimeOut()
|
||||||
err = executor(timeoutctx)
|
err = fileCommandInitErr
|
||||||
// Always take it, so the job-scoped error cannot leak onto a later step. A refusal
|
|
||||||
// fails the step as it does on GitHub, but the executor's own error wins.
|
|
||||||
insecureErr := rc.takeUnsecureCommandError()
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = insecureErr
|
err = rc.runWithCommands(timeoutctx, *step.getEnv(), executor)
|
||||||
}
|
}
|
||||||
fileCommandCtx := ctx
|
fileCommandCtx := ctx
|
||||||
if ctx.Err() != nil { // an interrupted step still records its file commands, a live deadline stays in force
|
if ctx.Err() != nil { // an interrupted step still records its file commands, a live deadline stays in force
|
||||||
@@ -196,17 +193,19 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
|||||||
fileCommandCtx, cancelFileCommands = context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
fileCommandCtx, cancelFileCommands = context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
||||||
defer cancelFileCommands()
|
defer cancelFileCommands()
|
||||||
}
|
}
|
||||||
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, envFileCommand, rc, rc.setEnvFile); fileErr != nil && err == nil {
|
if fileCommandInitErr == nil {
|
||||||
err = fileErr
|
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, envFileCommand, rc, rc.setEnvFile); fileErr != nil && err == nil {
|
||||||
}
|
err = fileErr
|
||||||
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, stateFileCommand, rc, rc.saveState); fileErr != nil && err == nil {
|
}
|
||||||
err = fileErr
|
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, stateFileCommand, rc, rc.saveState); fileErr != nil && err == nil {
|
||||||
}
|
err = fileErr
|
||||||
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, outputFileCommand, rc, rc.setOutput); fileErr != nil && err == nil {
|
}
|
||||||
err = fileErr
|
if fileErr := processRunnerEnvFileCommand(fileCommandCtx, outputFileCommand, rc, rc.setOutput); fileErr != nil && err == nil {
|
||||||
}
|
err = fileErr
|
||||||
if fileErr := rc.UpdateExtraPath(fileCommandCtx, path.Join(actPath, pathFileCommand)); fileErr != nil && err == nil {
|
}
|
||||||
err = fileErr
|
if fileErr := rc.UpdateExtraPath(fileCommandCtx, path.Join(actPath, pathFileCommand)); fileErr != nil && err == nil {
|
||||||
|
err = fileErr
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_ = rc.JobContainer.Copy(actPath, files...)(fileCommandCtx)
|
_ = rc.JobContainer.Copy(actPath, files...)(fileCommandCtx)
|
||||||
|
|
||||||
@@ -239,8 +238,11 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func evaluateStepTimeout(ctx context.Context, exprEval *expressionEvaluator, stepModel *model.Step) (context.Context, context.CancelFunc) {
|
func evaluateStepTimeout(ctx context.Context, step step) (context.Context, context.CancelFunc) {
|
||||||
timeout, err := exprEval.Interpolate(ctx, stepModel.TimeoutMinutes)
|
if step.getStepModel().TimeoutMinutes == "" {
|
||||||
|
return ctx, func() {}
|
||||||
|
}
|
||||||
|
timeout, err := step.getRunContext().NewStepExpressionEvaluator(ctx, step).Interpolate(ctx, step.getStepModel().TimeoutMinutes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Errorf("An error occurred when attempting to determine the step timeout: %s", err)
|
common.Logger(ctx).Errorf("An error occurred when attempting to determine the step timeout: %s", err)
|
||||||
} else if timeout != "" {
|
} else if timeout != "" {
|
||||||
@@ -257,14 +259,20 @@ func setupEnv(ctx context.Context, step step) error {
|
|||||||
env := step.getEnv()
|
env := step.getEnv()
|
||||||
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
||||||
mergeIntoMap(step, env, rc.containerSpec.Env, rc.GetEnv())
|
mergeIntoMap(step, env, rc.containerSpec.Env, rc.GetEnv())
|
||||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env)
|
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *env)
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
exprEval := rc.NewExpressionEvaluator(ctx)
|
exprEval := rc.NewExpressionEvaluator(ctx)
|
||||||
stepEnv := step.getStepModel().Environment()
|
stepEnv := step.getStepModel().Environment()
|
||||||
for k, v := range stepEnv {
|
if envNode := step.getStepModel().Env; strings.Contains(envNode.Value, "${{") { // only a whole-map expression has a value
|
||||||
if stepEnv[k], err = exprEval.Interpolate(ctx, v); err != nil {
|
if err := model.DecodeEvaluated("step env", envNode, exprEval.shared(ctx).EvaluateYamlNode, &stepEnv); err != nil {
|
||||||
return fmt.Errorf("unable to interpolate env %s: %w", k, err)
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for k, v := range stepEnv {
|
||||||
|
if stepEnv[k], err = exprEval.Interpolate(ctx, v); err != nil {
|
||||||
|
return fmt.Errorf("unable to interpolate env %s: %w", k, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// merge step env last, since it should not be overwritten
|
// merge step env last, since it should not be overwritten
|
||||||
@@ -300,7 +308,7 @@ func isStepEnabled(ctx context.Context, expr string, step step, stage stepStage)
|
|||||||
if rc.Parent != nil && stage == stepStageMain && jobContext.Status != "cancelled" {
|
if rc.Parent != nil && stage == stepStageMain && jobContext.Status != "cancelled" {
|
||||||
jobContext.Status = rc.ownStatus()
|
jobContext.Status = rc.ownStatus()
|
||||||
}
|
}
|
||||||
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext), expr, defaultStatusCheck)
|
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext, rc.getGithubContext(ctx)), expr, defaultStatusCheck)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("if-expression %q evaluation failed: %s", expr, err)
|
return false, fmt.Errorf("if-expression %q evaluation failed: %s", expr, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,15 +33,13 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
printRunActionHeader(ctx, sab.Step, sab.env, sab.RunContext)
|
printRunActionHeader(ctx, sab.Step, nil, sab.env, sab.RunContext)
|
||||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
|
||||||
defer rawLogger.Infof("::endgroup::")
|
|
||||||
|
|
||||||
rc := sab.RunContext
|
rc := sab.RunContext
|
||||||
rc.ApplyExtraPath(ctx, &sab.env)
|
rc.ApplyExtraPath(ctx, &sab.env)
|
||||||
return sab.run(ctx, &action.Context{
|
return sab.run(ctx, &action.Context{
|
||||||
Container: rc.JobContainer,
|
Container: rc.JobContainer,
|
||||||
Github: rc.getGithubContext(ctx),
|
Github: getStepGithubContext(ctx, sab),
|
||||||
Inputs: sab.Step.With,
|
Inputs: sab.Step.With,
|
||||||
HostWorkdir: rc.Config.Workdir,
|
HostWorkdir: rc.Config.Workdir,
|
||||||
Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir),
|
Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir),
|
||||||
@@ -60,10 +58,6 @@ func (sab *stepActionBuiltin) post() common.Executor { return common.NewPipeline
|
|||||||
|
|
||||||
func (sab *stepActionBuiltin) getRunContext() *RunContext { return sab.RunContext }
|
func (sab *stepActionBuiltin) getRunContext() *RunContext { return sab.RunContext }
|
||||||
|
|
||||||
func (sab *stepActionBuiltin) getGithubContext(ctx context.Context) *model.GithubContext {
|
|
||||||
return sab.RunContext.getGithubContext(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sab *stepActionBuiltin) getStepModel() *model.Step { return sab.Step }
|
func (sab *stepActionBuiltin) getStepModel() *model.Step { return sab.Step }
|
||||||
|
|
||||||
func (sab *stepActionBuiltin) getEnv() *map[string]string { return &sab.env }
|
func (sab *stepActionBuiltin) getEnv() *map[string]string { return &sab.env }
|
||||||
|
|||||||
@@ -42,10 +42,6 @@ func (sal *stepActionLocal) main() common.Executor {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
printRunActionHeader(ctx, sal.Step, sal.env, sal.getRunContext())
|
|
||||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
|
||||||
defer rawLogger.Infof("::endgroup::")
|
|
||||||
|
|
||||||
actionDir := filepath.Join(sal.getRunContext().Config.Workdir, sal.Step.Uses)
|
actionDir := filepath.Join(sal.getRunContext().Config.Workdir, sal.Step.Uses)
|
||||||
_, containerActionPath := getContainerActionPaths(sal)
|
_, containerActionPath := getContainerActionPaths(sal)
|
||||||
|
|
||||||
@@ -99,10 +95,6 @@ func (sal *stepActionLocal) getRunContext() *RunContext {
|
|||||||
return sal.RunContext
|
return sal.RunContext
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sal *stepActionLocal) getGithubContext(ctx context.Context) *model.GithubContext {
|
|
||||||
return sal.getRunContext().getGithubContext(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sal *stepActionLocal) getStepModel() *model.Step {
|
func (sal *stepActionLocal) getStepModel() *model.Step {
|
||||||
return sal.Step
|
return sal.Step
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/mock"
|
"github.com/stretchr/testify/mock"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
type stepActionMocks struct {
|
type stepActionMocks struct {
|
||||||
@@ -53,10 +54,11 @@ func TestStepActionLocalTest(t *testing.T) {
|
|||||||
readAction: salm.readAction,
|
readAction: salm.readAction,
|
||||||
runAction: salm.runAction,
|
runAction: salm.runAction,
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{"previous": {Outputs: map[string]string{"value": "previous"}}},
|
||||||
ExprEval: &expressionEvaluator{},
|
ExprEval: &expressionEvaluator{},
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
Workdir: "/tmp",
|
Workdir: "/tmp",
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
},
|
},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
@@ -67,25 +69,42 @@ func TestStepActionLocalTest(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
|
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/action", "GITHUB_ACTION_REF": "v1"},
|
||||||
|
Matrix: map[string]any{"value": "matrix"},
|
||||||
|
actionInputs: map[string]any{"shared": "input"},
|
||||||
},
|
},
|
||||||
Step: &model.Step{
|
Step: &model.Step{
|
||||||
ID: "1",
|
ID: "1",
|
||||||
Uses: "./path/to/action",
|
Uses: "./path/to/action",
|
||||||
|
If: yaml.Node{Value: "github.action_repository == 'other/action' && github.action_ref == 'v1' && " +
|
||||||
|
"env.STEP_VALUE == 'step' && env.GITHUB_ACTION_REPOSITORY == 'forged/repo' && env.GITHUB_ACTION_REF == 'forged-ref' && " +
|
||||||
|
"steps.previous.outputs.value == 'previous' && inputs.shared == 'input' && matrix.value == 'matrix'"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
require.NoError(t, sal.Step.Env.Encode(map[string]string{"STEP_VALUE": "step", "GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref"}))
|
||||||
|
require.NoError(t, sal.Step.RawWith.Encode(map[string]string{"identity": "${{ github.action_repository }}|${{ github.action_ref }}"}))
|
||||||
|
|
||||||
salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything).
|
salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything).
|
||||||
Return(&model.Action{}, nil)
|
Return(&model.Action{}, nil)
|
||||||
|
|
||||||
expectStepFileCommands(t, cm)
|
expectStepFileCommands(t, cm)
|
||||||
|
|
||||||
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(noopExecutor)
|
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error {
|
||||||
|
sal.RunContext.setOutput(ctx, map[string]string{"name": "value"}, "ran")
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
err := sal.pre()(ctx)
|
err := sal.pre()(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = sal.main()(ctx)
|
err = sal.main()(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, sal.env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Empty(t, sal.env["GITHUB_ACTION_REF"])
|
||||||
|
assert.Equal(t, "|", sal.Step.With["identity"])
|
||||||
|
assert.Equal(t, "ran", sal.RunContext.StepResults["1"].Outputs["value"])
|
||||||
|
assert.Equal(t, "other/action", sal.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Equal(t, "v1", sal.RunContext.Env["GITHUB_ACTION_REF"])
|
||||||
|
|
||||||
cm.AssertExpectations(t)
|
cm.AssertExpectations(t)
|
||||||
salm.AssertExpectations(t)
|
salm.AssertExpectations(t)
|
||||||
@@ -109,7 +128,7 @@ func TestStepActionPost(t *testing.T) {
|
|||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{GitHubInstance: "https://github.com", ActionCacheDir: t.TempDir()},
|
Config: &Config{GitHubInstance: "https://github.com", ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
||||||
StepResults: map[string]*model.StepResult{"step": {Conclusion: test.status}},
|
StepResults: map[string]*model.StepResult{"step": {Conclusion: test.status}},
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ type stepActionRemote struct {
|
|||||||
remoteAction *remoteAction
|
remoteAction *remoteAction
|
||||||
actionDir string // the repository at the resolved commit, shared by the job's actions in it
|
actionDir string // the repository at the resolved commit, shared by the job's actions in it
|
||||||
resolvedSha string // set on the first step preparing the repository
|
resolvedSha string // set on the first step preparing the repository
|
||||||
|
preStepStarted bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type actionDownload struct {
|
type actionDownload struct {
|
||||||
@@ -74,7 +75,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
|||||||
}
|
}
|
||||||
sar.Step.Uses = uses
|
sar.Step.Uses = uses
|
||||||
|
|
||||||
github := sar.getGithubContext(ctx) // read before remoteAction is set, so `$/` resolves against the enclosing action
|
github := sar.RunContext.getGithubContext(ctx)
|
||||||
if sar.remoteAction, err = newRemoteAction(sar.Step.Uses, github, sar.RunContext.Config.GitHubInstance); err != nil {
|
if sar.remoteAction, err = newRemoteAction(sar.Step.Uses, github, sar.RunContext.Config.GitHubInstance); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -142,19 +143,19 @@ func (sar *stepActionRemote) pre() common.Executor {
|
|||||||
|
|
||||||
return common.NewPipelineExecutor(
|
return common.NewPipelineExecutor(
|
||||||
sar.prepareActionExecutor(),
|
sar.prepareActionExecutor(),
|
||||||
runStepExecutor(sar, stepStagePre, runPreStep(sar)).If(hasPreStep(sar)).If(shouldRunPreStep(sar)))
|
runStepExecutor(sar, stepStagePre, func(ctx context.Context) error {
|
||||||
|
sar.preStepStarted = true
|
||||||
|
return runPreStep(sar)(ctx)
|
||||||
|
}).If(hasPreStep(sar)).If(shouldRunPreStep(sar)))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sar *stepActionRemote) main() common.Executor {
|
func (sar *stepActionRemote) main() common.Executor {
|
||||||
return common.NewPipelineExecutor(
|
return common.NewPipelineExecutor(
|
||||||
sar.prepareActionExecutor(),
|
sar.prepareActionExecutor(),
|
||||||
runStepExecutor(sar, stepStageMain, func(ctx context.Context) error {
|
runStepExecutor(sar, stepStageMain, func(ctx context.Context) error {
|
||||||
printRunActionHeader(ctx, sar.Step, sar.env, sar.RunContext)
|
github := getStepGithubContext(ctx, sar)
|
||||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
|
||||||
defer rawLogger.Infof("::endgroup::")
|
|
||||||
|
|
||||||
github := sar.getGithubContext(ctx)
|
|
||||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||||
|
printRunActionHeader(ctx, sar.Step, nil, sar.env, sar.RunContext)
|
||||||
if sar.RunContext.Config.BindWorkdir {
|
if sar.RunContext.Config.BindWorkdir {
|
||||||
common.Logger(ctx).Debugf("Skipping local actions/checkout because you bound your workspace")
|
common.Logger(ctx).Debugf("Skipping local actions/checkout because you bound your workspace")
|
||||||
return nil
|
return nil
|
||||||
@@ -180,17 +181,11 @@ func (sar *stepActionRemote) getRunContext() *RunContext {
|
|||||||
return sar.RunContext
|
return sar.RunContext
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sar *stepActionRemote) getGithubContext(ctx context.Context) *model.GithubContext {
|
func (sar *stepActionRemote) actionIdentity() (string, string) {
|
||||||
ghc := sar.getRunContext().getGithubContext(ctx)
|
if sar.remoteAction == nil {
|
||||||
|
return "", ""
|
||||||
// extend github context if we already have an initialized remoteAction
|
|
||||||
remoteAction := sar.remoteAction
|
|
||||||
if remoteAction != nil {
|
|
||||||
ghc.ActionRepository = fmt.Sprintf("%s/%s", remoteAction.Org, remoteAction.Repo)
|
|
||||||
ghc.ActionRef = remoteAction.Ref
|
|
||||||
}
|
}
|
||||||
|
return sar.remoteAction.Org + "/" + sar.remoteAction.Repo, sar.remoteAction.Ref
|
||||||
return ghc
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sar *stepActionRemote) getStepModel() *model.Step {
|
func (sar *stepActionRemote) getStepModel() *model.Step {
|
||||||
@@ -204,7 +199,7 @@ func (sar *stepActionRemote) getEnv() *map[string]string {
|
|||||||
func (sar *stepActionRemote) getIfExpression(ctx context.Context, stage stepStage) string {
|
func (sar *stepActionRemote) getIfExpression(ctx context.Context, stage stepStage) string {
|
||||||
switch stage {
|
switch stage {
|
||||||
case stepStagePre:
|
case stepStagePre:
|
||||||
github := sar.getGithubContext(ctx)
|
github := getStepGithubContext(ctx, sar)
|
||||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||||
// skip local checkout pre step
|
// skip local checkout pre step
|
||||||
return "false"
|
return "false"
|
||||||
|
|||||||
@@ -51,10 +51,12 @@ func TestShortSHAActionRejected(t *testing.T) {
|
|||||||
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "push.yml"), []byte(workflow), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "push.yml"), []byte(workflow), 0o644))
|
||||||
|
|
||||||
runner, err := New(&Config{
|
runner, err := New(&Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
Workdir: workflowDir,
|
Workdir: workflowDir,
|
||||||
EventName: "push",
|
EventName: "push",
|
||||||
GitHubInstance: "github.com",
|
GitHubInstance: "github.com",
|
||||||
DefaultActionInstance: actionRoot,
|
DefaultActionInstance: actionRoot,
|
||||||
|
ActionCacheDir: t.TempDir(),
|
||||||
ContainerMaxLifetime: time.Hour,
|
ContainerMaxLifetime: time.Hour,
|
||||||
PlatformPicker: func([]string) string { return baseImage },
|
PlatformPicker: func([]string) string { return baseImage },
|
||||||
})
|
})
|
||||||
@@ -74,7 +76,7 @@ func TestStepActionRemote(t *testing.T) {
|
|||||||
status model.StepStatus
|
status model.StepStatus
|
||||||
runError error
|
runError error
|
||||||
}{
|
}{
|
||||||
{name: "run-successful", status: model.StepStatusSuccess},
|
{name: "run-successful", condition: "github.action_repository == 'other/composite' && github.action_ref == 'v2'", status: model.StepStatusSuccess},
|
||||||
{name: "run-skipped", condition: "false", status: model.StepStatusSkipped},
|
{name: "run-skipped", condition: "false", status: model.StepStatusSkipped},
|
||||||
{name: "run-error", status: model.StepStatusFailure, runError: assert.AnError},
|
{name: "run-error", status: model.StepStatusFailure, runError: assert.AnError},
|
||||||
} {
|
} {
|
||||||
@@ -94,8 +96,9 @@ func TestStepActionRemote(t *testing.T) {
|
|||||||
sar := &stepActionRemote{
|
sar := &stepActionRemote{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
GitHubInstance: "github.com",
|
GitHubInstance: "github.com",
|
||||||
ActionCacheDir: t.TempDir(),
|
ActionCacheDir: t.TempDir(),
|
||||||
|
PresetGitHubContext: &model.GithubContext{},
|
||||||
},
|
},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
@@ -107,6 +110,7 @@ func TestStepActionRemote(t *testing.T) {
|
|||||||
},
|
},
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
|
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v2"},
|
||||||
},
|
},
|
||||||
Step: &model.Step{ID: "step", Uses: "remote/action@v1", If: yaml.Node{Value: tt.condition}},
|
Step: &model.Step{ID: "step", Uses: "remote/action@v1", If: yaml.Node{Value: tt.condition}},
|
||||||
readAction: sarm.readAction,
|
readAction: sarm.readAction,
|
||||||
@@ -124,6 +128,8 @@ func TestStepActionRemote(t *testing.T) {
|
|||||||
require.NoError(t, sar.pre()(ctx))
|
require.NoError(t, sar.pre()(ctx))
|
||||||
assert.Equal(t, tt.runError, sar.main()(ctx))
|
assert.Equal(t, tt.runError, sar.main()(ctx))
|
||||||
assert.True(t, clonedAction)
|
assert.True(t, clonedAction)
|
||||||
|
assert.Equal(t, "remote/action", sar.env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Equal(t, "v1", sar.env["GITHUB_ACTION_REF"])
|
||||||
assert.Equal(t, &model.StepResult{Conclusion: tt.status, Outcome: tt.status, Outputs: map[string]string{}}, sar.RunContext.StepResults["step"])
|
assert.Equal(t, &model.StepResult{Conclusion: tt.status, Outcome: tt.status, Outputs: map[string]string{}}, sar.RunContext.StepResults["step"])
|
||||||
|
|
||||||
sarm.AssertExpectations(t)
|
sarm.AssertExpectations(t)
|
||||||
@@ -134,7 +140,7 @@ func TestStepActionRemote(t *testing.T) {
|
|||||||
t.Run("refreshes composite inputs without leaking environment", func(t *testing.T) {
|
t.Run("refreshes composite inputs without leaking environment", func(t *testing.T) {
|
||||||
step := &stepActionRemote{
|
step := &stepActionRemote{
|
||||||
Step: &model.Step{Uses: "org/composite@v1", With: map[string]string{"SHARED": "first"}},
|
Step: &model.Step{Uses: "org/composite@v1", With: map[string]string{"SHARED": "first"}},
|
||||||
RunContext: &RunContext{Config: &Config{ActionCacheDir: t.TempDir()}, Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}}, JobContainer: &jobContainerMock{}},
|
RunContext: &RunContext{Config: &Config{ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}}, Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}}, JobContainer: &jobContainerMock{}},
|
||||||
action: &model.Action{Inputs: map[string]model.Input{"shared": {}}},
|
action: &model.Action{Inputs: map[string]model.Input{"shared": {}}},
|
||||||
env: map[string]string{},
|
env: map[string]string{},
|
||||||
remoteAction: mustNewRemoteAction(t, "org/composite@v1"),
|
remoteAction: mustNewRemoteAction(t, "org/composite@v1"),
|
||||||
@@ -161,7 +167,7 @@ func TestStepActionRemotePrepare(t *testing.T) {
|
|||||||
})
|
})
|
||||||
newJob := func(instance string) *RunContext {
|
newJob := func(instance string) *RunContext {
|
||||||
return &RunContext{
|
return &RunContext{
|
||||||
Config: &Config{GitHubInstance: instance, ActionCacheDir: t.TempDir()},
|
Config: &Config{GitHubInstance: instance, ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
||||||
JobContainer: &jobContainerMock{},
|
JobContainer: &jobContainerMock{},
|
||||||
}
|
}
|
||||||
@@ -179,7 +185,7 @@ func TestStepActionRemotePrepare(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
root, err := prepare(job, "org/repo@v1")
|
root, err := prepare(job, "org/repo@v1")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job}, "org/repo/save@v1")
|
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job, Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "org/repo", "GITHUB_ACTION_REF": "v1"}}, "$/save")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
other, err := prepare(job, "org/repo@v2")
|
other, err := prepare(job, "org/repo@v2")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -207,6 +213,60 @@ func TestStepActionRemotePrepare(t *testing.T) {
|
|||||||
assert.Equal(t, "restore", path.Base(restoreContainerDir))
|
assert.Equal(t, "restore", path.Base(restoreContainerDir))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStepActionRemotePreCleanup(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
preIf string
|
||||||
|
preError error
|
||||||
|
}{
|
||||||
|
{name: "pre ran and main skipped", preIf: "always()"},
|
||||||
|
{name: "pre failed before main", preIf: "always()", preError: assert.AnError},
|
||||||
|
{name: "pre skipped and main skipped", preIf: "false"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cm := &containerMock{}
|
||||||
|
sar := &stepActionRemote{
|
||||||
|
Step: &model.Step{ID: "step", Uses: "remote/action@v1", If: yaml.Node{Value: "false"}},
|
||||||
|
RunContext: &RunContext{
|
||||||
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
|
StepResults: map[string]*model.StepResult{},
|
||||||
|
JobContainer: cm,
|
||||||
|
},
|
||||||
|
action: &model.Action{Runs: model.ActionRuns{
|
||||||
|
Using: "node24", Pre: "pre.js", PreIf: test.preIf, Post: "post.js", PostIf: "always()",
|
||||||
|
}},
|
||||||
|
remoteAction: mustNewRemoteAction(t, "remote/action@v1"),
|
||||||
|
actionDir: t.TempDir(),
|
||||||
|
}
|
||||||
|
sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(t.Context())
|
||||||
|
if test.preIf != "false" {
|
||||||
|
_, actionDir := getContainerActionPaths(sar)
|
||||||
|
expectStepFileCommands(t, cm)
|
||||||
|
cm.On("CopyDir", actionDir+"/", sar.actionDir+"/", false, true).Return(noopExecutor)
|
||||||
|
cm.On("Exec", nodeActionCommand(path.Join(actionDir, "pre.js")), mock.Anything, "", "").Return(func(_ctx context.Context) error { return test.preError }).Once()
|
||||||
|
cm.On("Exec", nodeActionCommand(path.Join(actionDir, "post.js")), mock.Anything, "", "").Return(noopExecutor).Once()
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, test.preError, sar.pre()(t.Context()))
|
||||||
|
if test.preError == nil {
|
||||||
|
require.NoError(t, sar.main()(t.Context()))
|
||||||
|
assert.Equal(t, model.StepStatusSkipped, sar.RunContext.StepResults["step"].Conclusion)
|
||||||
|
}
|
||||||
|
require.NoError(t, sar.post()(t.Context()))
|
||||||
|
cm.AssertExpectations(t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
skippedComposite := &stepActionRemote{
|
||||||
|
Step: &model.Step{ID: "step"},
|
||||||
|
RunContext: &RunContext{StepResults: map[string]*model.StepResult{"step": {Conclusion: model.StepStatusSkipped}}},
|
||||||
|
action: &model.Action{Runs: model.ActionRuns{Using: "composite"}},
|
||||||
|
preStepStarted: true,
|
||||||
|
}
|
||||||
|
assert.False(t, shouldRunPostStep(skippedComposite)(t.Context()))
|
||||||
|
}
|
||||||
|
|
||||||
func Test_newRemoteAction(t *testing.T) {
|
func Test_newRemoteAction(t *testing.T) {
|
||||||
check := func(action string, want *remoteAction, wantCloneURL string) {
|
check := func(action string, want *remoteAction, wantCloneURL string) {
|
||||||
t.Run(action, func(t *testing.T) {
|
t.Run(action, func(t *testing.T) {
|
||||||
@@ -289,9 +349,6 @@ func Test_newRemoteActionSelfRepo(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{name: "unknown instance", action: "$/.gitea/actions/build", github: &model.GithubContext{Repository: "owner/workflow-repo", Sha: "abc123"}},
|
{name: "unknown instance", action: "$/.gitea/actions/build", github: &model.GithubContext{Repository: "owner/workflow-repo", Sha: "abc123"}},
|
||||||
{name: "empty path", action: "$/", github: workflow},
|
|
||||||
{name: "ref suffix is not allowed", action: "$/.gitea/actions/build@v1", github: workflow},
|
|
||||||
{name: "path traversal", action: "$/../escape", github: workflow},
|
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -351,6 +408,7 @@ func TestStepActionRemotePreResolvesDownloadedCommit(t *testing.T) {
|
|||||||
Step: &model.Step{Uses: "actions/setup-go@main"},
|
Step: &model.Step{Uses: "actions/setup-go@main"},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
GitHubInstance: "https://gitea.example.com",
|
GitHubInstance: "https://gitea.example.com",
|
||||||
DefaultActionInstance: instance,
|
DefaultActionInstance: instance,
|
||||||
ActionCacheDir: t.TempDir(),
|
ActionCacheDir: t.TempDir(),
|
||||||
@@ -430,6 +488,7 @@ func TestStepActionRemoteCloneTokenSurvivesNilSecrets(t *testing.T) {
|
|||||||
Step: &model.Step{Uses: "org/repo@v1"},
|
Step: &model.Step{Uses: "org/repo@v1"},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
Env: testCheckoutEnv(),
|
||||||
GitHubInstance: tt.gitHubInstance,
|
GitHubInstance: tt.gitHubInstance,
|
||||||
DefaultActionInstance: tt.defaultActionInstance,
|
DefaultActionInstance: tt.defaultActionInstance,
|
||||||
ActionCacheDir: t.TempDir(),
|
ActionCacheDir: t.TempDir(),
|
||||||
|
|||||||
@@ -35,10 +35,6 @@ func (sd *stepDocker) getRunContext() *RunContext {
|
|||||||
return sd.RunContext
|
return sd.RunContext
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sd *stepDocker) getGithubContext(ctx context.Context) *model.GithubContext {
|
|
||||||
return sd.getRunContext().getGithubContext(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sd *stepDocker) getStepModel() *model.Step {
|
func (sd *stepDocker) getStepModel() *model.Step {
|
||||||
return sd.Step
|
return sd.Step
|
||||||
}
|
}
|
||||||
@@ -60,13 +56,13 @@ func (sd *stepDocker) runUsesContainer() common.Executor {
|
|||||||
return errKubernetesDocker
|
return errKubernetesDocker
|
||||||
}
|
}
|
||||||
image := strings.TrimPrefix(step.Uses, "docker://")
|
image := strings.TrimPrefix(step.Uses, "docker://")
|
||||||
cmd, err := shellquote.Split(step.With["args"])
|
cmd, err := shellquote.Split(stepInput(step, "args"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var entrypoint []string
|
var entrypoint []string
|
||||||
if entry := step.With["entrypoint"]; entry != "" {
|
if entry := stepInput(step, "entrypoint"); entry != "" {
|
||||||
entrypoint = []string{entry}
|
entrypoint = []string{entry}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStepDockerMain(t *testing.T) {
|
func TestStepDockerMain(t *testing.T) {
|
||||||
@@ -36,6 +37,7 @@ func TestStepDockerMain(t *testing.T) {
|
|||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
|
PresetGitHubContext: &model.GithubContext{},
|
||||||
Secrets: map[string]string{
|
Secrets: map[string]string{
|
||||||
"DOCKER_USERNAME": "docker-user",
|
"DOCKER_USERNAME": "docker-user",
|
||||||
"DOCKER_PASSWORD": "docker-password",
|
"DOCKER_PASSWORD": "docker-password",
|
||||||
@@ -56,13 +58,16 @@ func TestStepDockerMain(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
|
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||||
},
|
},
|
||||||
Step: &model.Step{
|
Step: &model.Step{
|
||||||
ID: "1",
|
ID: "1",
|
||||||
Uses: "docker://node:14",
|
Uses: "docker://node:14",
|
||||||
WorkingDirectory: "workdir",
|
WorkingDirectory: "workdir",
|
||||||
|
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
require.NoError(t, sd.Step.RawWith.Encode(map[string]string{"args": "identity:${{ github.action_repository }}:${{ github.action_ref }}"}))
|
||||||
sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx)
|
sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx)
|
||||||
|
|
||||||
cm.On("Pull", false).Return(noopExecutor)
|
cm.On("Pull", false).Return(noopExecutor)
|
||||||
@@ -79,6 +84,11 @@ func TestStepDockerMain(t *testing.T) {
|
|||||||
require.NoError(t, sd.main()(ctx))
|
require.NoError(t, sd.main()(ctx))
|
||||||
|
|
||||||
assert.Equal(t, "node:14", input.Image)
|
assert.Equal(t, "node:14", input.Image)
|
||||||
|
assert.Equal(t, []string{"identity::"}, input.Cmd)
|
||||||
|
assert.Contains(t, input.Env, "GITHUB_ACTION_REPOSITORY=")
|
||||||
|
assert.Contains(t, input.Env, "GITHUB_ACTION_REF=")
|
||||||
|
assert.Equal(t, "other/composite", sd.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Equal(t, "v1", sd.RunContext.Env["GITHUB_ACTION_REF"])
|
||||||
|
|
||||||
assert.Empty(t, input.Username)
|
assert.Empty(t, input.Username)
|
||||||
assert.Empty(t, input.Password)
|
assert.Empty(t, input.Password)
|
||||||
@@ -116,7 +126,8 @@ func TestStepDockerNewStepContainer(t *testing.T) {
|
|||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
AllocatePTY: test.allocPTY,
|
PresetGitHubContext: &model.GithubContext{},
|
||||||
|
AllocatePTY: test.allocPTY,
|
||||||
PlatformPicker: func(_platforms []string) string {
|
PlatformPicker: func(_platforms []string) string {
|
||||||
return test.platform
|
return test.platform
|
||||||
},
|
},
|
||||||
|
|||||||
+20
-11
@@ -87,24 +87,34 @@ func (sr *stepRun) printRunScriptActionDetails(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// printRunActionHeader mirrors actions/runner's "Run <action>" header for `uses:` steps,
|
// printRunActionHeader mirrors actions/runner's "Run <action>" header for `uses:` steps,
|
||||||
// including the with: inputs and the step-level env: block. The caller is responsible
|
// including the with: inputs and the step-level env: block.
|
||||||
// for emitting ::endgroup:: after the action finishes.
|
func printRunActionHeader(ctx context.Context, step *model.Step, action *model.Action, env map[string]string, rc *RunContext) {
|
||||||
func printRunActionHeader(ctx context.Context, step *model.Step, env map[string]string, rc *RunContext) {
|
|
||||||
if step == nil {
|
if step == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
||||||
|
|
||||||
rawLogger.Infof("::group::Run %s", EscapeCommandData(cmp.Or(step.Name, step.Uses)))
|
rawLogger.Infof("::group::Run %s", EscapeCommandData(step.Uses))
|
||||||
|
|
||||||
if len(step.With) > 0 {
|
inputs := map[string]string{}
|
||||||
|
maps.Copy(inputs, step.With)
|
||||||
|
if action != nil {
|
||||||
|
for name := range action.Inputs {
|
||||||
|
if _, provided := lookupFold(step.With, name, true); !provided {
|
||||||
|
inputs[name] = env[inputEnvKey(name)]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
maps.DeleteFunc(inputs, func(_name, value string) bool { return value == "" })
|
||||||
|
if len(inputs) > 0 {
|
||||||
rawLogger.Infof("with:")
|
rawLogger.Infof("with:")
|
||||||
for _, k := range slices.Sorted(maps.Keys(step.With)) {
|
for _, k := range slices.Sorted(maps.Keys(inputs)) {
|
||||||
logKeyedValue(rawLogger, k, step.With[k])
|
logKeyedValue(rawLogger, k, inputs[k])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
printStepEnvBlock(ctx, step, env, rc)
|
printStepEnvBlock(ctx, step, env, rc)
|
||||||
|
rawLogger.Infof("::endgroup::")
|
||||||
}
|
}
|
||||||
|
|
||||||
// printStepEnvBlock emits the declared-env block (YAML order, internal vars filtered)
|
// printStepEnvBlock emits the declared-env block (YAML order, internal vars filtered)
|
||||||
@@ -212,10 +222,6 @@ func (sr *stepRun) getRunContext() *RunContext {
|
|||||||
return sr.RunContext
|
return sr.RunContext
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sr *stepRun) getGithubContext(ctx context.Context) *model.GithubContext {
|
|
||||||
return sr.getRunContext().getGithubContext(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sr *stepRun) getStepModel() *model.Step {
|
func (sr *stepRun) getStepModel() *model.Step {
|
||||||
return sr.Step
|
return sr.Step
|
||||||
}
|
}
|
||||||
@@ -259,6 +265,9 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
scCmd := step.ShellCommand()
|
scCmd := step.ShellCommand()
|
||||||
|
if step.Shell == "pwsh" || step.Shell == "powershell" {
|
||||||
|
scCmd = step.Shell + ` -command ". '{0}'"`
|
||||||
|
}
|
||||||
if implicitShell && (step.Shell == "bash" || step.Shell == "sh") {
|
if implicitShell && (step.Shell == "bash" || step.Shell == "sh") {
|
||||||
scCmd = step.Shell + " -e {0}"
|
scCmd = step.Shell + " -e {0}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ env:
|
|||||||
sr := &stepRun{
|
sr := &stepRun{
|
||||||
Step: &step,
|
Step: &step,
|
||||||
RunContext: &RunContext{},
|
RunContext: &RunContext{},
|
||||||
shellCommand: "pwsh -command . '{0}'",
|
shellCommand: `pwsh -command ". '{0}'"`,
|
||||||
interpolatedScript: "echo one\necho two\n",
|
interpolatedScript: "echo one\necho two\n",
|
||||||
env: map[string]string{
|
env: map[string]string{
|
||||||
"PATH_PREFIX": "/custom/bin",
|
"PATH_PREFIX": "/custom/bin",
|
||||||
@@ -106,7 +106,7 @@ env:
|
|||||||
"[j1] | ::group::Run echo one",
|
"[j1] | ::group::Run echo one",
|
||||||
"[j1] | echo one",
|
"[j1] | echo one",
|
||||||
"[j1] | echo two",
|
"[j1] | echo two",
|
||||||
"[j1] | shell: pwsh -command . '{0}'",
|
`[j1] | shell: pwsh -command ". '{0}'"`,
|
||||||
"[j1] | env:",
|
"[j1] | env:",
|
||||||
"[j1] | PATH_PREFIX: /custom/bin",
|
"[j1] | PATH_PREFIX: /custom/bin",
|
||||||
"[j1] | GREETING: hello",
|
"[j1] | GREETING: hello",
|
||||||
@@ -118,10 +118,12 @@ env:
|
|||||||
|
|
||||||
func TestPrintRunActionHeaderGolden(t *testing.T) {
|
func TestPrintRunActionHeaderGolden(t *testing.T) {
|
||||||
raw := `id: s1
|
raw := `id: s1
|
||||||
|
name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: "0"
|
fetch-depth: "0"
|
||||||
token: secret
|
Token: secret
|
||||||
|
clean: ""
|
||||||
env:
|
env:
|
||||||
CUSTOM: value
|
CUSTOM: value
|
||||||
GITHUB_TOKEN: tok
|
GITHUB_TOKEN: tok
|
||||||
@@ -134,15 +136,19 @@ env:
|
|||||||
entry := logger.With("job", "j1")
|
entry := logger.With("job", "j1")
|
||||||
ctx := common.WithLogger(t.Context(), entry)
|
ctx := common.WithLogger(t.Context(), entry)
|
||||||
|
|
||||||
printRunActionHeader(ctx, &step, map[string]string{"CUSTOM": "value", "GITHUB_TOKEN": "tok"}, &RunContext{})
|
action := &model.Action{Inputs: map[string]model.Input{"token": {}, "clean": {}, "path": {}, "lfs": {}}}
|
||||||
|
env := map[string]string{"CUSTOM": "value", "GITHUB_TOKEN": "tok", "INPUT_TOKEN": "secret", "INPUT_CLEAN": "", "INPUT_PATH": "sub"}
|
||||||
|
printRunActionHeader(ctx, &step, action, env, &RunContext{})
|
||||||
|
|
||||||
want := strings.Join([]string{
|
want := strings.Join([]string{
|
||||||
"[j1] | ::group::Run actions/checkout@v4",
|
"[j1] | ::group::Run actions/checkout@v4",
|
||||||
"[j1] | with:",
|
"[j1] | with:",
|
||||||
|
"[j1] | Token: secret",
|
||||||
"[j1] | fetch-depth: 0",
|
"[j1] | fetch-depth: 0",
|
||||||
"[j1] | token: secret",
|
"[j1] | path: sub",
|
||||||
"[j1] | env:",
|
"[j1] | env:",
|
||||||
"[j1] | CUSTOM: value",
|
"[j1] | CUSTOM: value",
|
||||||
|
"[j1] | ::endgroup::",
|
||||||
"",
|
"",
|
||||||
}, "\n")
|
}, "\n")
|
||||||
assert.Equal(t, want, buf.String())
|
assert.Equal(t, want, buf.String())
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/mock"
|
"github.com/stretchr/testify/mock"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
type shellContainerMock struct{ *containerMock }
|
type shellContainerMock struct{ *containerMock }
|
||||||
@@ -36,8 +37,7 @@ func TestStepRun(t *testing.T) {
|
|||||||
sr := &stepRun{
|
sr := &stepRun{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
ExprEval: &expressionEvaluator{},
|
Config: &Config{Env: testCheckoutEnv()},
|
||||||
Config: &Config{},
|
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
Workflow: &model.Workflow{
|
Workflow: &model.Workflow{
|
||||||
@@ -48,19 +48,33 @@ func TestStepRun(t *testing.T) {
|
|||||||
},
|
},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
jobRunDefaults: model.RunDefaults{Shell: "bash"},
|
jobRunDefaults: model.RunDefaults{Shell: "bash"},
|
||||||
|
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||||
},
|
},
|
||||||
Step: &model.Step{
|
Step: &model.Step{
|
||||||
ID: "1",
|
ID: "1",
|
||||||
Run: "cmd",
|
Run: "cmd${{ github.action_repository }}${{ github.action_ref }}",
|
||||||
WorkingDirectory: "workdir",
|
WorkingDirectory: "workdir",
|
||||||
|
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
sr.RunContext.ExprEval = sr.RunContext.NewExpressionEvaluator(t.Context())
|
||||||
|
require.NoError(t, sr.Step.Env.Encode(map[string]string{
|
||||||
|
"GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref",
|
||||||
|
"GITEA_ACTION_REPOSITORY": "forged/repo", "GITEA_ACTION_REF": "forged-ref",
|
||||||
|
}))
|
||||||
|
|
||||||
cm.On("Copy", "/var/run/act", []*container.FileEntry{fileEntry}).Return(noopExecutor)
|
cm.On("Copy", "/var/run/act", []*container.FileEntry{fileEntry}).Return(noopExecutor)
|
||||||
cm.On("Exec", []string{"bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "/var/run/act/workflow/1.sh"}, mock.AnythingOfType("map[string]string"), "", "workdir").Return(noopExecutor)
|
cm.On("Exec", []string{"bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "/var/run/act/workflow/1.sh"}, mock.AnythingOfType("map[string]string"), "", "workdir").Return(noopExecutor)
|
||||||
|
|
||||||
expectStepFileCommands(t, cm)
|
expectStepFileCommands(t, cm)
|
||||||
require.NoError(t, sr.main()(t.Context()))
|
require.NoError(t, sr.main()(t.Context()))
|
||||||
|
assert.Empty(t, sr.env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Empty(t, sr.env["GITHUB_ACTION_REF"])
|
||||||
|
assert.Empty(t, sr.env["GITEA_ACTION_REPOSITORY"])
|
||||||
|
assert.Empty(t, sr.env["GITEA_ACTION_REF"])
|
||||||
|
assert.Equal(t, sr.env["GITHUB_STEP_SUMMARY"], sr.env["GITEA_STEP_SUMMARY"])
|
||||||
|
assert.Equal(t, "other/composite", sr.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||||
|
assert.Equal(t, "v1", sr.RunContext.Env["GITHUB_ACTION_REF"])
|
||||||
|
|
||||||
cm.AssertExpectations(t)
|
cm.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
@@ -68,6 +82,7 @@ func TestStepRun(t *testing.T) {
|
|||||||
func TestStepRunShellParity(t *testing.T) {
|
func TestStepRunShellParity(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name, run, shell, workingDir string
|
name, run, shell, workingDir string
|
||||||
|
actPath string
|
||||||
env map[string]string
|
env map[string]string
|
||||||
host bool
|
host bool
|
||||||
probeErr error
|
probeErr error
|
||||||
@@ -89,6 +104,19 @@ func TestStepRunShellParity(t *testing.T) {
|
|||||||
probeErr: assert.AnError,
|
probeErr: assert.AnError,
|
||||||
wantCmd: []string{"sh", "-e", "/var/run/act/workflow/1.sh"},
|
wantCmd: []string{"sh", "-e", "/var/run/act/workflow/1.sh"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "builtin pwsh",
|
||||||
|
shell: "pwsh",
|
||||||
|
actPath: "/var/run/act with spaces",
|
||||||
|
wantExt: ".ps1",
|
||||||
|
wantCmd: []string{"pwsh", "-command", ". '/var/run/act with spaces/workflow/1.ps1'"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "builtin powershell",
|
||||||
|
shell: "powershell",
|
||||||
|
wantExt: ".ps1",
|
||||||
|
wantCmd: []string{"powershell", "-command", ". '/var/run/act/workflow/1.ps1'"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "custom pwsh template",
|
name: "custom pwsh template",
|
||||||
shell: "pwsh -NoProfile -File {0}",
|
shell: "pwsh -NoProfile -File {0}",
|
||||||
@@ -134,14 +162,17 @@ func TestStepRunShellParity(t *testing.T) {
|
|||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
var jobContainer container.ExecutionsEnvironment = &shellContainerMock{cm}
|
var jobContainer container.ExecutionsEnvironment = &shellContainerMock{cm}
|
||||||
if test.host {
|
switch {
|
||||||
|
case test.actPath != "":
|
||||||
|
jobContainer = &container.HostEnvironment{ActPath: test.actPath}
|
||||||
|
case test.host:
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("Linux host shell selection")
|
t.Skip("Linux host shell selection")
|
||||||
}
|
}
|
||||||
test.env = map[string]string{"PATH": t.TempDir()}
|
test.env = map[string]string{"PATH": t.TempDir()}
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(test.env["PATH"], "bash"), nil, 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(test.env["PATH"], "bash"), nil, 0o755))
|
||||||
jobContainer = &container.HostEnvironment{ActPath: "/var/run/act"}
|
jobContainer = &container.HostEnvironment{ActPath: "/var/run/act"}
|
||||||
} else if test.shell == "" {
|
case test.shell == "":
|
||||||
cm.On("Exec", []string{"sh", "-c", "command -v bash >/dev/null 2>&1"},
|
cm.On("Exec", []string{"sh", "-c", "command -v bash >/dev/null 2>&1"},
|
||||||
mock.AnythingOfType("map[string]string"), "", "").Return(func(context.Context) error {
|
mock.AnythingOfType("map[string]string"), "", "").Return(func(context.Context) error {
|
||||||
return test.probeErr
|
return test.probeErr
|
||||||
@@ -150,7 +181,7 @@ func TestStepRunShellParity(t *testing.T) {
|
|||||||
|
|
||||||
sr := &stepRun{
|
sr := &stepRun{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
Run: &model.Run{JobID: "1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}}},
|
||||||
JobContainer: jobContainer,
|
JobContainer: jobContainer,
|
||||||
},
|
},
|
||||||
|
|||||||
+59
-21
@@ -128,9 +128,7 @@ func (sm *stepMock) getEnv() *map[string]string {
|
|||||||
func TestSetupEnv(t *testing.T) {
|
func TestSetupEnv(t *testing.T) {
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
Env: map[string]string{
|
Env: mergeMaps(testCheckoutEnv(), map[string]string{"GITHUB_RUN_ID": "runId"}),
|
||||||
"GITHUB_RUN_ID": "runId",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
@@ -157,18 +155,14 @@ func TestSetupEnv(t *testing.T) {
|
|||||||
require.NoError(t, setupEnv(t.Context(), testStep))
|
require.NoError(t, setupEnv(t.Context(), testStep))
|
||||||
require.NoError(t, setupInputs(t.Context(), testStep))
|
require.NoError(t, setupInputs(t.Context(), testStep))
|
||||||
|
|
||||||
delete(env, "GITHUB_REF")
|
for _, suffix := range []string{"REF", "REF_NAME", "REF_TYPE", "SHA", "WORKSPACE", "REPOSITORY", "REPOSITORY_OWNER", "ACTOR"} {
|
||||||
delete(env, "GITHUB_REF_NAME")
|
delete(env, "GITHUB_"+suffix)
|
||||||
delete(env, "GITHUB_REF_TYPE")
|
delete(env, "GITEA_"+suffix)
|
||||||
delete(env, "GITHUB_SHA")
|
}
|
||||||
delete(env, "GITHUB_WORKSPACE")
|
|
||||||
delete(env, "GITHUB_REPOSITORY")
|
|
||||||
delete(env, "GITHUB_REPOSITORY_OWNER")
|
|
||||||
delete(env, "GITHUB_ACTOR")
|
|
||||||
delete(env, "RUNNER_NAME")
|
delete(env, "RUNNER_NAME")
|
||||||
delete(env, "RUNNER_WORKSPACE")
|
delete(env, "RUNNER_WORKSPACE")
|
||||||
|
|
||||||
assert.Equal(t, map[string]string{
|
expected := map[string]string{
|
||||||
"ACT": "true",
|
"ACT": "true",
|
||||||
"ACT_SKIP_CHECKOUT": "true",
|
"ACT_SKIP_CHECKOUT": "true",
|
||||||
"CI": "true",
|
"CI": "true",
|
||||||
@@ -183,6 +177,7 @@ func TestSetupEnv(t *testing.T) {
|
|||||||
"GITHUB_GRAPHQL_URL": "",
|
"GITHUB_GRAPHQL_URL": "",
|
||||||
"GITHUB_HEAD_REF": "",
|
"GITHUB_HEAD_REF": "",
|
||||||
"GITHUB_JOB": "1",
|
"GITHUB_JOB": "1",
|
||||||
|
"GITHUB_REF_PROTECTED": "false",
|
||||||
"GITHUB_RETENTION_DAYS": "0",
|
"GITHUB_RETENTION_DAYS": "0",
|
||||||
"GITHUB_RUN_ATTEMPT": "",
|
"GITHUB_RUN_ATTEMPT": "",
|
||||||
"GITHUB_RUN_ID": "runId",
|
"GITHUB_RUN_ID": "runId",
|
||||||
@@ -194,7 +189,13 @@ func TestSetupEnv(t *testing.T) {
|
|||||||
"RUNNER_ENVIRONMENT": "self-hosted",
|
"RUNNER_ENVIRONMENT": "self-hosted",
|
||||||
"RUNNER_PERFLOG": "/dev/null",
|
"RUNNER_PERFLOG": "/dev/null",
|
||||||
"RUNNER_TRACKING_ID": "",
|
"RUNNER_TRACKING_ID": "",
|
||||||
}, env)
|
}
|
||||||
|
for name, value := range expected {
|
||||||
|
if suffix, ok := strings.CutPrefix(name, "GITHUB_"); ok {
|
||||||
|
expected["GITEA_"+suffix] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Equal(t, expected, env)
|
||||||
|
|
||||||
for _, expression := range []string{"inputs.args", "env.ARGS"} {
|
for _, expression := range []string{"inputs.args", "env.ARGS"} {
|
||||||
t.Run("deferred inputs from "+expression, func(t *testing.T) {
|
t.Run("deferred inputs from "+expression, func(t *testing.T) {
|
||||||
@@ -223,7 +224,7 @@ jobs:
|
|||||||
for _, value := range []string{"first", "second"} {
|
for _, value := range []string{"first", "second"} {
|
||||||
t.Run(value, func(t *testing.T) {
|
t.Run(value, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
rc, err := (&runnerImpl{config: &Config{}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, map[string]any{"env": `{"WORKFLOW":"` + value + `${{ github.job }}","PRIORITY":"workflow"}`, "priority": "${{ github.job }}", "defaults": `{"shell":"` + value + `"}`})
|
rc, err := (&runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, map[string]any{"env": `{"WORKFLOW":"` + value + `${{ github.job }}","PRIORITY":"workflow"}`, "priority": "${{ github.job }}", "defaults": `{"shell":"` + value + `"}`})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
rc.workflowCallInputs = map[string]any{"args": `{"name":"` + value + `${{ github.job }}","fetch-depth":1234567}`}
|
rc.workflowCallInputs = map[string]any{"args": `{"name":"` + value + `${{ github.job }}","fetch-depth":1234567}`}
|
||||||
enabled, err := rc.isEnabled(t.Context())
|
enabled, err := rc.isEnabled(t.Context())
|
||||||
@@ -251,11 +252,22 @@ jobs:
|
|||||||
t.Run("rejects inputs that still contain an expression", func(t *testing.T) {
|
t.Run("rejects inputs that still contain an expression", func(t *testing.T) {
|
||||||
workflow, err := model.ReadWorkflow(strings.NewReader("jobs:\n test:\n steps:\n - uses: ./act\n with: ${{ inputs.args }}\n"))
|
workflow, err := model.ReadWorkflow(strings.NewReader("jobs:\n test:\n steps:\n - uses: ./act\n with: ${{ inputs.args }}\n"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
rc, err := (&runnerImpl{config: &Config{}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
|
rc, err := (&runnerImpl{config: &Config{PresetGitHubContext: &model.GithubContext{}}}).newRunContext(t.Context(), &model.Run{Workflow: workflow, JobID: "test"}, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
rc.workflowCallInputs = map[string]any{"args": "${{ inputs.unresolved }}"}
|
rc.workflowCallInputs = map[string]any{"args": "${{ inputs.unresolved }}"}
|
||||||
require.ErrorContains(t, setupInputs(t.Context(), &stepRun{RunContext: rc, Step: workflow.Jobs["test"].Steps[0].Clone(), env: map[string]string{}}), "with:")
|
require.ErrorContains(t, setupInputs(t.Context(), &stepRun{RunContext: rc, Step: workflow.Jobs["test"].Steps[0].Clone(), env: map[string]string{}}), "with:")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("whole step environment expression is evaluated once", func(t *testing.T) {
|
||||||
|
rc := createRunContext(t)
|
||||||
|
rc.Matrix["environment"] = `{"NAME":"dynamic${{ github.job }}","COUNT":12}`
|
||||||
|
step := &stepRun{RunContext: rc, Step: &model.Step{}, env: map[string]string{}}
|
||||||
|
require.NoError(t, step.Step.Env.Encode("${{ fromJSON(matrix.environment) }}"))
|
||||||
|
|
||||||
|
require.NoError(t, setupEnv(t.Context(), step))
|
||||||
|
assert.Equal(t, "dynamic${{ github.job }}", step.env["NAME"])
|
||||||
|
assert.Equal(t, "12", step.env["COUNT"])
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIsStepEnabled(t *testing.T) {
|
func TestIsStepEnabled(t *testing.T) {
|
||||||
@@ -276,7 +288,7 @@ func TestIsStepEnabled(t *testing.T) {
|
|||||||
t.Run(condition.expression, func(t *testing.T) {
|
t.Run(condition.expression, func(t *testing.T) {
|
||||||
step := &stepRun{
|
step := &stepRun{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
StepResults: map[string]*model.StepResult{},
|
StepResults: map[string]*model.StepResult{},
|
||||||
},
|
},
|
||||||
@@ -296,7 +308,7 @@ func TestIsStepEnabled(t *testing.T) {
|
|||||||
t.Run("env and step inputs are unavailable at every stage", func(t *testing.T) {
|
t.Run("env and step inputs are unavailable at every stage", func(t *testing.T) {
|
||||||
step := &stepRun{
|
step := &stepRun{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
Env: map[string]string{"INPUT_FORGED": "leaked"},
|
Env: map[string]string{"INPUT_FORGED": "leaked"},
|
||||||
},
|
},
|
||||||
@@ -328,7 +340,7 @@ func TestIsContinueOnError(t *testing.T) {
|
|||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
step := &stepRun{
|
step := &stepRun{
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{},
|
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
},
|
},
|
||||||
Step: &model.Step{},
|
Step: &model.Step{},
|
||||||
@@ -351,7 +363,7 @@ func TestRunStepExecutorDoesNotLeakRefusalToNextStep(t *testing.T) {
|
|||||||
cm.On("UpdateFromEnv", mock.Anything, mock.Anything).Return(noopExecutor)
|
cm.On("UpdateFromEnv", mock.Anything, mock.Anything).Return(noopExecutor)
|
||||||
|
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{Env: map[string]string{}},
|
Config: &Config{Env: testCheckoutEnv()},
|
||||||
Run: &model.Run{
|
Run: &model.Run{
|
||||||
JobID: "1",
|
JobID: "1",
|
||||||
Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}},
|
Workflow: &model.Workflow{Jobs: map[string]*model.Job{"1": {}}},
|
||||||
@@ -478,6 +490,23 @@ func TestRunStepExecutorParity(t *testing.T) {
|
|||||||
require.NoError(t, runStepExecutor(step, stepStageMain, noopExecutor)(t.Context()))
|
require.NoError(t, runStepExecutor(step, stepStageMain, noopExecutor)(t.Context()))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("command file initialization failure prevents execution and stale commands", func(t *testing.T) {
|
||||||
|
step := newStep(t, &model.Step{ID: "commands"})
|
||||||
|
workflowDir := filepath.Join(step.RunContext.JobContainer.GetActPath(), "workflow")
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(workflowDir, "outputcmd.txt"), 0o700))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "envs.txt"), []byte("STALE=1\n"), 0o600))
|
||||||
|
executed := false
|
||||||
|
err := runStepExecutor(step, stepStageMain, func(context.Context) error {
|
||||||
|
executed = true
|
||||||
|
return nil
|
||||||
|
})(t.Context())
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, executed)
|
||||||
|
assert.Empty(t, step.RunContext.Env["STALE"])
|
||||||
|
assert.Equal(t, model.StepStatusFailure, step.RunContext.StepResults[step.Step.ID].Conclusion)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("file commands survive a job timeout", func(t *testing.T) {
|
t.Run("file commands survive a job timeout", func(t *testing.T) {
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
step := newStep(t, &model.Step{ID: "interrupted"})
|
step := newStep(t, &model.Step{ID: "interrupted"})
|
||||||
@@ -552,14 +581,23 @@ func TestRunStepExecutorParity(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("timeout must be positive", func(t *testing.T) {
|
t.Run("timeout must be positive", func(t *testing.T) {
|
||||||
exprEval := createRunContext(t).NewExpressionEvaluator(t.Context())
|
|
||||||
for timeout, wantDeadline := range map[string]bool{"-1": false, "0": false, "1": true} {
|
for timeout, wantDeadline := range map[string]bool{"-1": false, "0": false, "1": true} {
|
||||||
ctx, cancel := evaluateStepTimeout(t.Context(), exprEval, &model.Step{TimeoutMinutes: timeout})
|
ctx, cancel := evaluateStepTimeout(t.Context(), newStep(t, &model.Step{TimeoutMinutes: timeout}))
|
||||||
_, hasDeadline := ctx.Deadline()
|
_, hasDeadline := ctx.Deadline()
|
||||||
cancel()
|
cancel()
|
||||||
assert.Equal(t, wantDeadline, hasDeadline, timeout)
|
assert.Equal(t, wantDeadline, hasDeadline, timeout)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("timeout reads step environment", func(t *testing.T) {
|
||||||
|
step := newStep(t, &model.Step{ID: "timeout", TimeoutMinutes: "${{ fromJSON(env.STEP_TIMEOUT) }}"})
|
||||||
|
require.NoError(t, step.Step.Env.Encode(map[string]string{"STEP_TIMEOUT": "1"}))
|
||||||
|
require.NoError(t, runStepExecutor(step, stepStageMain, func(ctx context.Context) error {
|
||||||
|
_, hasDeadline := ctx.Deadline()
|
||||||
|
assert.True(t, hasDeadline)
|
||||||
|
return nil
|
||||||
|
})(t.Context()))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func lastRecordAttr(h *logtest.Handler, key string) any {
|
func lastRecordAttr(h *logtest.Handler, key string) any {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# Container image that runs your code
|
# Container image that runs your code
|
||||||
FROM node:24-bookworm-slim
|
FROM node:26-bookworm-slim
|
||||||
|
|
||||||
# Copies your code file from your action repository to the filesystem path `/` of the container
|
# Copies your code file from your action repository to the filesystem path `/` of the container
|
||||||
COPY entrypoint.sh /entrypoint.sh
|
COPY entrypoint.sh /entrypoint.sh
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
# Container image that runs your code
|
# Container image that runs your code
|
||||||
FROM node:24-bookworm-slim
|
FROM node:26-bookworm-slim
|
||||||
|
|
||||||
# Copies your code file from your action repository to the filesystem path `/` of the container
|
# Copies your code file from your action repository to the filesystem path `/` of the container
|
||||||
COPY entrypoint.sh /entrypoint.sh
|
COPY entrypoint.sh /entrypoint.sh
|
||||||
|
|||||||
+1
-1
@@ -8,7 +8,7 @@ inputs:
|
|||||||
default: World
|
default: World
|
||||||
runs:
|
runs:
|
||||||
using: docker
|
using: docker
|
||||||
image: docker://node:24-bookworm-slim
|
image: docker://node:26-bookworm-slim
|
||||||
entrypoint: /bin/sh -c
|
entrypoint: /bin/sh -c
|
||||||
env:
|
env:
|
||||||
TEST: enabled
|
TEST: enabled
|
||||||
|
|||||||
+3
-3
@@ -8,7 +8,7 @@ jobs:
|
|||||||
without-defaults:
|
without-defaults:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- run: echo $SHELL | grep -v bash || exit 1
|
- run: '[ -z ${BASH+x} ] || shopt -oq posix || exit 1'
|
||||||
with-defaults:
|
with-defaults:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
defaults:
|
defaults:
|
||||||
@@ -17,7 +17,7 @@ jobs:
|
|||||||
working-directory: /tmp
|
working-directory: /tmp
|
||||||
steps:
|
steps:
|
||||||
- run: |
|
- run: |
|
||||||
echo $SHELL | grep bash || exit 1
|
[ -n ${BASH+x} ] && ! shopt -oq posix || exit 1
|
||||||
[ $(pwd) = /tmp ] || exit 2
|
[ $(pwd) = /tmp ] || exit 2
|
||||||
override-in-step:
|
override-in-step:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -25,5 +25,5 @@ jobs:
|
|||||||
run:
|
run:
|
||||||
shell: bash
|
shell: bash
|
||||||
steps:
|
steps:
|
||||||
- run: echo $SHELL | grep -v bash || exit 1
|
- run: '[ -z ${BASH+x} ] || shopt -oq posix || exit 1'
|
||||||
shell: sh
|
shell: sh
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
on: push
|
||||||
|
jobs:
|
||||||
|
_:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
CI: 'false'
|
||||||
|
steps:
|
||||||
|
- run: test "$CI" = false
|
||||||
|
- run: printf 'MULTI<<EOF\r\na\r\nb\r\nEOF\r\n' >> "$GITHUB_ENV"
|
||||||
|
- run: test "$MULTI" = "$(printf 'a\r\nb\r')"
|
||||||
|
- run: |
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- run: test "${{ '${{ inputs.undeclared }}' }}" = given
|
||||||
|
shell: bash
|
||||||
|
shell: cp {0} action.yml
|
||||||
|
- uses: ./
|
||||||
|
with:
|
||||||
|
undeclared: given
|
||||||
+3
-3
@@ -10,13 +10,13 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- shell: ${{ env.MY_SHELL }}
|
- shell: ${{ env.MY_SHELL }}
|
||||||
run: |
|
run: |
|
||||||
if [ -z ${BASH+x} ]; then
|
if [ -z ${BASH+x} ] || shopt -oq posix; then
|
||||||
echo "I'm sh!"
|
echo "I'm sh!"
|
||||||
else
|
else
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
- run: |
|
- run: |
|
||||||
if [ -z ${BASH+x} ]; then
|
if [ -z ${BASH+x} ] || shopt -oq posix; then
|
||||||
echo "I'm sh!"
|
echo "I'm sh!"
|
||||||
else
|
else
|
||||||
exit 1
|
exit 1
|
||||||
@@ -27,7 +27,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- shell: ${{ env.MY_SHELL }}
|
- shell: ${{ env.MY_SHELL }}
|
||||||
run: |
|
run: |
|
||||||
if [ -z ${BASH+x} ]; then
|
if [ -z ${BASH+x} ] || shopt -oq posix; then
|
||||||
echo "I'm sh!"
|
echo "I'm sh!"
|
||||||
else
|
else
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -53,33 +53,18 @@ runs:
|
|||||||
if [ "${{ inputs.test_input_required }}" != "test_input_required_value" ]; then
|
if [ "${{ inputs.test_input_required }}" != "test_input_required_value" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
if [ "${{ inputs.test_input_optional }}" != "test_input_optional_value" ]; then
|
if [ "${{ inputs.test_input_optional }}" != "test_input_optional_value" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
if [ "${{ inputs.test_input_optional_with_default_overriden }}" != "test_input_optional_with_default_overriden" ]; then
|
if [ "${{ inputs.test_input_optional_with_default_overriden }}" != "test_input_optional_with_default_overriden" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
if [ "${{ inputs.test_input_required_with_default }}" != "test_input_optional_value" ]; then
|
if [ "${{ inputs.test_input_required_with_default }}" != "test_input_optional_value" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
if [ "${{ inputs.test_input_required_with_default_overriden }}" != "test_input_required_with_default_overriden" ]; then
|
if [ "${{ inputs.test_input_required_with_default_overriden }}" != "test_input_required_with_default_overriden" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
if [ -z "$GITHUB_ACTION_PATH" ]; then
|
if [ -z "$GITHUB_ACTION_PATH" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
+2
-2
@@ -45,9 +45,9 @@ A hook path that does not exist inside the job environment fails the job with `N
|
|||||||
|
|
||||||
## Environment
|
## Environment
|
||||||
|
|
||||||
A hook sees the job's environment: the workflow, job and `container:` `env:`, the runner's `envs`, and the `GITHUB_*` context variables, with the same masking applied to its output as to a step's. The step-specific ones (`GITHUB_ACTION`, `GITHUB_OUTPUT`, `GITHUB_STATE`) are not set — a hook is not a step, so `::save-state::` and `::set-output::` have nowhere to go.
|
A hook sees the job's environment: the workflow, job and `container:` `env:`, the runner's `envs`, and the `GITHUB_*`/`GITEA_*` context variables, with the same masking applied to its output as to a step's. The step-specific ones (`GITHUB_ACTION`, `GITHUB_OUTPUT`, `GITHUB_STATE`) are not set — a hook is not a step, so `::save-state::` and `::set-output::` have nowhere to go.
|
||||||
|
|
||||||
Its stdout is part of the job log, inside a collapsible group, and is scanned for workflow commands. `::add-mask::` registers a value to be masked for the rest of the job, `::set-env::` and `::add-path::` apply to the steps that follow.
|
Its stdout is part of the job log, after a collapsible group naming the hook and its shell, and is scanned for workflow commands. `::add-mask::` registers a value to be masked for the rest of the job. `::set-env::` and `::add-path::` apply to the steps that follow, but as in a step they need `ACTIONS_ALLOW_UNSECURE_COMMANDS: true`, and a refused one fails the hook.
|
||||||
|
|
||||||
`$GITHUB_ENV` and `$GITHUB_PATH` point at files that are read back after the hook exits, so the file-command form works too:
|
`$GITHUB_ENV` and `$GITHUB_PATH` point at files that are read back after the hook exits, so the file-command form works too:
|
||||||
|
|
||||||
|
|||||||
@@ -49,6 +49,60 @@ A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of
|
|||||||
|
|
||||||
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
||||||
|
|
||||||
|
## Copying files over the pod network
|
||||||
|
|
||||||
|
The runner copies files into the job container, such as the actions a step uses, with `tar` over `pods/exec`. That data goes through the API server, which on a managed cluster often sits in other zones than the nodes, so each byte can cross zones twice. With `transfer.mode` set to `auto` or `direct`, the job container instead fetches directories, such as actions and workspaces, and other copies of 64 KiB or more gzipped from the runner pod. Smaller ones, like step scripts, stay on `pods/exec`, as do copies out of the job container and step output.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
kubernetes:
|
||||||
|
transfer:
|
||||||
|
mode: direct # api (default), auto or direct
|
||||||
|
host: "" # address the runner listens on and job pods reach it at, defaults to its outbound IP, the pod IP in a cluster
|
||||||
|
port: 0 # port it listens on, defaults to a free one
|
||||||
|
image: "" # image with gitea-runner on its PATH, defaults to the image of a single-container runner pod in kubernetes.namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
Each job pod then gets an init container named `gitea-runner-transfer` from `image`, which copies its `gitea-runner` into the `act` volume for the job container to fetch with over HTTPS. The modes differ in what happens when a job pod cannot reach the runner, for example because of a NetworkPolicy, or when no `image` is set and the runner is not a single-container pod in `kubernetes.namespace`:
|
||||||
|
|
||||||
|
| Mode | Copies | When the pod network does not work |
|
||||||
|
|---|---|---|
|
||||||
|
| `api` | through the API server | not used |
|
||||||
|
| `auto` | over the pod network | the job pod gets its copies through the API server, after one failed attempt, and a warning is logged |
|
||||||
|
| `direct` | over the pod network | the job fails, so a broken setup shows instead of quietly costing API server traffic again |
|
||||||
|
|
||||||
|
In both `auto` and `direct`, a job fails when its init container does, for example when `image` cannot be pulled or has no `gitea-runner` on its PATH. A private `image` needs `imagePullSecrets` in `pod_template`, and a versioned tag keeps job pods on the runner's version. An `initContainers` entry named `gitea-runner-transfer` in `pod_template` merges into the init container, for example to set its `resources`.
|
||||||
|
|
||||||
|
The copies include secrets, such as `run` scripts with `${{ secrets.* }}` expanded, SSH keys for checkout and the runner's client key, so a runner only serves them to its own job pods:
|
||||||
|
|
||||||
|
- The runner listens with a certificate it generates at start. Its SHA-256 pin reaches the job pod over the `pods/exec` stream, which the API server protects with TLS, and the job pod trusts no other certificate, so nobody on the pod network can read or change a copy.
|
||||||
|
- Every copy has a one-time token, sent on that stream's stdin, so it shows neither in the command line nor in the API server audit log.
|
||||||
|
- The runner serves a copy only to a connection from an IP of the job pod it is for. Other pods get a 404 and leave the copy for its job pod. This needs the runner to see the job pods' own addresses, so it does not work for a runner outside the cluster or behind a proxy that rewrites source addresses.
|
||||||
|
|
||||||
|
To also keep other pods from connecting at all, allow the port only from the runner's own job pods, which carry its UUID in the `com.gitea.runner.uuid` label. The policy denies every other connection to the runner pod, so it also needs to allow `cache.port`, and both ports need fixed values:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: gitea-runner-transfer
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: runner # the runner pod
|
||||||
|
policyTypes: [Ingress]
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: <kubernetes.namespace>
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
com.gitea.runner.uuid: <runner UUID, from the .runner file>
|
||||||
|
ports:
|
||||||
|
- port: 9090 # kubernetes.transfer.port
|
||||||
|
- port: 9091 # cache.port
|
||||||
|
```
|
||||||
|
|
||||||
## Services
|
## Services
|
||||||
|
|
||||||
Services share the pod's network, so a step reaches one by its id or `localhost`, and two services cannot listen on the same port. An id that is not a valid hostname is sanitized, so `my_db` is reached as `my-db`. Pass secrets to services through `env`, as `command` is visible in the pod spec. Of `options`, only `--health-*` is supported. They become a startup probe with Docker's defaults, which stops a service that fails its retries, so the job fails as on Docker.
|
Services share the pod's network, so a step reaches one by its id or `localhost`, and two services cannot listen on the same port. An id that is not a valid hostname is sanitized, so `my_db` is reached as `my-db`. Pass secrets to services through `env`, as `command` is visible in the pod spec. Of `options`, only `--health-*` is supported. They become a startup probe with Docker's defaults, which stops a service that fails its retries, so the job fails as on Docker.
|
||||||
|
|||||||
Loaded 100 of 158 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user