mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-05 14:55:44 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 | ||
|
|
c719a88f36 | ||
|
|
d882311ec6 | ||
|
|
923a0bfb59 | ||
|
|
6f0d4f1b48 | ||
|
|
430f7bb2cf | ||
|
|
befb1b4bb9 | ||
|
|
1a7f09224f | ||
|
|
af2b3f6df5 | ||
|
|
785313cc21 | ||
|
|
41f1e08bec | ||
|
|
4158e05dae | ||
|
|
e356c2ed85 | ||
|
|
ed940db4f0 | ||
|
|
755af87095 | ||
|
|
b9aa47d1a5 | ||
|
|
0d84ec6dab | ||
|
|
b769a9765f | ||
|
|
9c4c66e70f | ||
|
|
42858c61df | ||
|
|
470969dc74 | ||
|
|
be7ddcd15a | ||
|
|
c3c647ef83 | ||
|
|
160e18e705 | ||
|
|
20c8b4abc4 | ||
|
|
ccb9afd7a5 | ||
|
|
49e34813ae | ||
|
|
e0ef637805 | ||
|
|
8502663d0e | ||
|
|
4e8037a93b | ||
|
|
76110557ab | ||
|
|
3bc5cc03f8 | ||
|
|
de961b91eb | ||
|
|
ad9eeb668a | ||
|
|
b741335222 | ||
|
|
20143c9558 |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
|||||||
|
|
||||||
/bin
|
/bin
|
||||||
/dist
|
/dist
|
||||||
|
/.tmp
|
||||||
|
/.git/hooks
|
||||||
|
/.git/logs
|
||||||
|
/.git/rr-cache
|
||||||
/.env
|
/.env
|
||||||
/.runner
|
/.runner
|
||||||
|
/.runner.lock
|
||||||
/config.yaml
|
/config.yaml
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: go-setup
|
||||||
|
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||||
|
id: setup-go
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: false
|
||||||
|
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||||
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||||
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: cache-seeder
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- go.mod
|
||||||
|
- go.sum
|
||||||
|
- .golangci.yml
|
||||||
|
- Makefile
|
||||||
|
- .gitea/actions/go-setup/action.yml
|
||||||
|
- .gitea/workflows/cache-seeder.yml
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
seed:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- run: make deps-tools lint build
|
||||||
|
- name: compile the test binaries without running them
|
||||||
|
run: |
|
||||||
|
make test GOTEST_FLAGS='-race -exec true'
|
||||||
|
go test -tags e2e -exec true ./e2e/...
|
||||||
@@ -19,9 +19,9 @@ jobs:
|
|||||||
timeout-minutes: 5
|
timeout-minutes: 5
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 26
|
||||||
- run: make lint-pr-title
|
- run: make lint-pr-title
|
||||||
env:
|
env:
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
@@ -11,7 +11,6 @@ on:
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: nightly
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
goreleaser:
|
goreleaser:
|
||||||
@@ -36,7 +35,7 @@ jobs:
|
|||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release --nightly
|
args: release --nightly
|
||||||
@@ -80,7 +79,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
passphrase: ${{ secrets.PASSPHRASE }}
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release
|
args: release
|
||||||
@@ -69,7 +69,6 @@ jobs:
|
|||||||
image: catthehacker/ubuntu:act-latest
|
image: catthehacker/ubuntu:act-latest
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: latest
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
@@ -83,7 +82,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
+11
-15
@@ -17,25 +17,24 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||||
- name: pre-pull test images
|
- name: pre-pull test images
|
||||||
env:
|
env:
|
||||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:0e0ff40c39bc087845bfb27465a0df4ea419520094bc35842ff83dd8cbe6f9b6 # renovate: datasource=docker
|
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||||
run: |
|
run: |
|
||||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||||
for attempt in 1 2 3; do
|
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||||
docker pull "$image" && break
|
for attempt in 1 2 3; do
|
||||||
[ "$attempt" = 3 ] || sleep 5
|
docker pull "$image" && break
|
||||||
done
|
[ "$attempt" = 3 ] || sleep 5
|
||||||
|
done
|
||||||
|
fi
|
||||||
docker tag "$image" "${image%@*}"
|
docker tag "$image" "${image%@*}"
|
||||||
done
|
done
|
||||||
- name: lint
|
- name: lint
|
||||||
@@ -68,10 +67,7 @@ jobs:
|
|||||||
- gitea/gitea:main-nightly
|
- gitea/gitea:main-nightly
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
- name: e2e compatibility
|
- name: e2e compatibility
|
||||||
|
|||||||
@@ -39,6 +39,13 @@ linters:
|
|||||||
deny:
|
deny:
|
||||||
- pkg: golang.org/x/exp
|
- pkg: golang.org/x/exp
|
||||||
desc: it's experimental and unreliable
|
desc: it's experimental and unreliable
|
||||||
|
logging:
|
||||||
|
files:
|
||||||
|
- $all
|
||||||
|
- "!**/act/common/logging/**"
|
||||||
|
deny:
|
||||||
|
- pkg: log/slog
|
||||||
|
desc: log through gitea.com/gitea/runner/act/common/logging
|
||||||
nolintlint:
|
nolintlint:
|
||||||
allow-unused: false
|
allow-unused: false
|
||||||
require-explanation: true
|
require-explanation: true
|
||||||
|
|||||||
@@ -28,38 +28,18 @@ builds:
|
|||||||
ignore:
|
ignore:
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: arm
|
goarch: arm
|
||||||
- goos: darwin
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
- goos: freebsd
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
flags:
|
flags:
|
||||||
|
|||||||
+29
-4
@@ -28,8 +28,25 @@ depending on the prefix:
|
|||||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
||||||
encoded forms too.
|
encoded forms too.
|
||||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
|
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||||
decodes exactly those two when folding a location into an annotation.
|
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||||
|
|
||||||
|
## Logging
|
||||||
|
|
||||||
|
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
|
||||||
|
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
|
||||||
|
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
|
||||||
|
fields.
|
||||||
|
|
||||||
|
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
|
||||||
|
Going around either leaks them.
|
||||||
|
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
|
||||||
|
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
|
||||||
|
job logger.
|
||||||
|
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
|
||||||
|
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
|
||||||
|
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
|
||||||
|
package functions.
|
||||||
|
|
||||||
## Built-in actions
|
## Built-in actions
|
||||||
|
|
||||||
@@ -40,7 +57,15 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
|||||||
|
|
||||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||||
parallel.
|
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||||
|
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||||
|
|
||||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||||
|
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||||
|
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||||
|
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||||
|
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||||
|
Docker and Kubernetes.
|
||||||
|
|
||||||
|
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||||
+6
-4
@@ -1,7 +1,7 @@
|
|||||||
### BUILDER STAGE
|
### BUILDER STAGE
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM golang:1.27-alpine3.23 AS builder
|
FROM golang:1.27-alpine AS builder
|
||||||
|
|
||||||
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
||||||
RUN apk add --no-cache make git
|
RUN apk add --no-cache make git
|
||||||
@@ -12,12 +12,14 @@ ENV GOPROXY=${GOPROXY:-}
|
|||||||
COPY . /opt/src/runner
|
COPY . /opt/src/runner
|
||||||
WORKDIR /opt/src/runner
|
WORKDIR /opt/src/runner
|
||||||
|
|
||||||
RUN make clean && make build
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
make clean && make build
|
||||||
|
|
||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind AS dind
|
FROM docker:29.9.2-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -39,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind-rootless AS dind-rootless
|
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
|
|||||||
@@ -1,31 +1,22 @@
|
|||||||
DIST := dist
|
DIST := dist
|
||||||
EXECUTABLE := gitea-runner
|
EXECUTABLE := gitea-runner
|
||||||
DIST_DIRS := $(DIST)/binaries $(DIST)/release
|
|
||||||
GO ?= go
|
GO ?= go
|
||||||
SHASUM ?= shasum -a 256
|
|
||||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||||
XGO_PACKAGE ?= src.techknowlogick.com/[email protected] # renovate: datasource=go
|
|
||||||
XGO_VERSION := go-1.27.x
|
|
||||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
|
|
||||||
|
|
||||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
|
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
|
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||||
|
|
||||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
|
||||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
|
||||||
WINDOWS_ARCHS ?= windows/amd64
|
|
||||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||||
|
|
||||||
DOCKER_IMAGE ?= gitea/runner
|
DOCKER_IMAGE ?= gitea/runner
|
||||||
DOCKER_TAG ?= nightly
|
DOCKER_TAG ?= nightly
|
||||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
|
||||||
|
|
||||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||||
|
|
||||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||||
|
|
||||||
@@ -38,22 +29,16 @@ endif
|
|||||||
ifeq ($(HAS_GO), GO)
|
ifeq ($(HAS_GO), GO)
|
||||||
GOPATH ?= $(shell $(GO) env GOPATH)
|
GOPATH ?= $(shell $(GO) env GOPATH)
|
||||||
export PATH := $(GOPATH)/bin:$(PATH)
|
export PATH := $(GOPATH)/bin:$(PATH)
|
||||||
|
|
||||||
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
|
|
||||||
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(OS), Windows_NT)
|
ifeq ($(OS), Windows_NT)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else ifeq ($(OS), Windows)
|
else ifeq ($(OS), Windows)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else
|
else
|
||||||
GOFLAGS := -v
|
GOFLAGS := -v
|
||||||
EXECUTABLE ?= $(EXECUTABLE)
|
|
||||||
GO_ENV_WINDOWS := GOOS=windows
|
GO_ENV_WINDOWS := GOOS=windows
|
||||||
endif
|
endif
|
||||||
|
|
||||||
@@ -108,8 +93,6 @@ fmt-check: ## check the Go code formatting
|
|||||||
.PHONY: deps-tools
|
.PHONY: deps-tools
|
||||||
deps-tools: ## install tool dependencies
|
deps-tools: ## install tool dependencies
|
||||||
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
||||||
$(GO) install $(GXZ_PACKAGE) & \
|
|
||||||
$(GO) install $(XGO_PACKAGE) & \
|
|
||||||
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
||||||
wait
|
wait
|
||||||
|
|
||||||
@@ -171,7 +154,7 @@ E2E_CONCURRENCY ?= 8
|
|||||||
|
|
||||||
.PHONY: test-e2e
|
.PHONY: test-e2e
|
||||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||||
|
|
||||||
.PHONY: install
|
.PHONY: install
|
||||||
install: $(GOFILES) ## install the runner binary via `go install`
|
install: $(GOFILES) ## install the runner binary via `go install`
|
||||||
@@ -187,45 +170,6 @@ $(EXECUTABLE): $(GOFILES)
|
|||||||
deps-backend: ## install backend dependencies
|
deps-backend: ## install backend dependencies
|
||||||
$(GO) mod download
|
$(GO) mod download
|
||||||
|
|
||||||
.PHONY: release
|
|
||||||
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
|
|
||||||
|
|
||||||
$(DIST_DIRS):
|
|
||||||
mkdir -p $(DIST_DIRS)
|
|
||||||
|
|
||||||
.PHONY: release-windows
|
|
||||||
release-windows: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-linux
|
|
||||||
release-linux: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-darwin
|
|
||||||
release-darwin: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-copy
|
|
||||||
release-copy: | $(DIST_DIRS)
|
|
||||||
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
|
|
||||||
|
|
||||||
.PHONY: release-check
|
|
||||||
release-check: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
|
|
||||||
|
|
||||||
.PHONY: release-compress
|
|
||||||
release-compress: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
|
|
||||||
|
|
||||||
.PHONY: docker
|
.PHONY: docker
|
||||||
docker: ## build the docker image
|
docker: ## build the docker image
|
||||||
if ! docker buildx version >/dev/null 2>&1; then \
|
if ! docker buildx version >/dev/null 2>&1; then \
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
On the runner:
|
On the runner:
|
||||||
|
|
||||||
- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows.
|
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
|
||||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||||
|
|
||||||
Where steps run, in the job image or on the runner in host mode:
|
Where steps run, in the job image or on the runner in host mode:
|
||||||
@@ -242,7 +242,7 @@ A job's `services` are started before its steps run. When a service's image or i
|
|||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:17
|
image: postgres:18
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd pg_isready
|
--health-cmd pg_isready
|
||||||
--health-interval 5s
|
--health-interval 5s
|
||||||
@@ -303,6 +303,8 @@ Use `uses: builtin:checkout` to check out a repository without downloading `acti
|
|||||||
|
|
||||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||||
|
|
||||||
|
As on GitHub, a job restores caches saved for its own ref, its pull request's base branch and the default branch, so a cache saved by a pull request is only restored by that pull request.
|
||||||
|
|
||||||
**Sharing a cache between runners**
|
**Sharing a cache between runners**
|
||||||
|
|
||||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||||
@@ -368,7 +370,7 @@ While the runner is idle it cleans up after earlier jobs:
|
|||||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||||
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,5 @@
|
|||||||
//
|
//
|
||||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||||
//
|
//
|
||||||
// TODO: Authorization
|
|
||||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
|
||||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||||
package artifactcache
|
package artifactcache
|
||||||
+383
-220
File diff suppressed because it is too large.
Load diff
+364
-727
File diff suppressed because it is too large.
Load diff
@@ -15,8 +15,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The cache service v2 API. A client on this version talks twirp to
|
// The cache service v2 API. A client on this version talks twirp to
|
||||||
@@ -44,16 +42,16 @@ const (
|
|||||||
twirpUnauthenticated = "unauthenticated"
|
twirpUnauthenticated = "unauthenticated"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *Handler) registerV2Routes(router *httprouter.Router) {
|
func (h *Handler) registerV2Routes(router *http.ServeMux) {
|
||||||
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
||||||
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
||||||
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
||||||
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
||||||
}
|
}
|
||||||
|
|
||||||
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
||||||
// the upload.
|
// the upload.
|
||||||
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -75,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||||
// existing and silently never saved.
|
// existing and silently never saved.
|
||||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if existing != nil {
|
} else if existing != nil {
|
||||||
@@ -86,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
|
|
||||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||||
owner := hashedToken(bearerToken(r))
|
owner := hashedToken(bearerToken(r))
|
||||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||||
@@ -102,15 +100,17 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache := &Cache{
|
cache := &Cache{
|
||||||
Repo: cred.Repo,
|
Repo: cred.Repo,
|
||||||
Key: req.Key,
|
Ref: cred.Ref,
|
||||||
Version: req.Version,
|
Key: req.Key,
|
||||||
Owner: owner,
|
Version: req.Version,
|
||||||
Size: -1, // the size is only known at finalize time
|
Owner: owner,
|
||||||
CreatedAt: now,
|
Size: -1, // the size is only known at finalize time
|
||||||
UsedAt: now,
|
CreatedAt: now.Unix(),
|
||||||
|
CreatedAtNanos: now.Nanosecond(),
|
||||||
|
UsedAt: now.Unix(),
|
||||||
}
|
}
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -123,7 +123,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -138,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||||
cache = nil // not the reservation this job made, so not this job's to commit
|
cache = nil // not the reservation this job made, so not this job's to commit
|
||||||
}
|
}
|
||||||
if err == nil && cache == nil {
|
if err == nil && cache == nil {
|
||||||
// A retry whose first response was lost finds the entry already committed.
|
// A retry whose first response was lost finds the entry already committed.
|
||||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -172,7 +172,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
@@ -207,8 +207,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
||||||
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
||||||
// in order.
|
// in order.
|
||||||
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||||
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
|
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// v2Call posts a twirp request to the cache service and returns the decoded response.
|
|
||||||
// Field names are the proto ones, which is what the toolkit's client sends.
|
|
||||||
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -37,7 +35,6 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
|
|||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
|
|
||||||
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
|
|
||||||
func putBlob(t *testing.T, url string, content []byte) int {
|
func putBlob(t *testing.T, url string, content []byte) int {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -47,8 +44,6 @@ func putBlob(t *testing.T, url string, content []byte) int {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode == http.StatusCreated {
|
if resp.StatusCode == http.StatusCreated {
|
||||||
// The Azure SDK client dereferences this header without checking, so a blob upload that
|
|
||||||
// omits it panics the caller rather than failing it.
|
|
||||||
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
||||||
}
|
}
|
||||||
return resp.StatusCode
|
return resp.StatusCode
|
||||||
@@ -67,8 +62,6 @@ func getURL(t *testing.T, url string) []byte {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
|
|
||||||
// with the upload URL it used.
|
|
||||||
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -84,10 +77,8 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
|||||||
}), uploadURL
|
}), uploadURL
|
||||||
}
|
}
|
||||||
|
|
||||||
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
|
|
||||||
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
|
|
||||||
// or to replace a finalized entry.
|
|
||||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -98,10 +89,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
assert.NotEmpty(t, finalized["entry_id"])
|
assert.NotEmpty(t, finalized["entry_id"])
|
||||||
|
|
||||||
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
|
|
||||||
// and it is an upload URL only: nothing reads a blob back through it.
|
|
||||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
||||||
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
|
resp, err := http.Get(uploadURL)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
||||||
@@ -114,9 +103,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
assert.Equal(t, content, getURL(t, downloadURL))
|
assert.Equal(t, content, getURL(t, downloadURL))
|
||||||
}
|
}
|
||||||
|
|
||||||
// A large archive is staged as blocks and only put in order by the final block list, so
|
|
||||||
// blocks that arrive out of order must still be assembled the way the client asked.
|
|
||||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||||
@@ -130,7 +118,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
blocks[blockID] = []byte(part)
|
blocks[blockID] = []byte(part)
|
||||||
order = append(order, blockID)
|
order = append(order, blockID)
|
||||||
}
|
}
|
||||||
// Upload in an order that is not the block list order.
|
|
||||||
for _, blockID := range []string{order[2], order[0], order[1]} {
|
for _, blockID := range []string{order[2], order[0], order[1]} {
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
||||||
}
|
}
|
||||||
@@ -165,6 +152,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||||
require.Equal(t, true, saved["ok"])
|
require.Equal(t, true, saved["ok"])
|
||||||
@@ -174,8 +162,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The toolkit serialises with the proto field names; the camelCase spellings of the same
|
|
||||||
// proto JSON mapping are accepted alongside them.
|
|
||||||
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
||||||
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
||||||
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
||||||
@@ -191,9 +177,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, again["ok"])
|
assert.Equal(t, false, again["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// A key that is only a prefix of an existing one is a different entry, so the
|
|
||||||
// reservation check must be exact and not a restore-key prefix match, or the shorter
|
|
||||||
// key would be reported as existing and silently never saved.
|
|
||||||
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
||||||
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
||||||
require.Equal(t, true, reserved["ok"])
|
require.Equal(t, true, reserved["ok"])
|
||||||
@@ -219,7 +202,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The size the client declares is what Commit validates the assembled archive against.
|
|
||||||
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
||||||
@@ -230,8 +212,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// Both API versions are served from one store, so an entry written through v2 is a hit for
|
|
||||||
// a v1 client asking for the same key and version.
|
|
||||||
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
||||||
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -244,7 +224,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.NotEmpty(t, got["archiveLocation"])
|
assert.NotEmpty(t, got["archiveLocation"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The cache of one repository must stay invisible to another, as it does for the v1 API.
|
|
||||||
t.Run("another repository sees nothing", func(t *testing.T) {
|
t.Run("another repository sees nothing", func(t *testing.T) {
|
||||||
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
||||||
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
||||||
@@ -252,9 +231,28 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||||
|
clients := map[string]*http.Client{"x": testClient}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||||
|
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||||
|
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
require.Equal(t, true, created["ok"])
|
||||||
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||||
|
}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||||
|
}
|
||||||
|
for content, client := range clients {
|
||||||
|
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -291,6 +289,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -321,20 +320,12 @@ func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
|||||||
"another job's upload is not this job's to finalize")
|
"another job's upload is not this job's to finalize")
|
||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
reserveV1, err := json.Marshal(&Request{Key: "v1-made", Version: "v1", Size: int64(len(content))})
|
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
|
||||||
require.NoError(t, err)
|
|
||||||
resp, err := testClient.Post(base+"/caches", "application/json", bytes.NewReader(reserveV1))
|
|
||||||
require.NoError(t, err)
|
|
||||||
var madeByV1 struct {
|
|
||||||
CacheID uint64 `json:"cacheId"`
|
|
||||||
}
|
|
||||||
require.NoError(t, json.UnmarshalRead(resp.Body, &madeByV1))
|
|
||||||
resp.Body.Close()
|
|
||||||
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
||||||
fmt.Sprintf("%s/caches/%d", base, madeByV1.CacheID), bytes.NewReader(content))
|
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
||||||
resp, err = testClient.Do(upload)
|
resp, err := testClient.Do(upload)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
Key string `json:"key" `
|
Key string `json:"key" `
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
@@ -11,13 +13,19 @@ type Request struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Cache struct {
|
type Cache struct {
|
||||||
ID uint64 `json:"id" boltholdKey:"ID"`
|
ID uint64 `json:"id"`
|
||||||
Repo string `json:"repo" boltholdIndex:"Repo"`
|
Repo string `json:"repo"`
|
||||||
Key string `json:"key" boltholdIndex:"Key"`
|
Ref string `json:"ref,omitzero"`
|
||||||
Version string `json:"version" boltholdIndex:"Version"`
|
Key string `json:"key"`
|
||||||
Size int64 `json:"cacheSize"`
|
Version string `json:"version"`
|
||||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
Size int64 `json:"cacheSize"`
|
||||||
Complete bool `json:"complete" boltholdIndex:"Complete"`
|
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||||
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
|
Complete bool `json:"complete"`
|
||||||
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
|
UsedAt int64 `json:"usedAt"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cache *Cache) createdAtNano() int64 {
|
||||||
|
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"cmp"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -74,14 +75,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
||||||
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
||||||
if cred.InsecureTLS && target.Scheme == "https" {
|
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
|
||||||
return true
|
|
||||||
}
|
|
||||||
scheme := r.Header.Get("X-Forwarded-Proto")
|
|
||||||
if scheme == "" {
|
|
||||||
scheme = "http"
|
|
||||||
}
|
|
||||||
return scheme != target.Scheme
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeQuery(target, request string) string {
|
func mergeQuery(target, request string) string {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFrontResultsService(t *testing.T) {
|
func TestFrontResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var gotHost, gotPath, gotProto string
|
var gotHost, gotPath, gotProto string
|
||||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
// S3Options configures the bucket that stores completed cache entries.
|
// S3Options configures the bucket that stores completed cache entries.
|
||||||
@@ -47,7 +51,7 @@ type cacheStorage interface {
|
|||||||
Remove(uint64) error
|
Remove(uint64) error
|
||||||
}
|
}
|
||||||
|
|
||||||
const s3PartSize = 5 << 30 // the largest part S3 accepts
|
const s3PartSize = 64 << 20
|
||||||
|
|
||||||
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
||||||
type s3Storage struct {
|
type s3Storage struct {
|
||||||
@@ -69,6 +73,7 @@ func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
|
|||||||
}
|
}
|
||||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||||
transport = transport.Clone()
|
transport = transport.Clone()
|
||||||
|
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
|
||||||
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
||||||
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
||||||
if opts.PathStyle != nil {
|
if opts.PathStyle != nil {
|
||||||
@@ -209,26 +214,43 @@ func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// upload always goes in parts, the only way S3 takes an object over 5 GiB.
|
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
|
||||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) error {
|
if size <= s3PartSize {
|
||||||
|
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
var created struct {
|
var created struct {
|
||||||
UploadID string `xml:"UploadId"`
|
UploadID string `xml:"UploadId"`
|
||||||
}
|
}
|
||||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
if uploadErr != nil && created.UploadID != "" {
|
||||||
|
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
|
||||||
|
}
|
||||||
|
}()
|
||||||
var completion struct {
|
var completion struct {
|
||||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||||
Parts []s3Part `xml:"Part"`
|
Parts []s3Part `xml:"Part"`
|
||||||
}
|
}
|
||||||
for offset := int64(0); offset == 0 || offset < size; offset += s3PartSize {
|
partSize := max(int64(s3PartSize), (size-1)/10000+1)
|
||||||
number, length := len(completion.Parts)+1, min(s3PartSize, size-offset)
|
completion.Parts = make([]s3Part, (size-1)/partSize+1)
|
||||||
query := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {created.UploadID}}
|
workers, workerCtx := errgroup.WithContext(ctx)
|
||||||
header, err := s.call(ctx, http.MethodPut, key, query, io.NewSectionReader(file, offset, length), nil)
|
workers.SetLimit(4)
|
||||||
if err != nil {
|
for part := range completion.Parts {
|
||||||
|
workers.Go(func() (err error) {
|
||||||
|
defer common.RecoverPanic(&err)
|
||||||
|
offset := int64(part) * partSize
|
||||||
|
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
|
||||||
|
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
|
||||||
return err
|
return err
|
||||||
}
|
})
|
||||||
completion.Parts = append(completion.Parts, s3Part{number, header.Get("ETag")})
|
}
|
||||||
|
if err := workers.Wait(); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
body, err := xml.Marshal(completion)
|
body, err := xml.Marshal(completion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,40 +4,135 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
|
||||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||||
backend := s3mem.New()
|
t.Parallel()
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
var objects sync.Map
|
||||||
server := httptest.NewServer(gofakes3.New(backend).Server())
|
var uploads atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
key := strings.TrimPrefix(request.URL.Path, "/cache/")
|
||||||
|
switch {
|
||||||
|
case request.Method == http.MethodPut:
|
||||||
|
if strings.Contains(key, "/blobs/") {
|
||||||
|
uploads.Add(1)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(request.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
objects.Store(key, body)
|
||||||
|
case request.URL.Query().Has("list-type"):
|
||||||
|
_, _ = io.WriteString(writer, "<ListBucketResult>")
|
||||||
|
objects.Range(func(stored, _ any) bool {
|
||||||
|
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
_, _ = io.WriteString(writer, "</ListBucketResult>")
|
||||||
|
default:
|
||||||
|
if body, ok := objects.Load(key); ok {
|
||||||
|
_, _ = writer.Write(body.([]byte))
|
||||||
|
} else {
|
||||||
|
writer.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
||||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
|
assert.EqualValues(t, 1, uploads.Load())
|
||||||
|
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
cache, err := saver.s3.fetch(t.Context(), id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||||
require.Equal(t, true, found["ok"])
|
require.Equal(t, true, found["ok"])
|
||||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
require.NoError(t, backend.ForceDeleteBucket("cache"))
|
for _, cache := range []*Cache{
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||||
|
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||||
|
} {
|
||||||
|
require.NoError(t, saver.s3.publish(cache))
|
||||||
|
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||||
|
}
|
||||||
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
|
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||||
|
require.Equal(t, true, found["ok"])
|
||||||
|
assert.Equal(t, "deps-new", found["matched_key"])
|
||||||
|
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
|
objects.Clear()
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type s3TestTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
return transport(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, failingPart := range []string{"", "2"} {
|
||||||
|
var parts, aborts atomic.Int32
|
||||||
|
var completion struct {
|
||||||
|
Parts []s3Part `xml:"Part"`
|
||||||
|
}
|
||||||
|
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
|
||||||
|
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
|
||||||
|
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
|
||||||
|
switch {
|
||||||
|
case query.Has("uploads"):
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
|
||||||
|
case req.Method == http.MethodPut:
|
||||||
|
parts.Add(1)
|
||||||
|
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
|
||||||
|
if query.Get("partNumber") == failingPart {
|
||||||
|
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
|
||||||
|
}
|
||||||
|
case req.Method == http.MethodPost:
|
||||||
|
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
|
||||||
|
case req.Method == http.MethodDelete:
|
||||||
|
aborts.Add(1)
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
|
||||||
|
assert.EqualValues(t, 3, parts.Load())
|
||||||
|
if failingPart == "" {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
|
||||||
|
assert.Zero(t, aborts.Load())
|
||||||
|
} else {
|
||||||
|
require.ErrorContains(t, err, "InvalidPart")
|
||||||
|
assert.EqualValues(t, 1, aborts.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|||||||
@@ -32,13 +32,11 @@ func NewStorage(rootDir string) (*Storage, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Exist(id uint64) (bool, error) {
|
func (s *Storage) Exist(id uint64) (bool, error) {
|
||||||
name := s.filename(id)
|
_, err := os.Stat(s.filename(id))
|
||||||
if _, err := os.Stat(name); os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
} else if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
}
|
||||||
return true, nil
|
return err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
||||||
@@ -92,9 +90,7 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||||
defer func() {
|
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
|
||||||
_ = os.RemoveAll(s.tempDir(id))
|
|
||||||
}()
|
|
||||||
|
|
||||||
name := s.filename(id)
|
name := s.filename(id)
|
||||||
tempNames, err := s.tempNames(id)
|
tempNames, err := s.tempNames(id)
|
||||||
@@ -153,8 +149,7 @@ func assemble(name string, tempNames []string) (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||||
name := s.filename(id)
|
http.ServeFile(w, r, s.filename(id))
|
||||||
http.ServeFile(w, r, name)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
||||||
|
|||||||
+39
-38
@@ -13,14 +13,13 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileContainerResourceURL struct {
|
type FileContainerResourceURL struct {
|
||||||
@@ -67,39 +66,44 @@ func writeJSON(w http.ResponseWriter, value any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploads(router *httprouter.Router, baseDir string) {
|
func uploads(router *http.ServeMux, baseDir string) {
|
||||||
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
writeJSON(w, FileContainerResourceURL{
|
writeJSON(w, FileContainerResourceURL{
|
||||||
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
|
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
if req.Header.Get("Content-Encoding") == "gzip" {
|
if req.Header.Get("Content-Encoding") == "gzip" {
|
||||||
itemPath += gzipExtension
|
itemPath += gzipExtension
|
||||||
}
|
}
|
||||||
|
|
||||||
safeRunPath := safeResolve(baseDir, runID)
|
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
|
||||||
safePath := safeResolve(safeRunPath, itemPath)
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
var start int64
|
||||||
appendUpload := req.Header.Get("Content-Range")
|
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if start == 0 {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
}
|
}
|
||||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
if req.Body == nil {
|
if req.Body == nil {
|
||||||
panic(errors.New("no body given"))
|
panic(errors.New("no body given"))
|
||||||
}
|
}
|
||||||
@@ -114,21 +118,19 @@ func uploads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
|
||||||
writeJSON(w, ResponseMessage{
|
writeJSON(w, ResponseMessage{
|
||||||
Message: "success",
|
Message: "success",
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloads(router *httprouter.Router, baseDir string) {
|
func downloads(router *http.ServeMux, baseDir string) {
|
||||||
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
runID := req.PathValue("runId")
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, runID)
|
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
entries, err := os.ReadDir(safePath)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,13 +148,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
container := params.ByName("container")
|
container := req.PathValue("container")
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
||||||
|
|
||||||
var files []ContainerItem
|
var files []ContainerItem
|
||||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !entry.IsDir() {
|
if !entry.IsDir() {
|
||||||
rel, err := filepath.Rel(safePath, path)
|
rel, err := filepath.Rel(safePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -161,20 +166,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
|
|
||||||
// if it was upload as gzip
|
// if it was upload as gzip
|
||||||
rel = strings.TrimSuffix(rel, gzipExtension)
|
rel = strings.TrimSuffix(rel, gzipExtension)
|
||||||
path := filepath.Join(itemPath, rel)
|
|
||||||
|
|
||||||
rel = filepath.ToSlash(rel)
|
|
||||||
path = filepath.ToSlash(path)
|
|
||||||
|
|
||||||
files = append(files, ContainerItem{
|
files = append(files, ContainerItem{
|
||||||
Path: path,
|
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||||
ItemType: "file",
|
ItemType: "file",
|
||||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
|
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,10 +184,8 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
path := params.ByName("path")[1:]
|
safePath := safeResolve(baseDir, req.PathValue("path"))
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, path)
|
|
||||||
|
|
||||||
file, err := os.Open(safePath)
|
file, err := os.Open(safePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -214,11 +213,12 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
return cancel
|
return cancel
|
||||||
}
|
}
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
|
|
||||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: net.JoinHostPort(addr, port),
|
Addr: net.JoinHostPort(addr, port),
|
||||||
@@ -230,7 +230,8 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
go func() {
|
go func() {
|
||||||
logger.Infof("Start server on http://%s", server.Addr)
|
logger.Infof("Start server on http://%s", server.Addr)
|
||||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
logger.Fatal(err)
|
logger.Error(err)
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -19,16 +19,16 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFlow(t *testing.T) {
|
func TestArtifactFlow(t *testing.T) {
|
||||||
artifactPath := t.TempDir()
|
artifactPath := t.TempDir()
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
server := httptest.NewServer(router)
|
server := httptest.NewServer(router)
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -49,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
return resp.StatusCode, data
|
return resp.StatusCode, data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("no-artifacts", func(t *testing.T) {
|
||||||
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
var list NamedFileContainerResourceURLResponse
|
||||||
|
require.NoError(t, json.Unmarshal(data, &list))
|
||||||
|
require.Zero(t, list.Count)
|
||||||
|
require.Empty(t, list.Value)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("upload-and-download", func(t *testing.T) {
|
t.Run("upload-and-download", func(t *testing.T) {
|
||||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||||
|
|
||||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
@@ -82,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(data, &items))
|
require.NoError(t, json.Unmarshal(data, &items))
|
||||||
require.Len(t, items.Value, 1)
|
require.Len(t, items.Value, 1)
|
||||||
require.Equal(t, "file", items.Value[0].ItemType)
|
require.Equal(t, "file", items.Value[0].ItemType)
|
||||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
require.Equal(t, item, items.Value[0].Path)
|
||||||
|
|
||||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(stored))
|
require.Equal(t, content, string(stored))
|
||||||
})
|
})
|
||||||
@@ -96,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
t.Run("content-range", func(t *testing.T) {
|
t.Run("content-range", func(t *testing.T) {
|
||||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
|
||||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
for range 2 {
|
||||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "first-second", string(stored))
|
require.Equal(t, "first-second", string(stored))
|
||||||
|
|
||||||
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||||
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, stored)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||||
@@ -157,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("v4", func(t *testing.T) {
|
||||||
|
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||||
|
call := func(method, body string) (int, map[string]any) {
|
||||||
|
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||||
|
var out map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||||
|
return status, out
|
||||||
|
}
|
||||||
|
put := func(rawURL, body string) int {
|
||||||
|
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
download := func() (string, http.Header) {
|
||||||
|
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, signed)
|
||||||
|
resp, err := client.Get(signed["signedUrl"].(string))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(body), resp.Header
|
||||||
|
}
|
||||||
|
|
||||||
|
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
upload := created["signedUploadUrl"].(string)
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||||
|
for attempt := range 2 {
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusBadRequest, status)
|
||||||
|
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status, finalized)
|
||||||
|
id := finalized["artifactId"].(string)
|
||||||
|
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
require.Equal(t, id, refinalized["artifactId"])
|
||||||
|
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusConflict, status)
|
||||||
|
require.Equal(t, "already_exists", conflict["code"])
|
||||||
|
|
||||||
|
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
artifacts := listed["artifacts"].([]any)
|
||||||
|
require.Len(t, artifacts, 1)
|
||||||
|
artifact := artifacts[0].(map[string]any)
|
||||||
|
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||||
|
delete(artifact, "createdAt")
|
||||||
|
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||||
|
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||||
|
require.Empty(t, other["artifacts"])
|
||||||
|
|
||||||
|
body, header := download()
|
||||||
|
require.Equal(t, "hello world", body)
|
||||||
|
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusNotFound, status)
|
||||||
|
require.Equal(t, "not_found", missing["code"])
|
||||||
|
|
||||||
|
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||||
|
require.Equal(t, http.StatusCreated, status)
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
body, header = download()
|
||||||
|
require.Equal(t, "gz", body)
|
||||||
|
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
rerun := http.NewServeMux()
|
||||||
|
registerV4(rerun, artifactPath)
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||||
|
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSafeResolve(t *testing.T) {
|
func TestSafeResolve(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||||
|
dir string
|
||||||
|
mu sync.Mutex
|
||||||
|
lastID int64
|
||||||
|
created map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
zipMimeType = "application/zip"
|
||||||
|
maxTwirpBytes = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerV4(router *http.ServeMux, baseDir string) {
|
||||||
|
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||||
|
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||||
|
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
type twirpError struct {
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *twirpError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
func notFound(name string) error {
|
||||||
|
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(format string, args ...any) error {
|
||||||
|
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req Req
|
||||||
|
var resp Resp
|
||||||
|
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||||
|
if err != nil {
|
||||||
|
err = badRequest("malformed request: %v", err)
|
||||||
|
} else {
|
||||||
|
resp, err = handle(r, req)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err != nil {
|
||||||
|
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||||
|
errors.As(err, &terr)
|
||||||
|
w.WriteHeader(terr.status)
|
||||||
|
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.MarshalWrite(w, resp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||||
|
sum := sha256.Sum256([]byte(name))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) runDir(runID string) string {
|
||||||
|
return filepath.Join(s.dir, fileName(runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) artifactDir(runID, name string) string {
|
||||||
|
return filepath.Join(s.runDir(runID), fileName(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) urlDir(r *http.Request) string {
|
||||||
|
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMeta(dir string) (*monolithArtifact, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var meta monolithArtifact
|
||||||
|
return &meta, json.Unmarshal(data, &meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||||
|
data, err := json.Marshal(meta)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||||
|
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.created[dir] {
|
||||||
|
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.lastID++
|
||||||
|
if err := writeMeta(dir, &monolithArtifact{
|
||||||
|
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||||
|
DatabaseID: s.lastID,
|
||||||
|
Name: req.Name,
|
||||||
|
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||||
|
}); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.created[dir] = true
|
||||||
|
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockList struct {
|
||||||
|
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||||
|
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||||
|
var err error
|
||||||
|
switch comp := r.URL.Query().Get("comp"); comp {
|
||||||
|
case "block":
|
||||||
|
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||||
|
case "blocklist":
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||||
|
case "": // Put Blob, which docker/build-push-action uses
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||||
|
_, err := io.Copy(out, r.Body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(name string, body io.Reader) error {
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, body)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||||
|
out, err := os.CreateTemp(dir, "blob")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||||
|
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(out.Name()))
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||||
|
var list blockList
|
||||||
|
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range list.Blocks {
|
||||||
|
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendBlock(out io.Writer, name string) error {
|
||||||
|
in, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
} else if meta.Digest != "" {
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
size, err := io.Copy(hash, blob)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if size != req.Size {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||||
|
}
|
||||||
|
if req.Hash != "" && req.Hash != digest {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||||
|
}
|
||||||
|
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
meta.Size, meta.Digest = size, digest
|
||||||
|
if err := writeMeta(dir, meta); err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||||
|
run := s.runDir(req.WorkflowRunBackendID)
|
||||||
|
entries, err := os.ReadDir(run)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return listArtifactsResponse{}, err
|
||||||
|
}
|
||||||
|
var resp listArtifactsResponse
|
||||||
|
for _, entry := range entries {
|
||||||
|
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||||
|
if err != nil || meta.Digest == "" ||
|
||||||
|
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||||
|
}
|
||||||
|
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||||
|
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||||
|
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil || meta.Digest == "" {
|
||||||
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||||
|
contentType, filename := string(data), meta.Name
|
||||||
|
if !strings.Contains(contentType, "/") {
|
||||||
|
contentType = zipMimeType
|
||||||
|
}
|
||||||
|
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||||
|
filename += ".zip"
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||||
|
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return deleteArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
delete(s.created, dir)
|
||||||
|
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||||
|
now := time.Now()
|
||||||
|
claims, _ := json.Marshal(map[string]any{
|
||||||
|
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||||
|
"ac": `[{"Scope":"","Permission":3}]`,
|
||||||
|
"nbf": now.Unix(),
|
||||||
|
"exp": now.Add(lifetime).Unix(),
|
||||||
|
"jti": rand.Text(),
|
||||||
|
})
|
||||||
|
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
|
||||||
|
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||||
|
createArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
SignedUploadURL string `json:"signedUploadUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeArtifactRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
Size int64 `json:",string"`
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
finalizeArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
ArtifactID int64 `json:"artifactId,string"`
|
||||||
|
}
|
||||||
|
|
||||||
|
listArtifactsRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
NameFilter string
|
||||||
|
IDFilter int64 `json:",string"`
|
||||||
|
}
|
||||||
|
listArtifactsResponse struct {
|
||||||
|
Artifacts []monolithArtifact `json:"artifacts"`
|
||||||
|
}
|
||||||
|
monolithArtifact struct {
|
||||||
|
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||||
|
DatabaseID int64 `json:"databaseId,string"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size int64 `json:"size,string"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||||
|
Digest string `json:"digest"` // empty until the upload is finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
getSignedArtifactURLRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
getSignedArtifactURLResponse struct {
|
||||||
|
SignedURL string `json:"signedUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
createArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactResponse = finalizeArtifactResponse
|
||||||
|
)
|
||||||
@@ -4,15 +4,15 @@
|
|||||||
package common
|
package common
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDryrunContext(t *testing.T) {
|
func TestDryrunContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Dryrun(ctx) {
|
if Dryrun(ctx) {
|
||||||
t.Fatal("plain context should not be dryrun")
|
t.Fatal("plain context should not be dryrun")
|
||||||
}
|
}
|
||||||
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestJobErrorContainer(t *testing.T) {
|
func TestJobErrorContainer(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
err := errors.New("job failed")
|
err := errors.New("job failed")
|
||||||
|
|
||||||
SetJobError(ctx, err)
|
SetJobError(ctx, err)
|
||||||
@@ -40,34 +40,27 @@ func TestJobErrorContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoggerAndHookContext(t *testing.T) {
|
func TestLoggerAndHandlerContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Logger(ctx) != logrus.StandardLogger() {
|
if Logger(ctx) != logging.Default() {
|
||||||
t.Fatal("plain context should use standard logger")
|
t.Fatal("plain context should use the default logger")
|
||||||
}
|
}
|
||||||
if LoggerHook(ctx) != nil {
|
if LoggerHandler(ctx) != nil {
|
||||||
t.Fatal("plain context should not have a logger hook")
|
t.Fatal("plain context should not have a logger handler")
|
||||||
|
}
|
||||||
|
if StepIDs(ctx) != nil {
|
||||||
|
t.Fatal("plain context should not have step IDs")
|
||||||
}
|
}
|
||||||
|
|
||||||
logger := logrus.New()
|
logger := logging.New(logtest.New())
|
||||||
ctx = WithLogger(ctx, logger)
|
ctx = WithLogger(ctx, logger)
|
||||||
if Logger(ctx) != logger {
|
if Logger(ctx) != logger {
|
||||||
t.Fatal("WithLogger should set logger")
|
t.Fatal("WithLogger should set logger")
|
||||||
}
|
}
|
||||||
|
|
||||||
hook := testHook{}
|
handler := logtest.New()
|
||||||
ctx = WithLoggerHook(ctx, hook)
|
ctx = WithLoggerHandler(ctx, handler)
|
||||||
if LoggerHook(ctx) != hook {
|
if LoggerHandler(ctx) != handler {
|
||||||
t.Fatal("WithLoggerHook should set hook")
|
t.Fatal("WithLoggerHandler should set handler")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type testHook struct{}
|
|
||||||
|
|
||||||
func (testHook) Levels() []logrus.Level {
|
|
||||||
return logrus.AllLevels
|
|
||||||
}
|
|
||||||
|
|
||||||
func (testHook) Fire(*logrus.Entry) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -14,13 +14,8 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
|
|||||||
|
|
||||||
// Dryrun returns true if the current context is dryrun
|
// Dryrun returns true if the current context is dryrun
|
||||||
func Dryrun(ctx context.Context) bool {
|
func Dryrun(ctx context.Context) bool {
|
||||||
val := ctx.Value(dryrunContextKeyVal)
|
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
|
||||||
if val != nil {
|
return dryrun
|
||||||
if dryrun, ok := val.(bool); ok {
|
|
||||||
return dryrun
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithDryrun adds a value to the context for dryrun
|
// WithDryrun adds a value to the context for dryrun
|
||||||
|
|||||||
+25
-29
@@ -9,7 +9,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Executor define contract for the steps of a workflow
|
// Executor define contract for the steps of a workflow
|
||||||
@@ -21,8 +21,7 @@ type Conditional func(ctx context.Context) bool
|
|||||||
// NewInfoExecutor is an executor that logs messages
|
// NewInfoExecutor is an executor that logs messages
|
||||||
func NewInfoExecutor(format string, args ...any) Executor {
|
func NewInfoExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Infof(format, args...)
|
||||||
logger.Infof(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -30,8 +29,7 @@ func NewInfoExecutor(format string, args ...any) Executor {
|
|||||||
// NewDebugExecutor is an executor that logs messages
|
// NewDebugExecutor is an executor that logs messages
|
||||||
func NewDebugExecutor(format string, args ...any) Executor {
|
func NewDebugExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Debugf(format, args...)
|
||||||
logger.Debugf(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -39,9 +37,7 @@ func NewDebugExecutor(format string, args ...any) Executor {
|
|||||||
// NewPipelineExecutor creates a new executor from a series of other executors
|
// NewPipelineExecutor creates a new executor from a series of other executors
|
||||||
func NewPipelineExecutor(executors ...Executor) Executor {
|
func NewPipelineExecutor(executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
return func(ctx context.Context) error {
|
return NewErrorExecutor(nil)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
var rtn Executor
|
var rtn Executor
|
||||||
for _, executor := range executors {
|
for _, executor := range executors {
|
||||||
@@ -61,6 +57,13 @@ func NewErrorExecutor(err error) Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
|
||||||
|
if recovered := recover(); recovered != nil {
|
||||||
|
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
|
||||||
|
*err = fmt.Errorf("panic: %v", recovered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewParallelExecutor creates a new executor from a parallel of other executors
|
// NewParallelExecutor creates a new executor from a parallel of other executors
|
||||||
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
@@ -74,34 +77,29 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
|||||||
errs := make(chan error, len(executors))
|
errs := make(chan error, len(executors))
|
||||||
|
|
||||||
if 1 > parallel {
|
if 1 > parallel {
|
||||||
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
||||||
parallel = 1
|
parallel = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||||
|
|
||||||
for i := 0; i < parallel; i++ {
|
for workerID := range parallel {
|
||||||
go func(workerID int, work <-chan Executor, errs chan<- error) {
|
go func() {
|
||||||
log.Debugf("Worker %d started", workerID)
|
logging.Debugf("Worker %d started", workerID)
|
||||||
taskCount := 0
|
taskCount := 0
|
||||||
for executor := range work {
|
for executor := range work {
|
||||||
taskCount++
|
taskCount++
|
||||||
log.Debugf("Worker %d executing task %d", workerID, taskCount)
|
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
|
||||||
// Recover from panics in executors to avoid crashing the worker
|
// Recover from panics in executors to avoid crashing the worker
|
||||||
// goroutine which would leave the runner process hung.
|
// goroutine which would leave the runner process hung.
|
||||||
// https://gitea.com/gitea/runner/issues/371
|
// https://gitea.com/gitea/runner/issues/371
|
||||||
errs <- func() (err error) {
|
errs <- func() (err error) {
|
||||||
defer func() {
|
defer RecoverPanic(&err)
|
||||||
if r := recover(); r != nil {
|
|
||||||
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
|
|
||||||
err = fmt.Errorf("panic: %v", r)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return executor(ctx)
|
return executor(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||||
}(i, work, errs)
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range executors {
|
for i := range executors {
|
||||||
@@ -150,12 +148,7 @@ func (e Executor) If(conditional Conditional) Executor {
|
|||||||
|
|
||||||
// IfNot only runs this executor if conditional is true
|
// IfNot only runs this executor if conditional is true
|
||||||
func (e Executor) IfNot(conditional Conditional) Executor {
|
func (e Executor) IfNot(conditional Conditional) Executor {
|
||||||
return func(ctx context.Context) error {
|
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
|
||||||
if !conditional(ctx) {
|
|
||||||
return e(ctx)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IfBool only runs this executor if conditional is true
|
// IfBool only runs this executor if conditional is true
|
||||||
@@ -171,7 +164,10 @@ func (e Executor) Finally(finally Executor) Executor {
|
|||||||
err := e(ctx)
|
err := e(ctx)
|
||||||
err2 := finally(ctx)
|
err2 := finally(ctx)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+65
-106
@@ -7,8 +7,7 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -18,42 +17,25 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestNewWorkflow(t *testing.T) {
|
func TestNewWorkflow(t *testing.T) {
|
||||||
assert := assert.New(t)
|
ctx := t.Context()
|
||||||
|
require.NoError(t, NewPipelineExecutor()(ctx))
|
||||||
|
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// empty
|
|
||||||
emptyWorkflow := NewPipelineExecutor()
|
|
||||||
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// error case
|
|
||||||
errorWorkflow := NewErrorExecutor(errors.New("test error"))
|
|
||||||
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// multiple success case
|
|
||||||
runcount := 0
|
runcount := 0
|
||||||
successWorkflow := NewPipelineExecutor(
|
increment := func(_ctx context.Context) error {
|
||||||
func(ctx context.Context) error {
|
runcount++
|
||||||
runcount++
|
return nil
|
||||||
return nil
|
}
|
||||||
},
|
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
|
||||||
func(ctx context.Context) error {
|
assert.Equal(t, 2, runcount)
|
||||||
runcount++
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
assert.Equal(2, runcount)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
|
||||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
|
||||||
// find the gate already open so the last one still finishes with no partner left.
|
|
||||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||||
var counted, active, peak atomic.Int32
|
var counted, active, peak atomic.Int32
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
reached := make(chan struct{})
|
reached := make(chan struct{})
|
||||||
|
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
counted.Add(1)
|
counted.Add(1)
|
||||||
running := active.Add(1)
|
running := active.Add(1)
|
||||||
for {
|
for {
|
||||||
@@ -72,93 +54,72 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutor(t *testing.T) {
|
func TestNewParallelExecutor(t *testing.T) {
|
||||||
ctx := context.Background()
|
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
exec, count, maxActive := concurrencyProbe(2)
|
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
|
||||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
assert.Equal(t, int32(3), count.Load())
|
||||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
|
||||||
|
})
|
||||||
// parallelism below 1 falls back to a single worker
|
}
|
||||||
exec, count, maxActive = concurrencyProbe(1)
|
|
||||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
|
||||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
require.NoError(t, NewParallelExecutor(2)(ctx))
|
require.NoError(t, NewParallelExecutor(2)(ctx))
|
||||||
|
|
||||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
canceledCtx, cancel := context.WithCancel(t.Context())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
err := NewParallelExecutor(2)(canceledCtx)
|
err := NewParallelExecutor(2)(canceledCtx)
|
||||||
assert.ErrorIs(err, context.Canceled)
|
assert.ErrorIs(err, context.Canceled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorFailed(t *testing.T) {
|
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
|
||||||
assert := assert.New(t)
|
for name, tc := range map[string]struct {
|
||||||
|
canceled bool
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
parallel, tasks int
|
||||||
cancel()
|
}{
|
||||||
|
"failure": {false, 2, 5},
|
||||||
count := 0
|
"canceled": {true, 2, 5},
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
"canceled single task": {true, 1, 1},
|
||||||
count++
|
} {
|
||||||
return errors.New("fake error")
|
t.Run(name, func(t *testing.T) {
|
||||||
})
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
err := NewParallelExecutor(1, errorWorkflow)(ctx)
|
defer cancel()
|
||||||
assert.Equal(1, count)
|
if tc.canceled {
|
||||||
assert.ErrorIs(context.Canceled, err)
|
cancel()
|
||||||
}
|
}
|
||||||
|
var calls atomic.Int32
|
||||||
func TestNewParallelExecutorCanceled(t *testing.T) {
|
executors := make([]Executor, tc.tasks)
|
||||||
assert := assert.New(t)
|
for i := range executors {
|
||||||
|
executors[i] = func(_ctx context.Context) error {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
calls.Add(1)
|
||||||
cancel()
|
if i == tc.tasks/2 {
|
||||||
|
return errors.New("fake error")
|
||||||
errExpected := errors.New("fake error")
|
}
|
||||||
|
return nil
|
||||||
var count atomic.Int32
|
}
|
||||||
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return errExpected
|
|
||||||
})
|
|
||||||
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
|
|
||||||
assert.Equal(int32(3), count.Load())
|
|
||||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
|
||||||
var successCount atomic.Int32
|
|
||||||
executors := make([]Executor, 5)
|
|
||||||
for i := range executors {
|
|
||||||
executors[i] = func(ctx context.Context) error {
|
|
||||||
if i == 2 {
|
|
||||||
return errors.New("fake error")
|
|
||||||
}
|
}
|
||||||
successCount.Add(1)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
|
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
|
||||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
if tc.canceled {
|
||||||
|
require.ErrorIs(t, err, context.Canceled)
|
||||||
|
} else {
|
||||||
|
require.EqualError(t, err, "fake error")
|
||||||
|
}
|
||||||
|
assert.Equal(t, int32(tc.tasks), calls.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
var calls []string
|
var calls []string
|
||||||
record := func(name string) Executor {
|
record := func(name string) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
calls = append(calls, name)
|
calls = append(calls, name)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -170,19 +131,17 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
||||||
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
||||||
|
|
||||||
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
|
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||||
if !reflect.DeepEqual(calls, want) {
|
|
||||||
t.Fatalf("calls = %v, want %v", calls, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
mainErr := errors.New("main failed")
|
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
|
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||||
require.Error(t, err)
|
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||||
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
|
require.ErrorIs(t, err, finalErr)
|
||||||
t.Fatalf("finally error = %q, want both cleanup and original error", err)
|
if mainErr != nil {
|
||||||
|
require.ErrorIs(t, err, mainErr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"math/rand/v2"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
var actionFetches sync.Map
|
||||||
|
|
||||||
|
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||||
|
|
||||||
|
type treeFile struct {
|
||||||
|
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetch struct {
|
||||||
|
done chan struct{}
|
||||||
|
dir, sha string
|
||||||
|
err error
|
||||||
|
retry bool // the leader was cancelled or panicked, so followers fetch themselves
|
||||||
|
}
|
||||||
|
|
||||||
|
var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/runner backs off
|
||||||
|
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
||||||
|
return min(max(time.Duration(seconds)*time.Second, 10*time.Second), 10*time.Minute)
|
||||||
|
}
|
||||||
|
return time.Duration(10+rand.IntN(21)) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) { // input.Dir is one clone URL and ref's, holding a directory per commit
|
||||||
|
if !reflect.ValueOf(input).Comparable() {
|
||||||
|
return fetchAction(ctx, input)
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
call := &actionFetch{done: make(chan struct{}), retry: true}
|
||||||
|
previous, loaded := actionFetches.LoadOrStore(input, call)
|
||||||
|
if !loaded {
|
||||||
|
func() {
|
||||||
|
defer func() { actionFetches.Delete(input); close(call.done) }()
|
||||||
|
call.dir, call.sha, call.err = fetchAction(ctx, input)
|
||||||
|
call.retry = ctx.Err() != nil
|
||||||
|
}()
|
||||||
|
if call.retry {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return call.dir, call.sha, call.err
|
||||||
|
}
|
||||||
|
call, _ = previous.(*actionFetch)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-call.done:
|
||||||
|
if !call.retry && ctx.Err() == nil {
|
||||||
|
return call.dir, call.sha, call.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", "", ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) {
|
||||||
|
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.MkdirAll(input.Dir, 0o755); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
tree := func(sha string) string { return filepath.Join(input.Dir, sha) }
|
||||||
|
record := filepath.Join(input.Dir, "ref")
|
||||||
|
data, _ := os.ReadFile(record)
|
||||||
|
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||||
|
switch {
|
||||||
|
case !isFullObjectID(sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||||
|
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||||
|
case !treeIntact(input.Dir, sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||||
|
return tree(sha), sha, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
newSha, newEtag, err := downloadArchive(ctx, input, sha, etag)
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
common.Logger(ctx).Debugf("Cloning %s@%s: %v", input.URL, input.Ref, err)
|
||||||
|
newEtag = ""
|
||||||
|
newSha, err = exportClone(ctx, input)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if newSha != sha || newEtag != etag {
|
||||||
|
if err := os.WriteFile(record, []byte(newSha+"\n"+newEtag), 0o600); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tree(newSha), newSha, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadArchive(ctx context.Context, input NewGitCloneExecutorInput, sha, etag string) (string, string, error) {
|
||||||
|
rewritten, err := gitOutput(ctx, "", "ls-remote", "--get-url", input.URL)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
source, err := url.Parse(rewritten)
|
||||||
|
if err != nil || source.Scheme != "http" && source.Scheme != "https" || hexRefRegex.MatchString(input.Ref) && !isFullObjectID(input.Ref) { // only git tells a short commit from a branch
|
||||||
|
return "", "", errors.New("no archive")
|
||||||
|
}
|
||||||
|
archive := source.JoinPath("archive", input.Ref+".tar.gz")
|
||||||
|
if source.Host == "github.com" {
|
||||||
|
source.Host = "codeload.github.com"
|
||||||
|
archive = source.JoinPath("tar.gz", input.Ref)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: input.Transport, Timeout: 5 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
||||||
|
if len(via) >= 10 {
|
||||||
|
return errors.New("stopped after 10 redirects")
|
||||||
|
}
|
||||||
|
if request.URL.Host != via[0].URL.Host || request.URL.Scheme != via[0].URL.Scheme { // the token stays on its origin
|
||||||
|
request.Header.Del("Authorization")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
newSha, newEtag, retryAfter, err := downloadArchiveOnce(ctx, client, archive.String(), input.Token, sha, etag, input.Dir)
|
||||||
|
if retryAfter == 0 || attempt == 3 || ctx.Err() != nil {
|
||||||
|
return newSha, newEtag, err
|
||||||
|
}
|
||||||
|
common.Logger(ctx).Warnf("Failed to download action '%s'. Error: %v. Back off %s before retry.", archive.Redacted(), err, retryAfter)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", "", ctx.Err()
|
||||||
|
case <-time.After(retryAfter):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, token, sha, etag, dir string) (string, string, time.Duration, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archive, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", 0, err
|
||||||
|
}
|
||||||
|
if etag != "" {
|
||||||
|
request.Header.Set("If-None-Match", etag)
|
||||||
|
}
|
||||||
|
if token != "" {
|
||||||
|
request.SetBasicAuth("token", token)
|
||||||
|
}
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
var opErr *net.OpError
|
||||||
|
if errors.As(err, &opErr) && opErr.Op == "dial" {
|
||||||
|
return "", "", 0, err
|
||||||
|
}
|
||||||
|
return "", "", archiveRetryDelay(""), err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
switch status := response.StatusCode; {
|
||||||
|
case status == http.StatusNotModified && sha != "":
|
||||||
|
return sha, etag, 0, nil
|
||||||
|
case status == http.StatusOK:
|
||||||
|
newEtag := response.Header.Get("ETag")
|
||||||
|
gz, err := gzip.NewReader(response.Body)
|
||||||
|
if err == nil {
|
||||||
|
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||||
|
}
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||||
|
return "", "", archiveRetryDelay(""), err
|
||||||
|
}
|
||||||
|
return sha, newEtag, 0, err
|
||||||
|
case status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError:
|
||||||
|
return "", "", archiveRetryDelay(response.Header.Get("Retry-After")), errors.New(response.Status)
|
||||||
|
default:
|
||||||
|
return "", "", 0, errors.New(response.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, error) {
|
||||||
|
clone := input
|
||||||
|
clone.Dir = filepath.Join(input.Dir, "git")
|
||||||
|
sha, _, err := resolveClone(ctx, clone)
|
||||||
|
if err != nil || treeIntact(input.Dir, sha) {
|
||||||
|
return sha, err
|
||||||
|
}
|
||||||
|
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||||
|
return sha, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||||
|
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(temp)
|
||||||
|
root, err := os.OpenRoot(temp)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
|
||||||
|
sha := ""
|
||||||
|
sizes := map[string]int64{}
|
||||||
|
reader := tar.NewReader(archive)
|
||||||
|
for {
|
||||||
|
header, err := reader.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||||
|
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := header.Name
|
||||||
|
if strip {
|
||||||
|
_, name, _ = strings.Cut(name, "/")
|
||||||
|
}
|
||||||
|
if name = strings.TrimSuffix(name, "/"); name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch header.Typeflag {
|
||||||
|
case tar.TypeDir:
|
||||||
|
err = root.MkdirAll(name, 0o755)
|
||||||
|
case tar.TypeReg:
|
||||||
|
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||||
|
case tar.TypeSymlink:
|
||||||
|
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||||
|
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isFullObjectID(sha) {
|
||||||
|
return "", errors.New("the archive names no commit")
|
||||||
|
}
|
||||||
|
if err := root.Close(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||||
|
|
||||||
|
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||||
|
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||||
|
files := make([]treeFile, 0, len(sizes))
|
||||||
|
for name, size := range sizes {
|
||||||
|
files = append(files, treeFile{[]byte(name), size})
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tree := filepath.Join(dir, sha)
|
||||||
|
if !exists(tree) {
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||||
|
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||||
|
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||||
|
func treeIntact(dir, sha string) bool {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||||
|
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var files []treeFile
|
||||||
|
if json.Unmarshal(data, &files) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||||
|
perm := os.FileMode(0o644)
|
||||||
|
if mode&0o111 != 0 {
|
||||||
|
perm = 0o755
|
||||||
|
}
|
||||||
|
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
n, err := io.Copy(file, content)
|
||||||
|
return n, errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func exists(name string) bool {
|
||||||
|
_, err := os.Stat(name)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,412 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cgi"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFetchAction(t *testing.T) {
|
||||||
|
original := archiveRetryDelay
|
||||||
|
archiveRetryDelay = func(_retryAfter string) time.Duration { return time.Nanosecond }
|
||||||
|
t.Cleanup(func() { archiveRetryDelay = original })
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
remote := filepath.Join(root, "org", "action")
|
||||||
|
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||||
|
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||||
|
expectedSha := gitRevParse(t, remote, "main")
|
||||||
|
|
||||||
|
gitPath, err := exec.LookPath("git")
|
||||||
|
require.NoError(t, err)
|
||||||
|
backend := &cgi.Handler{Path: gitPath, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
|
||||||
|
var mu sync.Mutex
|
||||||
|
var archiveRequests []http.Header
|
||||||
|
var mirror *httptest.Server
|
||||||
|
archiveCache := map[string][]byte{}
|
||||||
|
requests, archives, failures := 0, "", 0
|
||||||
|
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
requests++
|
||||||
|
ref, ok := strings.CutPrefix(request.URL.Path, "/org/action/archive/")
|
||||||
|
if !ok {
|
||||||
|
backend.ServeHTTP(writer, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
archiveRequests = append(archiveRequests, request.Header.Clone())
|
||||||
|
switch {
|
||||||
|
case failures > 0:
|
||||||
|
failures--
|
||||||
|
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
case archives == "none":
|
||||||
|
http.NotFound(writer, request)
|
||||||
|
return
|
||||||
|
case archives == "unreachable":
|
||||||
|
http.Redirect(writer, request, "http://127.0.0.1:1"+request.URL.Path, http.StatusFound)
|
||||||
|
return
|
||||||
|
case archives == "redirect" && request.Host != mirror.Listener.Addr().String():
|
||||||
|
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||||
|
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||||
|
if request.Header.Get("If-None-Match") == etag {
|
||||||
|
writer.WriteHeader(http.StatusNotModified)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
args := []string{"archive", "--format=tar.gz", sha}
|
||||||
|
if archives != "unprefixed" {
|
||||||
|
args = append(args, "--prefix=action/")
|
||||||
|
}
|
||||||
|
archiveKey := strings.Join(args, "\x00")
|
||||||
|
archive, ok := archiveCache[archiveKey]
|
||||||
|
if !ok {
|
||||||
|
archive, _ = runGit(request.Context(), remote, nil, args...)
|
||||||
|
archiveCache[archiveKey] = archive
|
||||||
|
}
|
||||||
|
writer.Header().Set("ETag", etag)
|
||||||
|
_, _ = writer.Write(archive)
|
||||||
|
})
|
||||||
|
server := httptest.NewServer(handler)
|
||||||
|
mirror = httptest.NewServer(handler)
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
t.Cleanup(mirror.Close)
|
||||||
|
serve := func(mode string, failing int) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
requests, archives, failures, archiveRequests = 0, mode, failing, nil
|
||||||
|
}
|
||||||
|
served := func() (int, []http.Header) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return requests, append([]http.Header(nil), archiveRequests...)
|
||||||
|
}
|
||||||
|
newInput := func(ref string) NewGitCloneExecutorInput {
|
||||||
|
return NewGitCloneExecutorInput{URL: server.URL + "/org/action", Ref: ref, Dir: t.TempDir(), Token: "secret"}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput("main")
|
||||||
|
started, release := make(chan struct{}), make(chan struct{})
|
||||||
|
releaseFetches := sync.OnceFunc(func() { close(release) })
|
||||||
|
t.Cleanup(releaseFetches)
|
||||||
|
var once sync.Once
|
||||||
|
var panicked atomic.Bool
|
||||||
|
transport := actionFetchTransport(func(request *http.Request) (*http.Response, error) {
|
||||||
|
once.Do(func() { close(started) })
|
||||||
|
select {
|
||||||
|
case <-request.Context().Done():
|
||||||
|
return nil, request.Context().Err()
|
||||||
|
case <-release:
|
||||||
|
}
|
||||||
|
if scenario == "leader panics" && panicked.CompareAndSwap(false, true) {
|
||||||
|
panic("broken archive")
|
||||||
|
}
|
||||||
|
return http.DefaultTransport.RoundTrip(request)
|
||||||
|
})
|
||||||
|
input.Transport = &transport
|
||||||
|
leaderCtx, cancelLeader := context.WithCancel(t.Context())
|
||||||
|
defer cancelLeader()
|
||||||
|
waiting := &actionFetchContext{Context: t.Context(), waiting: make(chan struct{})}
|
||||||
|
followerInput := input
|
||||||
|
expectedRequests := 1
|
||||||
|
if scenario == "different token" {
|
||||||
|
followerInput.Token, expectedRequests = "other", 2
|
||||||
|
}
|
||||||
|
leaderDone, followerDone := make(chan error, 1), make(chan error, 1)
|
||||||
|
fetch := func(ctx context.Context, input NewGitCloneExecutorInput, done chan<- error) {
|
||||||
|
var err error
|
||||||
|
defer func() { done <- err }()
|
||||||
|
defer common.RecoverPanic(&err)
|
||||||
|
var tree, sha string
|
||||||
|
if tree, sha, err = FetchAction(ctx, input); err == nil {
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go fetch(leaderCtx, input, leaderDone)
|
||||||
|
<-started
|
||||||
|
go fetch(waiting, followerInput, followerDone)
|
||||||
|
<-waiting.waiting
|
||||||
|
if scenario == "leader cancelled" {
|
||||||
|
cancelLeader()
|
||||||
|
require.ErrorIs(t, <-leaderDone, context.Canceled)
|
||||||
|
}
|
||||||
|
releaseFetches()
|
||||||
|
switch scenario {
|
||||||
|
case "leader cancelled":
|
||||||
|
case "leader panics":
|
||||||
|
require.ErrorContains(t, <-leaderDone, "panic: broken archive")
|
||||||
|
default:
|
||||||
|
require.NoError(t, <-leaderDone)
|
||||||
|
}
|
||||||
|
require.NoError(t, <-followerDone)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, expectedRequests)
|
||||||
|
if scenario == "different token" {
|
||||||
|
assert.NotEqual(t, headers[0].Get("Authorization"), headers[1].Get("Authorization"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads, revalidates and follows a moved tag", func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput("v1")
|
||||||
|
tree, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
|
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||||
|
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||||
|
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "action.yml", link)
|
||||||
|
|
||||||
|
again, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tree, again)
|
||||||
|
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||||
|
expectedSha = gitRevParse(t, remote, "v1")
|
||||||
|
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, movedSha)
|
||||||
|
assert.NotEqual(t, tree, moved)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
|
||||||
|
input.OfflineMode = true
|
||||||
|
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, movedSha, offlineSha)
|
||||||
|
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||||
|
reused, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, moved, reused)
|
||||||
|
|
||||||
|
requests, headers := served()
|
||||||
|
assert.Equal(t, 4, requests)
|
||||||
|
require.Len(t, headers, 4)
|
||||||
|
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||||
|
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||||
|
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||||
|
assert.Equal(t, headers[1].Get("If-None-Match"), headers[2].Get("If-None-Match"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
sha := expectedSha
|
||||||
|
input := newInput(sha)
|
||||||
|
for range 2 {
|
||||||
|
tree, got, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, sha, got)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
}
|
||||||
|
_, _, err := FetchAction(t.Context(), newInput(sha[:7]))
|
||||||
|
require.ErrorIs(t, err, ErrShortRef)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||||
|
sha := gitRevParse(t, remote, "main")
|
||||||
|
for _, mode := range []string{"prefixed", "none"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
for _, damage := range []func() error{
|
||||||
|
func() error { return os.Truncate(yml, 4) },
|
||||||
|
func() error { return os.Remove(tree + treeSizes) },
|
||||||
|
} {
|
||||||
|
require.NoError(t, damage(), mode)
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, "name: action\n", string(body), mode)
|
||||||
|
}
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 3, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.Truncate(yml, 4))
|
||||||
|
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||||
|
require.NoError(t, err)
|
||||||
|
release := sync.OnceFunc(unlock)
|
||||||
|
defer release()
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||||
|
close(waiting)
|
||||||
|
return AcquireCloneLock(ctx, dir)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||||
|
case <-waiting:
|
||||||
|
}
|
||||||
|
info, err := os.Stat(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||||
|
release()
|
||||||
|
require.NoError(t, <-done)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "name: action\n", string(body))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||||
|
serve("redirect", 2)
|
||||||
|
input := newInput("main")
|
||||||
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
_, headers := served()
|
||||||
|
require.Len(t, headers, 4)
|
||||||
|
assert.NotEmpty(t, headers[2].Get("Authorization"))
|
||||||
|
assert.Empty(t, headers[3].Get("Authorization"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("keeps an unprefixed archive whole, downloads where insteadOf points and exports a clone without an archive or from an unreachable host", func(t *testing.T) {
|
||||||
|
for _, mode := range []string{"unprefixed", "none", "unreachable"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, expectedSha, sha, mode)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||||
|
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
config := filepath.Join(t.TempDir(), "gitconfig")
|
||||||
|
require.NoError(t, os.WriteFile(config, []byte("[url \""+server.URL+"/org/\"]\n\tinsteadOf = https://example.invalid/org/\n"), 0o644))
|
||||||
|
t.Setenv("GIT_CONFIG_GLOBAL", config)
|
||||||
|
input := newInput("main")
|
||||||
|
input.URL = "https://example.invalid/org/action"
|
||||||
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetchTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (transport actionFetchTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
return transport(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetchContext struct {
|
||||||
|
context.Context
|
||||||
|
waiting chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ctx *actionFetchContext) Done() <-chan struct{} {
|
||||||
|
ctx.once.Do(func() { close(ctx.waiting) })
|
||||||
|
return ctx.Context.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||||
|
outside := t.TempDir()
|
||||||
|
for name, entries := range map[string][]*tar.Header{
|
||||||
|
"parent path": {{Name: "../escape", Typeflag: tar.TypeReg}},
|
||||||
|
"through a symlink": {{Name: "out", Typeflag: tar.TypeSymlink, Linkname: outside}, {Name: "out/escape", Typeflag: tar.TypeReg}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||||
|
for _, entry := range entries {
|
||||||
|
require.NoError(t, writer.WriteHeader(entry))
|
||||||
|
}
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||||
|
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||||
|
}
|
||||||
|
sha := strings.Repeat("a", 40)
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||||
|
_, err := writer.Write([]byte("x"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||||
|
assert.True(t, treeIntact(dir, sha))
|
||||||
|
}
|
||||||
+145
-68
@@ -10,8 +10,12 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -21,16 +25,13 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
|
||||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
|
||||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||||
|
|
||||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||||
@@ -44,12 +45,29 @@ var (
|
|||||||
ErrShortRef = errors.New("short SHA references are not supported")
|
ErrShortRef = errors.New("short SHA references are not supported")
|
||||||
)
|
)
|
||||||
|
|
||||||
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
|
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
|
||||||
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
|
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
|
||||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
dir = filepath.Clean(dir)
|
||||||
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
|
if runtime.GOOS == "plan9" {
|
||||||
func AcquireCloneLock(dir string) func() {
|
return cloneLocks.Lock(dir), nil
|
||||||
return cloneLocks.Lock(dir)
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
release, err := lock.TryLock(dir)
|
||||||
|
if err == nil {
|
||||||
|
return func() { _ = release() }, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, lock.ErrLocked) {
|
||||||
|
return cloneLocks.Lock(dir), nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Error struct {
|
type Error struct {
|
||||||
@@ -62,25 +80,25 @@ func (e *Error) Unwrap() error { return e.err }
|
|||||||
func (e *Error) Commit() string { return e.commit }
|
func (e *Error) Commit() string { return e.commit }
|
||||||
|
|
||||||
// FindGitRevision get the current git revision
|
// FindGitRevision get the current git revision
|
||||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
func FindGitRevision(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||||
return "", "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if len(sha) < 7 {
|
if len(sha) < 7 {
|
||||||
return "", "", errors.New("HEAD could not be resolved")
|
return "", errors.New("HEAD could not be resolved")
|
||||||
}
|
}
|
||||||
logger.Debugf("Found revision: %s", sha)
|
logger.Debugf("Found revision: %s", sha)
|
||||||
return sha[:7], sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindGitRef get the current git ref
|
// FindGitRef get the current git ref
|
||||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
logger.Debugf("Loading revision from git directory")
|
logger.Debugf("Loading revision from git directory")
|
||||||
_, revision, err := FindGitRevision(ctx, file)
|
revision, err := FindGitRevision(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -109,13 +127,12 @@ func FindGitRef(ctx context.Context, file string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FindGithubRepo get the repo
|
// FindGithubRepo get the repo
|
||||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
func FindGithubRepo(ctx context.Context, file string) (string, error) {
|
||||||
url, err := findGitRemoteURL(ctx, file)
|
url, err := findGitRemoteURL(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
_, slug := findGitSlug(url, githubInstance)
|
return findGitSlug(url)
|
||||||
return slug, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||||
@@ -130,25 +147,25 @@ func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
|||||||
return url, nil
|
return url, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitSlug(url, githubInstance string) (string, string) {
|
func findGitSlug(remoteURL string) (string, error) {
|
||||||
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
|
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
|
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if githubInstance != "github.com" {
|
|
||||||
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
|
|
||||||
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
|
|
||||||
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return "", url
|
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
|
||||||
|
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(remoteURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("cannot parse the origin remote URL")
|
||||||
|
}
|
||||||
|
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
|
||||||
|
owner = path.Base(owner)
|
||||||
|
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
|
||||||
|
return "", errors.New("cannot determine owner and repository from the origin remote URL")
|
||||||
|
}
|
||||||
|
return owner + "/" + repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
||||||
@@ -163,17 +180,14 @@ type NewGitCloneExecutorInput struct {
|
|||||||
// 0 for full clone.
|
// 0 for full clone.
|
||||||
Depth int
|
Depth int
|
||||||
|
|
||||||
// Quiet drops the informational clone line to debug level, for callers that log their own
|
|
||||||
// download summary (the setup section's action report).
|
|
||||||
Quiet bool
|
|
||||||
|
|
||||||
// For Gitea
|
// For Gitea
|
||||||
InsecureSkipTLS bool
|
InsecureSkipTLS bool
|
||||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||||
|
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneIfRequired reports whether an existing local clone was reused.
|
// CloneIfRequired reports whether an existing local clone was reused.
|
||||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) {
|
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
|
||||||
var origin string
|
var origin string
|
||||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -182,6 +196,9 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
if err == nil && origin == input.URL {
|
if err == nil && origin == input.URL {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
|
||||||
|
return false, ctxErr
|
||||||
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||||
@@ -203,32 +220,18 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
if input.Quiet {
|
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||||
logger.Debugf("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
} else {
|
|
||||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
}
|
|
||||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||||
defer AcquireCloneLock(input.Dir)()
|
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
reused, err := CloneIfRequired(ctx, input, logger)
|
resolved, reused, err := resolveClone(ctx, input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
|
||||||
if err := fetch(ctx, input); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||||
}
|
}
|
||||||
@@ -242,6 +245,28 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
|
||||||
|
logger := common.Logger(ctx)
|
||||||
|
reused, err := CloneIfRequired(ctx, input, logger)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||||
|
if err := fetch(ctx, input); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||||
|
}
|
||||||
|
return resolved, reused, err
|
||||||
|
}
|
||||||
|
|
||||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||||
@@ -282,7 +307,7 @@ func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
args := []string{"fetch", "--force", "--prune"}
|
args := []string{"fetch", "--force", "--no-prune"}
|
||||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||||
args = append(args, "--depth", "1", "--no-tags")
|
args = append(args, "--depth", "1", "--no-tags")
|
||||||
}
|
}
|
||||||
@@ -296,9 +321,18 @@ func cachedRef(ref string) string {
|
|||||||
|
|
||||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||||
if !isFullObjectID(ref) {
|
if !isFullObjectID(ref) {
|
||||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
if strings.ContainsAny(ref, "\x00\n") {
|
||||||
return sha, nil
|
for _, revision := range revisions {
|
||||||
|
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||||
|
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||||
|
if isFullObjectID(sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -364,7 +398,49 @@ func ConfigEnv(config ...string) map[string]string {
|
|||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
||||||
|
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var files, repos []string
|
||||||
|
for entry := range strings.SplitSeq(string(listed), "\x00") {
|
||||||
|
tag, file, ok := strings.Cut(entry, " ")
|
||||||
|
if tag != "?" {
|
||||||
|
var meta string
|
||||||
|
meta, file, ok = strings.Cut(file, "\t")
|
||||||
|
if strings.HasPrefix(meta, "160000 ") {
|
||||||
|
file += "/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
|
||||||
|
repos = append(repos, repo)
|
||||||
|
} else if ok {
|
||||||
|
files = append(files, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(repos)
|
||||||
|
for _, repo := range slices.Compact(repos) {
|
||||||
|
files = append(files, repo)
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
|
||||||
|
continue // git lists an uninitialized submodule as "./" from inside it
|
||||||
|
}
|
||||||
|
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, file := range nested {
|
||||||
|
files = append(files, repo+"/"+file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||||
|
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
@@ -376,6 +452,7 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
|||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
|
cmd.Stdin = stdin
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||||
name, _, _ := strings.Cut(entry, "=")
|
name, _, _ := strings.Cut(entry, "=")
|
||||||
return slices.Contains(LocalEnvVars, name)
|
return slices.Contains(LocalEnvVars, name)
|
||||||
|
|||||||
+214
-251
@@ -18,40 +18,70 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
logrustest "github.com/sirupsen/logrus/hooks/test"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFindGitSlug(t *testing.T) {
|
var gitFixtureDir string
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
slugTests := []struct {
|
func TestMain(m *testing.M) {
|
||||||
url string // input
|
var err error
|
||||||
provider string // expected result
|
gitFixtureDir, err = os.MkdirTemp("", "runner-git-fixture-")
|
||||||
slug string // expected result
|
if err != nil {
|
||||||
}{
|
fmt.Fprintln(os.Stderr, err)
|
||||||
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "CodeCommit", "my-repo-name"},
|
os.Exit(1)
|
||||||
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "CodeCommit", "my-repo"},
|
|
||||||
{"[email protected]:nektos/act.git", "GitHub", "nektos/act"},
|
|
||||||
{"[email protected]:nektos/act", "GitHub", "nektos/act"},
|
|
||||||
{"https://github.com/nektos/act.git", "GitHub", "nektos/act"},
|
|
||||||
{"http://github.com/nektos/act.git", "GitHub", "nektos/act"},
|
|
||||||
{"https://github.com/nektos/act", "GitHub", "nektos/act"},
|
|
||||||
{"http://github.com/nektos/act", "GitHub", "nektos/act"},
|
|
||||||
{"git+ssh://[email protected]/owner/repo.git", "GitHub", "owner/repo"},
|
|
||||||
{"http://myotherrepo.com/act.git", "", "http://myotherrepo.com/act.git"},
|
|
||||||
}
|
}
|
||||||
|
code := 1
|
||||||
|
if err = gitCmd("init", "--quiet", "--template=", "--bare", "--initial-branch=main", filepath.Join(gitFixtureDir, "remote")); err == nil {
|
||||||
|
err = gitCmd("init", "--quiet", "--template=", "--initial-branch=main", filepath.Join(gitFixtureDir, "work"))
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
code = m.Run()
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(gitFixtureDir); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
code = 1
|
||||||
|
}
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
for _, tt := range slugTests {
|
func TestFindGitSlug(t *testing.T) {
|
||||||
provider, slug := findGitSlug(tt.url, "github.com")
|
for _, testcase := range []struct {
|
||||||
assert.Equal(tt.provider, provider)
|
remoteURL string
|
||||||
assert.Equal(tt.slug, slug)
|
slug string
|
||||||
|
}{
|
||||||
|
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "my-repo-name"},
|
||||||
|
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "my-repo"},
|
||||||
|
{"[email protected]:nektos/act.git", "nektos/act"},
|
||||||
|
{"[email protected]:nektos/act", "nektos/act"},
|
||||||
|
{"https://github.com/nektos/act.git", "nektos/act"},
|
||||||
|
{"http://github.com/nektos/act.git", "nektos/act"},
|
||||||
|
{"https://github.com/nektos/act", "nektos/act"},
|
||||||
|
{"http://github.com/nektos/act", "nektos/act"},
|
||||||
|
{"git+ssh://[email protected]/owner/repo.git", "owner/repo"},
|
||||||
|
{"https://example.com:3000/gitea/owner/repo.git/", "owner/repo"},
|
||||||
|
{"ssh://[email protected]:2222/gitea/owner/repo.git", "owner/repo"},
|
||||||
|
{"example.com:owner/repo.git", "owner/repo"},
|
||||||
|
{"http://myotherrepo.com/act.git", ""},
|
||||||
|
{"https://user:[email protected]/act.git", ""},
|
||||||
|
{"/tmp/owner/repo.git", ""},
|
||||||
|
{"./owner/repo:branch", ""},
|
||||||
|
} {
|
||||||
|
t.Run(testcase.remoteURL, func(t *testing.T) {
|
||||||
|
slug, err := findGitSlug(testcase.remoteURL)
|
||||||
|
assert.Equal(t, testcase.slug == "", err != nil)
|
||||||
|
assert.NotContains(t, fmt.Sprint(err), "secret")
|
||||||
|
assert.Equal(t, testcase.slug, slug)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,52 +92,30 @@ func TestErrorWrapsCommitAndCause(t *testing.T) {
|
|||||||
require.Equal(t, "abc123", err.Commit())
|
require.Equal(t, "abc123", err.Commit())
|
||||||
}
|
}
|
||||||
|
|
||||||
func cleanGitHooks(dir string) error {
|
|
||||||
hooksDir := filepath.Join(dir, ".git", "hooks")
|
|
||||||
files, err := os.ReadDir(hooksDir)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, f := range files {
|
|
||||||
if f.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
relName := filepath.Join(hooksDir, f.Name())
|
|
||||||
if err := os.Remove(relName); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
basedir := t.TempDir()
|
basedir := t.TempDir()
|
||||||
const remoteURL = "https://github.com/owner/repo.git"
|
const remoteURL = "https://github.com/owner/repo.git"
|
||||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||||
require.NoError(t, cleanGitHooks(basedir))
|
require.NoError(t, os.RemoveAll(filepath.Join(basedir, ".git", "hooks")))
|
||||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||||
|
|
||||||
shortSHA, sha, err := FindGitRevision(context.Background(), basedir)
|
sha, err := FindGitRevision(t.Context(), basedir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
||||||
require.Len(t, sha, 64)
|
require.Len(t, sha, 64)
|
||||||
require.Equal(t, sha[:7], shortSHA)
|
|
||||||
|
|
||||||
ref, err := FindGitRef(context.Background(), basedir)
|
ref, err := FindGitRef(t.Context(), basedir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "refs/heads/main", ref)
|
require.Equal(t, "refs/heads/main", ref)
|
||||||
|
|
||||||
url, err := findGitRemoteURL(context.Background(), basedir)
|
url, err := findGitRemoteURL(t.Context(), basedir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, remoteURL, url)
|
require.Equal(t, remoteURL, url)
|
||||||
|
|
||||||
slug, err := FindGithubRepo(context.Background(), basedir, "github.com")
|
slug, err := FindGithubRepo(t.Context(), basedir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "owner/repo", slug)
|
require.Equal(t, "owner/repo", slug)
|
||||||
}
|
}
|
||||||
@@ -117,89 +125,65 @@ func TestGitFindRef(t *testing.T) {
|
|||||||
basedir := t.TempDir()
|
basedir := t.TempDir()
|
||||||
|
|
||||||
for name, tt := range map[string]struct {
|
for name, tt := range map[string]struct {
|
||||||
Prepare func(t *testing.T, dir string)
|
commands [][]string
|
||||||
Assert func(t *testing.T, ref string, err error)
|
ref string
|
||||||
}{
|
}{
|
||||||
"new_repo": {
|
"new_repo": {},
|
||||||
Prepare: func(t *testing.T, dir string) {},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.Error(t, err)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"new_repo_with_commit": {
|
"new_repo_with_commit": {
|
||||||
Prepare: func(t *testing.T, dir string) {
|
commands: [][]string{{"commit", "--allow-empty", "-m", "msg"}},
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
ref: "refs/heads/master",
|
||||||
},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "refs/heads/master", ref)
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"current_head_is_tag": {
|
"current_head_is_tag": {
|
||||||
Prepare: func(t *testing.T, dir string) {
|
commands: [][]string{
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "commit msg"))
|
{"commit", "--allow-empty", "-m", "commit msg"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.2.3"))
|
{"tag", "v1.2.3"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "checkout", "v1.2.3"))
|
{"checkout", "v1.2.3"},
|
||||||
},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "refs/tags/v1.2.3", ref)
|
|
||||||
},
|
},
|
||||||
|
ref: "refs/tags/v1.2.3",
|
||||||
},
|
},
|
||||||
"current_head_is_same_as_tag": {
|
"current_head_is_same_as_tag": {
|
||||||
Prepare: func(t *testing.T, dir string) {
|
commands: [][]string{
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "1.4.2 release"))
|
{"commit", "--allow-empty", "-m", "1.4.2 release"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
{"tag", "v1.4.2"},
|
||||||
},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "refs/tags/v1.4.2", ref)
|
|
||||||
},
|
},
|
||||||
|
ref: "refs/tags/v1.4.2",
|
||||||
},
|
},
|
||||||
"current_head_is_not_tag": {
|
"current_head_is_not_tag": {
|
||||||
Prepare: func(t *testing.T, dir string) {
|
commands: [][]string{
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
{"commit", "--allow-empty", "-m", "msg"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
{"tag", "v1.4.2"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg2"))
|
{"commit", "--allow-empty", "-m", "msg2"},
|
||||||
},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "refs/heads/master", ref)
|
|
||||||
},
|
},
|
||||||
|
ref: "refs/heads/master",
|
||||||
},
|
},
|
||||||
"current_head_is_another_branch": {
|
"current_head_is_another_branch": {
|
||||||
Prepare: func(t *testing.T, dir string) {
|
commands: [][]string{
|
||||||
require.NoError(t, gitCmd("-C", dir, "checkout", "-b", "mybranch"))
|
{"checkout", "-b", "mybranch"},
|
||||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
{"commit", "--allow-empty", "-m", "msg"},
|
||||||
},
|
|
||||||
Assert: func(t *testing.T, ref string, err error) {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "refs/heads/mybranch", ref)
|
|
||||||
},
|
},
|
||||||
|
ref: "refs/heads/mybranch",
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
dir := filepath.Join(basedir, name)
|
dir := filepath.Join(basedir, name)
|
||||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
|
require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/master\n"), 0o644))
|
||||||
require.NoError(t, cleanGitHooks(dir))
|
for _, command := range tt.commands {
|
||||||
tt.Prepare(t, dir)
|
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
||||||
ref, err := FindGitRef(context.Background(), dir)
|
}
|
||||||
tt.Assert(t, ref, err)
|
ref, err := FindGitRef(t.Context(), dir)
|
||||||
|
if tt.ref == "" {
|
||||||
|
require.Error(t, err)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tt.ref, ref)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutor(t *testing.T) {
|
func TestGitCloneExecutor(t *testing.T) {
|
||||||
// Build a local bare "remote" so this runs offline and fast. The cases below mirror
|
remoteDir, workDir := newGitRemote(t)
|
||||||
// the tag/branch/sha/short-sha ref paths the executor handles, formerly exercised by
|
|
||||||
// cloning actions/checkout and anchore/scan-action over the network.
|
|
||||||
remoteDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||||
@@ -207,14 +191,11 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||||
|
|
||||||
// A branch with a dash in the name (mirrors the historical scan-action@act-fails case).
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
||||||
|
|
||||||
out, err := exec.Command("git", "-C", workDir, "rev-parse", "main").Output()
|
fullSha := gitRevParse(t, workDir, "main")
|
||||||
require.NoError(t, err)
|
|
||||||
fullSha := strings.TrimSpace(string(out))
|
|
||||||
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||||
|
|
||||||
@@ -223,6 +204,7 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||||
t.Setenv("GIT_DIR", t.TempDir())
|
t.Setenv("GIT_DIR", t.TempDir())
|
||||||
|
|
||||||
|
var clones sync.WaitGroup
|
||||||
for name, tt := range map[string]struct {
|
for name, tt := range map[string]struct {
|
||||||
Err error
|
Err error
|
||||||
Ref string
|
Ref string
|
||||||
@@ -250,43 +232,40 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
Ref: fullSha[:4],
|
Ref: fullSha[:4],
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
clones.Go(func() {
|
||||||
dir := t.TempDir()
|
t.Run(name, func(t *testing.T) {
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
dir := t.TempDir()
|
||||||
URL: "https://example.invalid/action",
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
Ref: tt.Ref,
|
URL: "https://example.invalid/action",
|
||||||
Dir: dir,
|
Ref: tt.Ref,
|
||||||
})
|
Dir: dir,
|
||||||
|
})
|
||||||
|
|
||||||
err := clone(context.Background())
|
err := clone(t.Context())
|
||||||
if tt.Err != nil {
|
if tt.Err != nil {
|
||||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.Error(t, err)
|
||||||
assert.Equal(t, tt.Err, err)
|
assert.Equal(t, tt.Err, err)
|
||||||
assert.Equal(t, tt.Err, clone(context.Background()), "a retry on the cache keeps the error")
|
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
marker := filepath.Join(dir, "marker")
|
marker := filepath.Join(dir, "marker")
|
||||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||||
require.NoError(t, clone(context.Background()))
|
require.NoError(t, clone(t.Context()))
|
||||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||||
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
clones.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
createRemote := func(message string) string {
|
createRemote := func(message string) string {
|
||||||
remoteDir := t.TempDir()
|
remoteDir, workDir := newGitRemote(t, message)
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
|
|
||||||
return remoteDir
|
return remoteDir
|
||||||
@@ -310,6 +289,12 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
|||||||
markerPath := filepath.Join(cacheDir, "stale-marker")
|
markerPath := filepath.Join(cacheDir, "stale-marker")
|
||||||
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
|
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
|
||||||
|
|
||||||
|
cancelled, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
_, err := CloneIfRequired(cancelled, NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: cacheDir}, logging.New(logtest.New()))
|
||||||
|
require.ErrorIs(t, err, context.Canceled)
|
||||||
|
require.FileExists(t, markerPath)
|
||||||
|
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: newRemoteDir,
|
URL: newRemoteDir,
|
||||||
Ref: "main",
|
Ref: "main",
|
||||||
@@ -330,60 +315,56 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
|||||||
|
|
||||||
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
remoteDir := t.TempDir()
|
remoteDir, workDir := newGitRemote(t, "initial", "feature-1")
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
|
||||||
require.NoError(t, clone(context.Background()))
|
require.NoError(t, clone(t.Context()))
|
||||||
|
require.NoError(t, gitCmd("-C", dir, "config", "fetch.prune", "true"))
|
||||||
|
require.NoError(t, clone(t.Context()))
|
||||||
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
|
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
|
||||||
require.NoError(t, clone(context.Background()), "a non-fast-forward ref must update")
|
require.NoError(t, clone(t.Context()), "a non-fast-forward ref must update")
|
||||||
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
|
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
|
||||||
|
|
||||||
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: t.TempDir()})
|
tagDir := t.TempDir()
|
||||||
require.Error(t, tagClone(context.Background()))
|
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: tagDir})
|
||||||
|
require.Error(t, tagClone(t.Context()))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
|
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
|
||||||
require.NoError(t, tagClone(context.Background()), "a tag published after a failed lookup must be fetched")
|
require.NoError(t, tagClone(t.Context()), "a tag published after a failed lookup must be fetched")
|
||||||
|
|
||||||
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "tag-moved"))
|
||||||
|
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v9"))
|
||||||
|
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "v9"))
|
||||||
|
for range 2 {
|
||||||
|
require.NoError(t, tagClone(t.Context()), "a moved tag must update and stay current")
|
||||||
|
assert.Equal(t, "tag-moved", gitHeadSubject(t, tagDir))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
// Build a local "remote" with a single commit on main.
|
remoteDir, workDir := newGitRemote(t, "initial")
|
||||||
remoteDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
|
|
||||||
// Prime the cache with an online clone of main.
|
|
||||||
cacheDir := t.TempDir()
|
cacheDir := t.TempDir()
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: remoteDir,
|
URL: remoteDir,
|
||||||
Ref: "main",
|
Ref: "main",
|
||||||
Dir: cacheDir,
|
Dir: cacheDir,
|
||||||
})(context.Background()))
|
})(t.Context()))
|
||||||
|
|
||||||
t.Run("cached branch resolves without fetching", func(t *testing.T) {
|
t.Run("cached branch resolves without fetching", func(t *testing.T) {
|
||||||
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
|
|
||||||
// finds no local refs/heads/<ref>.
|
|
||||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: remoteDir,
|
URL: remoteDir,
|
||||||
Ref: "main",
|
Ref: "main",
|
||||||
Dir: cacheDir,
|
Dir: cacheDir,
|
||||||
OfflineMode: true,
|
OfflineMode: true,
|
||||||
})(context.Background())
|
})(t.Context())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
|
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
|
||||||
@@ -392,67 +373,19 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("unresolvable cached ref returns error", func(t *testing.T) {
|
t.Run("unresolvable cached ref returns error", func(t *testing.T) {
|
||||||
// The ref was never cached; offline mode cannot resolve it and must return an error.
|
|
||||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: remoteDir,
|
URL: remoteDir,
|
||||||
Ref: "never-fetched",
|
Ref: "never-fetched",
|
||||||
Dir: cacheDir,
|
Dir: cacheDir,
|
||||||
OfflineMode: true,
|
OfflineMode: true,
|
||||||
})(context.Background())
|
})(t.Context())
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
remoteDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
|
||||||
|
|
||||||
// Quiet callers report the download themselves, so the clone line must not reach the job log.
|
|
||||||
for name, quiet := range map[string]bool{"quiet": true, "not quiet": false} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
logger, hook := logrustest.NewNullLogger()
|
|
||||||
logger.SetLevel(log.InfoLevel)
|
|
||||||
ctx := common.WithLogger(context.Background(), logger.WithField("job", "j1"))
|
|
||||||
|
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
|
||||||
URL: remoteDir,
|
|
||||||
Ref: "main",
|
|
||||||
Dir: t.TempDir(),
|
|
||||||
Quiet: quiet,
|
|
||||||
})(ctx))
|
|
||||||
|
|
||||||
var cloneLines int
|
|
||||||
for _, entry := range hook.AllEntries() {
|
|
||||||
if strings.HasPrefix(entry.Message, "git clone ") {
|
|
||||||
cloneLines++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if quiet {
|
|
||||||
assert.Zero(t, cloneLines)
|
|
||||||
} else {
|
|
||||||
assert.Equal(t, 1, cloneLines)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGitCloneExecutorShallow(t *testing.T) {
|
func TestGitCloneExecutorShallow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
|
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
||||||
remoteDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
for _, m := range []string{"c1", "c2", "c3"} {
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
|
|
||||||
}
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||||
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
@@ -460,25 +393,17 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
|
|
||||||
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
||||||
|
|
||||||
t.Run("branch is cloned shallowly", func(t *testing.T) {
|
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
||||||
dir := t.TempDir()
|
t.Run(name, func(t *testing.T) {
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
dir := t.TempDir()
|
||||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
})(t.Context()))
|
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
||||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
})(t.Context()))
|
||||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
||||||
})
|
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||||
|
})
|
||||||
t.Run("tag is cloned shallowly", func(t *testing.T) {
|
}
|
||||||
dir := t.TempDir()
|
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
|
||||||
URL: remoteDir, Ref: "v1", Dir: dir, Depth: 1,
|
|
||||||
})(t.Context()))
|
|
||||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
|
||||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
|
||||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
|
||||||
})
|
|
||||||
|
|
||||||
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
||||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||||
@@ -558,12 +483,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
|
|
||||||
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
remoteDir := t.TempDir()
|
remoteDir, workDir := newGitRemote(t, "pinned")
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
|
||||||
workDir := t.TempDir()
|
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "pinned"))
|
|
||||||
pinned := gitRevParse(t, workDir, "HEAD")
|
pinned := gitRevParse(t, workDir, "HEAD")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
||||||
@@ -581,6 +501,21 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
remoteDir, workDir := t.TempDir(), t.TempDir()
|
||||||
|
require.NoError(t, os.CopyFS(remoteDir, os.DirFS(filepath.Join(gitFixtureDir, "remote"))))
|
||||||
|
require.NoError(t, os.CopyFS(workDir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
|
config, err := os.ReadFile(filepath.Join(workDir, ".git", "config"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".git", "config"),
|
||||||
|
fmt.Appendf(config, "\n[remote \"origin\"]\n\turl = %s\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", strconv.Quote(remoteDir)), 0o644))
|
||||||
|
for _, message := range messages {
|
||||||
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||||
|
}
|
||||||
|
return remoteDir, workDir
|
||||||
|
}
|
||||||
|
|
||||||
func gitRevParse(t *testing.T, dir, rev string) string {
|
func gitRevParse(t *testing.T, dir, rev string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()
|
out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()
|
||||||
@@ -608,9 +543,6 @@ func gitCmd(args ...string) error {
|
|||||||
cmd := exec.Command("git", args...)
|
cmd := exec.Command("git", args...)
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
cmd.Stderr = os.Stderr
|
cmd.Stderr = os.Stderr
|
||||||
// Inject a deterministic identity and ignore the host's global/system config so commits
|
|
||||||
// succeed regardless of the host having no user.name/user.email (e.g. CI, GITHUB_ACTIONS
|
|
||||||
// unset) or a global commit.gpgsign, and without mutating the developer's ~/.gitconfig.
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
"GIT_AUTHOR_NAME=Unit Test",
|
"GIT_AUTHOR_NAME=Unit Test",
|
||||||
"[email protected]",
|
"[email protected]",
|
||||||
@@ -631,14 +563,42 @@ func gitCmd(args ...string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAcquireCloneLock(t *testing.T) {
|
func TestAcquireCloneLock(t *testing.T) {
|
||||||
|
t.Run("blocks on a lock held by another process despite a trailing separator", func(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "cache")
|
||||||
|
release, err := lock.TryLock(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { require.NoError(t, release()) }()
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err := AcquireCloneLock(ctx, dir+string(filepath.Separator))
|
||||||
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("read-only parent falls back to an in-process lock", func(t *testing.T) {
|
||||||
|
parent := t.TempDir()
|
||||||
|
dir := filepath.Join(parent, "cache")
|
||||||
|
require.NoError(t, os.Mkdir(dir, 0o755))
|
||||||
|
require.NoError(t, os.Chmod(parent, 0o555))
|
||||||
|
defer func() { require.NoError(t, os.Chmod(parent, 0o755)) }()
|
||||||
|
unlock, err := AcquireCloneLock(t.Context(), dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
unlock()
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("same directory serializes", func(t *testing.T) {
|
t.Run("same directory serializes", func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
unlock1 := AcquireCloneLock(dir)
|
unlock1, err := AcquireCloneLock(t.Context(), dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
secondAcquired := make(chan struct{})
|
secondAcquired := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
unlock := AcquireCloneLock(dir)
|
unlock, err := AcquireCloneLock(t.Context(), dir)
|
||||||
|
if !assert.NoError(t, err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
close(secondAcquired)
|
close(secondAcquired)
|
||||||
unlock()
|
unlock()
|
||||||
}()
|
}()
|
||||||
@@ -662,12 +622,16 @@ func TestAcquireCloneLock(t *testing.T) {
|
|||||||
dirA := t.TempDir()
|
dirA := t.TempDir()
|
||||||
dirB := t.TempDir()
|
dirB := t.TempDir()
|
||||||
|
|
||||||
unlockA := AcquireCloneLock(dirA)
|
unlockA, err := AcquireCloneLock(t.Context(), dirA)
|
||||||
|
require.NoError(t, err)
|
||||||
defer unlockA()
|
defer unlockA()
|
||||||
|
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
unlock := AcquireCloneLock(dirB)
|
unlock, err := AcquireCloneLock(t.Context(), dirB)
|
||||||
|
if !assert.NoError(t, err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
unlock()
|
unlock()
|
||||||
close(done)
|
close(done)
|
||||||
}()
|
}()
|
||||||
@@ -680,7 +644,6 @@ func TestAcquireCloneLock(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
|
|
||||||
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
block := make(chan struct{})
|
block := make(chan struct{})
|
||||||
|
|||||||
@@ -14,13 +14,8 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
|
|||||||
|
|
||||||
// JobError returns the job error for current context if any
|
// JobError returns the job error for current context if any
|
||||||
func JobError(ctx context.Context) error {
|
func JobError(ctx context.Context) error {
|
||||||
val := ctx.Value(jobErrorContextKeyVal)
|
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
|
||||||
if val != nil {
|
return container["error"]
|
||||||
if container, ok := val.(map[string]error); ok {
|
|
||||||
return container["error"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetJobError(ctx context.Context, err error) {
|
func SetJobError(ctx context.Context, err error) {
|
||||||
@@ -31,6 +26,5 @@ func SetJobError(ctx context.Context, err error) {
|
|||||||
|
|
||||||
// WithJobErrorContainer adds a value to the context as a container for an error
|
// WithJobErrorContainer adds a value to the context as a container for an error
|
||||||
func WithJobErrorContainer(ctx context.Context) context.Context {
|
func WithJobErrorContainer(ctx context.Context) context.Context {
|
||||||
container := map[string]error{}
|
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
|
||||||
return context.WithValue(ctx, jobErrorContextKeyVal, container)
|
|
||||||
}
|
}
|
||||||
@@ -26,9 +26,7 @@ type lineWriter struct {
|
|||||||
|
|
||||||
// NewLineWriter creates a new instance of a line writer
|
// NewLineWriter creates a new instance of a line writer
|
||||||
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
||||||
w := new(lineWriter)
|
return &lineWriter{handlers: handlers}
|
||||||
w.handlers = handlers
|
|
||||||
return w
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
||||||
@@ -39,24 +37,15 @@ func FlushWriter(w io.Writer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lw *lineWriter) Write(p []byte) (n int, err error) {
|
func (lw *lineWriter) Write(p []byte) (int, error) {
|
||||||
pBuf := bytes.NewBuffer(p)
|
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
|
||||||
written := 0
|
_, _ = lw.buffer.Write(line)
|
||||||
for {
|
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||||
line, err := pBuf.ReadString('\n')
|
lw.handleLine(lw.buffer.String())
|
||||||
w, _ := lw.buffer.WriteString(line)
|
lw.buffer.Reset()
|
||||||
written += w
|
|
||||||
if err != nil {
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return written, err
|
|
||||||
}
|
}
|
||||||
lw.handleLine(lw.buffer.String())
|
|
||||||
lw.buffer.Reset()
|
|
||||||
}
|
}
|
||||||
|
return len(p), nil
|
||||||
return written, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
||||||
@@ -72,8 +61,7 @@ func (lw *lineWriter) Flush() {
|
|||||||
|
|
||||||
func (lw *lineWriter) handleLine(line string) {
|
func (lw *lineWriter) handleLine(line string) {
|
||||||
for _, h := range lw.handlers {
|
for _, h := range lw.handlers {
|
||||||
ok := h(line)
|
if !h(line) {
|
||||||
if !ok {
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,24 +9,24 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLineWriter(t *testing.T) {
|
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
|
||||||
lines := make([]string, 0)
|
var lines []string
|
||||||
lineHandler := func(s string) bool {
|
lineWriter := NewLineWriter(func(s string) bool {
|
||||||
lines = append(lines, s)
|
lines = append(lines, s)
|
||||||
return true
|
return true
|
||||||
}
|
})
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
write := func(s string) {
|
write := func(s string) {
|
||||||
n, err := lineWriter.Write([]byte(s))
|
n, err := lineWriter.Write([]byte(s))
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(len(s), n, s)
|
assert.Equal(len(s), n, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
write("")
|
||||||
write("hello")
|
write("hello")
|
||||||
write(" ")
|
write(" ")
|
||||||
write("world!!\nextra")
|
write("world!!\nextra")
|
||||||
@@ -34,39 +34,17 @@ func TestLineWriter(t *testing.T) {
|
|||||||
write(" line\n")
|
write(" line\n")
|
||||||
write("no newline here...")
|
write("no newline here...")
|
||||||
|
|
||||||
assert.Len(lines, 4)
|
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
|
||||||
assert.Equal("hello world!!\n", lines[0])
|
assert.Equal(expected, lines)
|
||||||
assert.Equal("extra line\n", lines[1])
|
|
||||||
assert.Equal(" and another\n", lines[2])
|
|
||||||
assert.Equal("last line\n", lines[3])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLineWriterFlush(t *testing.T) {
|
|
||||||
lines := make([]string, 0)
|
|
||||||
lineHandler := func(s string) bool {
|
|
||||||
lines = append(lines, s)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
|
||||||
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
|
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
|
|
||||||
|
|
||||||
// Only the newline-terminated line is emitted before flushing.
|
|
||||||
assert.Equal([]string{"complete line\n"}, lines)
|
|
||||||
|
|
||||||
// Flushing emits the buffered, not-yet-terminated trailing line.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
|
expected = append(expected, "no newline here...")
|
||||||
|
assert.Equal(expected, lines)
|
||||||
|
|
||||||
// Flushing again is a no-op: nothing is buffered.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Len(lines, 2)
|
assert.Equal(expected, lines)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
||||||
// FlushWriter must be a safe no-op for writers that do not buffer lines.
|
|
||||||
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
||||||
}
|
}
|
||||||
+30
-25
@@ -7,46 +7,51 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
type loggerContextKey string
|
type loggerContextKey string
|
||||||
|
|
||||||
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
|
const loggerContextKeyVal = loggerContextKey("logging.Logger")
|
||||||
|
|
||||||
// Logger returns the appropriate logger for current context
|
// Logger returns the appropriate logger for current context
|
||||||
func Logger(ctx context.Context) logrus.FieldLogger {
|
func Logger(ctx context.Context) *logging.Logger {
|
||||||
val := ctx.Value(loggerContextKeyVal)
|
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
|
||||||
if val != nil {
|
return logger
|
||||||
if logger, ok := val.(logrus.FieldLogger); ok {
|
|
||||||
return logger
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return logrus.StandardLogger()
|
return logging.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLogger adds a value to the context for the logger
|
// WithLogger adds a value to the context for the logger
|
||||||
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
|
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
|
||||||
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
||||||
}
|
}
|
||||||
|
|
||||||
type loggerHookKey string
|
type loggerHandlerKey string
|
||||||
|
|
||||||
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
|
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
|
||||||
|
|
||||||
// LoggerHook returns the appropriate logger hook for current context
|
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
|
||||||
// the hook affects job logger, not global logger
|
// It affects the job logger, not the global logger.
|
||||||
func LoggerHook(ctx context.Context) logrus.Hook {
|
func LoggerHandler(ctx context.Context) logging.Handler {
|
||||||
val := ctx.Value(loggerHookKeyVal)
|
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
|
||||||
if val != nil {
|
return h
|
||||||
if hook, ok := val.(logrus.Hook); ok {
|
|
||||||
return hook
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLoggerHook adds a value to the context for the logger hook
|
// WithLoggerHandler adds the extra handler for the job logger to the context
|
||||||
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
|
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
|
||||||
return context.WithValue(ctx, loggerHookKeyVal, hook)
|
return context.WithValue(ctx, loggerHandlerKeyVal, h)
|
||||||
|
}
|
||||||
|
|
||||||
|
type stepIDsContextKey struct{}
|
||||||
|
|
||||||
|
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
|
||||||
|
func StepIDs(ctx context.Context) []string {
|
||||||
|
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithStepIDs replaces the step IDs in the context
|
||||||
|
func WithStepIDs(ctx context.Context, ids []string) context.Context {
|
||||||
|
return context.WithValue(ctx, stepIDsContextKey{}, ids)
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ansiReset = "\x1b[0m"
|
||||||
|
ansiRed = "\x1b[31m"
|
||||||
|
ansiYellow = "\x1b[33m"
|
||||||
|
ansiCyan = "\x1b[36m"
|
||||||
|
ansiDimCyan = "\x1b[2;36m"
|
||||||
|
ansiDimWhite = "\x1b[2;37m"
|
||||||
|
)
|
||||||
|
|
||||||
|
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
|
||||||
|
var consoleStart = time.Now()
|
||||||
|
|
||||||
|
// console is the state shared by every consoleHandler derived from one another.
|
||||||
|
type console struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
out io.Writer
|
||||||
|
terminal bool
|
||||||
|
opts TextOptions
|
||||||
|
reportCaller bool
|
||||||
|
level slog.LevelVar
|
||||||
|
}
|
||||||
|
|
||||||
|
func newConsole(out io.Writer) *console {
|
||||||
|
c := &console{}
|
||||||
|
c.setOutput(out)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *console) setOutput(w io.Writer) {
|
||||||
|
terminal := checkIfTerminal(w)
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.out, c.terminal = w, terminal
|
||||||
|
}
|
||||||
|
|
||||||
|
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
|
||||||
|
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
|
||||||
|
// and a colored prefix with elapsed or full time on a terminal.
|
||||||
|
type consoleHandler struct {
|
||||||
|
c *console
|
||||||
|
attrs []slog.Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||||
|
return level >= h.c.level.Level()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||||
|
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
|
||||||
|
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
|
||||||
|
|
||||||
|
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||||
|
c := h.c
|
||||||
|
c.mu.Lock()
|
||||||
|
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
|
||||||
|
for _, a := range h.attrs {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
}
|
||||||
|
r.Attrs(func(a slog.Attr) bool {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
|
||||||
|
var caller *runtime.Frame
|
||||||
|
if reportCaller && r.PC != 0 {
|
||||||
|
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
|
||||||
|
caller = &frame
|
||||||
|
}
|
||||||
|
prefixFieldClashes(data, caller != nil)
|
||||||
|
|
||||||
|
keys := slices.Sorted(maps.Keys(data))
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
if terminal && !opts.DisableColors {
|
||||||
|
printColored(&b, opts, r, caller, keys, data)
|
||||||
|
} else {
|
||||||
|
printPlain(&b, opts, r, caller, keys, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
line := b.Bytes()
|
||||||
|
if opts.Filter != nil {
|
||||||
|
line = opts.Filter(line)
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
_, err := c.out.Write(line)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
|
||||||
|
func prefixFieldClashes(data map[string]any, reportCaller bool) {
|
||||||
|
for _, key := range []string{"time", "level", "msg"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
delete(data, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reportCaller {
|
||||||
|
for _, key := range []string{"func", "file"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
if !opts.DisableTimestamp {
|
||||||
|
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
|
||||||
|
if r.Message != "" {
|
||||||
|
appendKeyValue(b, "msg", r.Message)
|
||||||
|
}
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if funcVal != "" {
|
||||||
|
appendKeyValue(b, "func", funcVal)
|
||||||
|
}
|
||||||
|
if fileVal != "" {
|
||||||
|
appendKeyValue(b, "file", fileVal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
appendKeyValue(b, k, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
|
||||||
|
if opts.CallerPrettyfier != nil {
|
||||||
|
return opts.CallerPrettyfier(caller)
|
||||||
|
}
|
||||||
|
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
msg := strings.TrimSuffix(r.Message, "\n")
|
||||||
|
|
||||||
|
var callerStr string
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if opts.CallerPrettyfier == nil && caller.Function != "" {
|
||||||
|
funcVal += "()"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case fileVal == "":
|
||||||
|
callerStr = funcVal
|
||||||
|
case funcVal == "":
|
||||||
|
callerStr = fileVal
|
||||||
|
default:
|
||||||
|
callerStr = fileVal + " " + funcVal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
level := Level(r.Level)
|
||||||
|
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
|
||||||
|
switch {
|
||||||
|
case opts.DisableTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
|
||||||
|
case !opts.FullTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
b.WriteString(colorize(level, k))
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainLevel spells Warn the way logrus did, as "warning".
|
||||||
|
func plainLevel(l Level) string {
|
||||||
|
if s := l.String(); s != "warn" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "warning"
|
||||||
|
}
|
||||||
|
|
||||||
|
func colorize(l Level, s string) string {
|
||||||
|
var color string
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
color = ansiDimWhite
|
||||||
|
case l <= LevelDebug:
|
||||||
|
color = ansiDimCyan
|
||||||
|
case l <= LevelInfo:
|
||||||
|
color = ansiCyan
|
||||||
|
case l <= LevelWarn:
|
||||||
|
color = ansiYellow
|
||||||
|
default:
|
||||||
|
color = ansiRed
|
||||||
|
}
|
||||||
|
return color + s + ansiReset
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendKeyValue(b *bytes.Buffer, key string, value any) {
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
b.WriteString(key)
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendValue(b *bytes.Buffer, value any) {
|
||||||
|
switch v := value.(type) {
|
||||||
|
case string:
|
||||||
|
appendString(b, v)
|
||||||
|
case []byte:
|
||||||
|
appendString(b, string(v))
|
||||||
|
case error:
|
||||||
|
appendFormatted(b, v, v.Error)
|
||||||
|
case fmt.Stringer:
|
||||||
|
appendFormatted(b, v, v.String)
|
||||||
|
default:
|
||||||
|
appendString(b, fmt.Sprint(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
|
||||||
|
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
|
||||||
|
appendString(b, "<nil>")
|
||||||
|
} else {
|
||||||
|
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
appendString(b, get())
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendString(b *bytes.Buffer, s string) {
|
||||||
|
if needsQuoting(s) {
|
||||||
|
b.WriteString(strconv.Quote(s))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.WriteString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsQuoting(s string) bool {
|
||||||
|
for i := range len(s) {
|
||||||
|
c := s[i]
|
||||||
|
switch {
|
||||||
|
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
|
||||||
|
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
|
||||||
|
t.Helper()
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
|
||||||
|
c.level.Set(slog.LevelDebug)
|
||||||
|
build(New(NewSlogHandler(&consoleHandler{c: c})))
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlain(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
|
||||||
|
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
|
||||||
|
l.With("msg", "clash", "empty", "").Info("")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
|
||||||
|
"level=info empty= fields.msg=clash\n", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
|
||||||
|
opts := TextOptions{
|
||||||
|
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
|
||||||
|
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
|
||||||
|
},
|
||||||
|
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
|
||||||
|
}
|
||||||
|
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
|
||||||
|
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleColored(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
|
||||||
|
l.With("job", "j").Warn("careful")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
|
||||||
|
|
||||||
|
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
|
||||||
|
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
|
||||||
|
|
||||||
|
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
|
||||||
|
|
||||||
|
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Equal(t, "level=info msg=x\n", plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleLevelAndOutput(t *testing.T) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
h := &consoleHandler{c: c}
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
|
||||||
|
c.level.Set(slog.LevelError)
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
|
||||||
|
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
|
||||||
|
|
||||||
|
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
|
||||||
|
rec.AddAttrs(slog.String("c", "3"))
|
||||||
|
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
|
||||||
|
assert.Contains(t, out.String(), "a=1 c=3")
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TextOptions describes how the process-wide Logger writes a record as text.
|
||||||
|
type TextOptions struct {
|
||||||
|
DisableColors bool
|
||||||
|
DisableTimestamp bool
|
||||||
|
FullTimestamp bool
|
||||||
|
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
|
||||||
|
CallerPrettyfier func(*runtime.Frame) (function, file string)
|
||||||
|
// Filter rewrites each formatted line, for example to mask secrets.
|
||||||
|
Filter func(line []byte) []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
std atomic.Pointer[Logger]
|
||||||
|
stdState = newConsole(os.Stderr)
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default returns the process-wide Logger.
|
||||||
|
func Default() *Logger { return std.Load() }
|
||||||
|
|
||||||
|
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
|
||||||
|
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
|
||||||
|
// initial console Logger.
|
||||||
|
func SetDefault(l *Logger) { std.Store(l) }
|
||||||
|
|
||||||
|
// SetLevel sets the minimum level of the initial process-wide Logger.
|
||||||
|
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
|
||||||
|
|
||||||
|
// GetLevel returns the minimum level of the initial process-wide Logger.
|
||||||
|
func GetLevel() Level { return Level(stdState.level.Level()) }
|
||||||
|
|
||||||
|
// SetOutput sets where the initial process-wide Logger writes.
|
||||||
|
func SetOutput(w io.Writer) { stdState.setOutput(w) }
|
||||||
|
|
||||||
|
// Output returns where the initial process-wide Logger writes.
|
||||||
|
func Output() io.Writer {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
return stdState.out
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetTextFormat sets the text format of the initial process-wide Logger.
|
||||||
|
func SetTextFormat(o TextOptions) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.opts = o
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
|
||||||
|
func SetReportCaller(on bool) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.reportCaller = on
|
||||||
|
}
|
||||||
|
|
||||||
|
func Enabled(level Level) bool { return Default().Enabled(level) }
|
||||||
|
|
||||||
|
func With(args ...any) *Logger { return Default().With(args...) }
|
||||||
|
func WithError(err error) *Logger { return Default().WithError(err) }
|
||||||
|
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
|
||||||
|
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
|
||||||
|
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
|
||||||
|
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
|
||||||
|
func Debug(args ...any) { Default().log(LevelDebug, args) }
|
||||||
|
func Info(args ...any) { Default().log(LevelInfo, args) }
|
||||||
|
func Warn(args ...any) { Default().log(LevelWarn, args) }
|
||||||
|
func Error(args ...any) { Default().log(LevelError, args) }
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Attr is a structured key/value pair attached to a log record.
|
||||||
|
type Attr struct {
|
||||||
|
Key string
|
||||||
|
Value any
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
|
||||||
|
// Logger.With are accumulated by the Handler.
|
||||||
|
type Record struct {
|
||||||
|
Time time.Time
|
||||||
|
Level Level
|
||||||
|
Message string
|
||||||
|
PC uintptr // program counter of the logging call, 0 if unknown
|
||||||
|
Attrs []Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attr returns the value of the last attribute named key.
|
||||||
|
func (r Record) Attr(key string) (any, bool) {
|
||||||
|
for _, a := range slices.Backward(r.Attrs) {
|
||||||
|
if a.Key == key {
|
||||||
|
return a.Value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
|
||||||
|
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
|
||||||
|
type Handler interface {
|
||||||
|
Enabled(ctx context.Context, level Level) bool
|
||||||
|
Handle(ctx context.Context, r Record) error
|
||||||
|
// WithAttrs returns a Handler that adds attrs to every record it handles.
|
||||||
|
WithAttrs(attrs []Attr) Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
type multiHandler []Handler
|
||||||
|
|
||||||
|
// NewMultiHandler fans every record out to all handlers that have its level enabled.
|
||||||
|
func NewMultiHandler(handlers ...Handler) Handler {
|
||||||
|
return multiHandler(handlers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, level) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
var errs []error
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, r.Level) {
|
||||||
|
errs = append(errs, h.Handle(ctx, r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
out := make(multiHandler, len(m))
|
||||||
|
for i, h := range m {
|
||||||
|
out[i] = h.WithAttrs(attrs)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
type discardHandler struct{}
|
||||||
|
|
||||||
|
// DiscardHandler drops every record.
|
||||||
|
var DiscardHandler Handler = discardHandler{}
|
||||||
|
|
||||||
|
func (discardHandler) Enabled(context.Context, Level) bool { return false }
|
||||||
|
func (discardHandler) Handle(context.Context, Record) error { return nil }
|
||||||
|
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
|
||||||
|
|
||||||
|
type levelHandler struct {
|
||||||
|
h Handler
|
||||||
|
minLevel Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLevelHandler passes only records at or above minLevel on to h.
|
||||||
|
func NewLevelHandler(h Handler, minLevel Level) Handler {
|
||||||
|
return levelHandler{h: h, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
return level >= l.minLevel && l.h.Enabled(ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
if r.Level < l.minLevel {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return l.h.Handle(ctx, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Level is a log severity. The values match log/slog, with Trace added below Debug.
|
||||||
|
type Level int
|
||||||
|
|
||||||
|
const (
|
||||||
|
LevelTrace Level = -8
|
||||||
|
LevelDebug Level = -4
|
||||||
|
LevelInfo Level = 0
|
||||||
|
LevelWarn Level = 4
|
||||||
|
LevelError Level = 8
|
||||||
|
)
|
||||||
|
|
||||||
|
func (l Level) String() string {
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
return "trace"
|
||||||
|
case l <= LevelDebug:
|
||||||
|
return "debug"
|
||||||
|
case l <= LevelInfo:
|
||||||
|
return "info"
|
||||||
|
case l <= LevelWarn:
|
||||||
|
return "warn"
|
||||||
|
default:
|
||||||
|
return "error"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
|
||||||
|
// the Logger has no level above it.
|
||||||
|
func ParseLevel(s string) (Level, error) {
|
||||||
|
switch strings.ToLower(s) {
|
||||||
|
case "trace":
|
||||||
|
return LevelTrace, nil
|
||||||
|
case "debug":
|
||||||
|
return LevelDebug, nil
|
||||||
|
case "info":
|
||||||
|
return LevelInfo, nil
|
||||||
|
case "warn", "warning":
|
||||||
|
return LevelWarn, nil
|
||||||
|
case "error", "fatal", "panic":
|
||||||
|
return LevelError, nil
|
||||||
|
}
|
||||||
|
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const badKey = "!BADKEY"
|
||||||
|
|
||||||
|
// Logger writes printf-style messages with structured attributes to a Handler.
|
||||||
|
type Logger struct {
|
||||||
|
h Handler
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(h Handler) *Logger {
|
||||||
|
return &Logger{h: h, ctx: context.Background()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
|
||||||
|
func (l *Logger) With(args ...any) *Logger {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) WithError(err error) *Logger {
|
||||||
|
return l.With("error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithContext returns a Logger that passes ctx to its Handler.
|
||||||
|
func (l *Logger) WithContext(ctx context.Context) *Logger {
|
||||||
|
return &Logger{h: l.h, ctx: ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Enabled(level Level) bool {
|
||||||
|
return l.h.Enabled(l.ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
|
||||||
|
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
|
||||||
|
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
|
||||||
|
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
|
||||||
|
|
||||||
|
// Trace and the methods below format their arguments like fmt.Sprint.
|
||||||
|
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
|
||||||
|
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
|
||||||
|
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
|
||||||
|
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
|
||||||
|
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
|
||||||
|
|
||||||
|
func (l *Logger) logf(level Level, format string, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) log(level Level, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprint(args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// emit drops the Handler error, as a logger has nowhere to report it.
|
||||||
|
func (l *Logger) emit(pc uintptr, level Level, msg string) {
|
||||||
|
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
|
||||||
|
func callerPC(skip int) uintptr {
|
||||||
|
var pcs [1]uintptr
|
||||||
|
runtime.Callers(skip, pcs[:])
|
||||||
|
return pcs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func argsToAttrs(args []any) []Attr {
|
||||||
|
attrs := make([]Attr, 0, len(args)/2+1)
|
||||||
|
for len(args) > 0 {
|
||||||
|
switch k := args[0].(type) {
|
||||||
|
case Attr:
|
||||||
|
attrs = append(attrs, k)
|
||||||
|
args = args[1:]
|
||||||
|
case string:
|
||||||
|
if len(args) == 1 {
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = nil
|
||||||
|
} else {
|
||||||
|
attrs = append(attrs, Attr{Key: k, Value: args[1]})
|
||||||
|
args = args[2:]
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return attrs
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseLevel(t *testing.T) {
|
||||||
|
for in, want := range map[string]logging.Level{
|
||||||
|
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
|
||||||
|
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
|
||||||
|
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
|
||||||
|
} {
|
||||||
|
got, err := logging.ParseLevel(in)
|
||||||
|
require.NoError(t, err, in)
|
||||||
|
assert.Equal(t, want, got, in)
|
||||||
|
}
|
||||||
|
_, err := logging.ParseLevel("loud")
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, "warn", logging.LevelWarn.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingStringer struct{ n *int }
|
||||||
|
|
||||||
|
func (c countingStringer) String() string { *c.n++; return "x" }
|
||||||
|
|
||||||
|
func TestLogger(t *testing.T) {
|
||||||
|
h := logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
assert.Nil(t, h.LastRecord())
|
||||||
|
l := logging.New(h)
|
||||||
|
|
||||||
|
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
|
||||||
|
l.Info("a", 1, 2, "b")
|
||||||
|
l.With("k").With(7).Debug("x")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 3)
|
||||||
|
assert.Equal(t, "step 2", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
assert.False(t, recs[0].Time.IsZero())
|
||||||
|
job, _ := recs[0].Attr("job")
|
||||||
|
assert.Equal(t, "j1", job)
|
||||||
|
errv, _ := recs[0].Attr("error")
|
||||||
|
require.EqualError(t, errv.(error), "boom")
|
||||||
|
assert.Equal(t, "a1 2b", recs[1].Message)
|
||||||
|
bad, _ := recs[2].Attr("!BADKEY")
|
||||||
|
assert.Equal(t, 7, bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
|
||||||
|
n := 0
|
||||||
|
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
|
||||||
|
l.Debugf("%v", countingStringer{&n})
|
||||||
|
assert.Zero(t, n)
|
||||||
|
assert.False(t, l.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiHandler(t *testing.T) {
|
||||||
|
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
|
||||||
|
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
|
||||||
|
l.Info("i")
|
||||||
|
l.Error("e")
|
||||||
|
assert.Len(t, all.AllRecords(), 2)
|
||||||
|
require.Len(t, warnOnly.AllRecords(), 1)
|
||||||
|
v, _ := warnOnly.LastRecord().Attr("a")
|
||||||
|
assert.Equal(t, 1, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
|
||||||
|
t.Helper()
|
||||||
|
require.NotZero(t, pc)
|
||||||
|
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCallerPC(t *testing.T) {
|
||||||
|
l, h := logtest.NewLogger()
|
||||||
|
l.Infof("method")
|
||||||
|
l.Info("method")
|
||||||
|
|
||||||
|
prev := logging.Default()
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
logging.SetDefault(l)
|
||||||
|
logging.Infof("global")
|
||||||
|
logging.Error("global")
|
||||||
|
|
||||||
|
require.Len(t, h.AllRecords(), 4)
|
||||||
|
for _, r := range h.AllRecords() {
|
||||||
|
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultLevel(t *testing.T) {
|
||||||
|
prev := logging.GetLevel()
|
||||||
|
t.Cleanup(func() { logging.SetLevel(prev) })
|
||||||
|
|
||||||
|
logging.SetLevel(logging.LevelWarn)
|
||||||
|
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
|
||||||
|
assert.True(t, logging.Enabled(logging.LevelError))
|
||||||
|
assert.False(t, logging.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogHandler(t *testing.T) {
|
||||||
|
var buf strings.Builder
|
||||||
|
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
|
||||||
|
|
||||||
|
assert.False(t, l.Enabled(logging.LevelTrace))
|
||||||
|
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
|
||||||
|
|
||||||
|
out := buf.String()
|
||||||
|
assert.Contains(t, out, "level=ERROR")
|
||||||
|
assert.Contains(t, out, `msg="fail x"`)
|
||||||
|
assert.Contains(t, out, "job=j1")
|
||||||
|
assert.Contains(t, out, "error=boom")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogBridge(t *testing.T) {
|
||||||
|
h := logtest.SetDefault(t)
|
||||||
|
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
|
||||||
|
_, _, line, _ := runtime.Caller(0)
|
||||||
|
slog.Debug("dropped")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 1)
|
||||||
|
assert.Equal(t, "from a library", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
lib, _ := recs[0].Attr("lib")
|
||||||
|
assert.Equal(t, "actionslib", lib)
|
||||||
|
n, _ := recs[0].Attr("n")
|
||||||
|
assert.EqualValues(t, 1, n)
|
||||||
|
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
// Package logtest provides a Handler that keeps records in memory for assertions.
|
||||||
|
package logtest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
)
|
||||||
|
|
||||||
|
type store struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
records []logging.Record
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
|
||||||
|
type Handler struct {
|
||||||
|
store *store
|
||||||
|
attrs []logging.Attr
|
||||||
|
minLevel logging.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New records from LevelInfo up, like the logger of logrus's hooks/test.
|
||||||
|
func New() *Handler {
|
||||||
|
return NewWithLevel(logging.LevelInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
|
||||||
|
func NewLogger() (*logging.Logger, *Handler) {
|
||||||
|
h := New()
|
||||||
|
return logging.New(h), h
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWithLevel(minLevel logging.Level) *Handler {
|
||||||
|
return &Handler{store: &store{}, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
|
||||||
|
|
||||||
|
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
|
||||||
|
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
|
||||||
|
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
h.store.records = append(h.store.records, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
|
||||||
|
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) AllRecords() []logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
return slices.Clone(h.store.records)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LastRecord returns nil when nothing was logged.
|
||||||
|
func (h *Handler) LastRecord() *logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
if len(h.store.records) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r := h.store.records[len(h.store.records)-1]
|
||||||
|
return &r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
|
||||||
|
func SetDefault(t testing.TB) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
prev := logging.Default()
|
||||||
|
logger, h := NewLogger()
|
||||||
|
logging.SetDefault(logger)
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
return h
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
slog.SetDefault(slog.New(slogBridge{}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// slogBridge hands what libraries log through slog's global logger to the process-wide Logger.
|
||||||
|
type slogBridge struct {
|
||||||
|
attrs []Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) Enabled(ctx context.Context, level slog.Level) bool {
|
||||||
|
return Default().h.Enabled(ctx, Level(level))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) Handle(ctx context.Context, r slog.Record) error {
|
||||||
|
h := Default().h
|
||||||
|
if len(b.attrs) > 0 {
|
||||||
|
h = h.WithAttrs(b.attrs)
|
||||||
|
}
|
||||||
|
attrs := make([]Attr, 0, r.NumAttrs())
|
||||||
|
r.Attrs(func(a slog.Attr) bool {
|
||||||
|
attrs = append(attrs, fromSlogAttr(a))
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return h.Handle(ctx, Record{Time: r.Time, Level: Level(r.Level), Message: r.Message, PC: r.PC, Attrs: attrs})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||||
|
out := slices.Clone(b.attrs)
|
||||||
|
for _, a := range attrs {
|
||||||
|
out = append(out, fromSlogAttr(a))
|
||||||
|
}
|
||||||
|
return slogBridge{attrs: out}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) WithGroup(string) slog.Handler { return b }
|
||||||
|
|
||||||
|
func fromSlogAttr(a slog.Attr) Attr {
|
||||||
|
return Attr{Key: a.Key, Value: a.Value.Resolve().Any()}
|
||||||
|
}
|
||||||
|
|
||||||
|
type slogHandler struct {
|
||||||
|
h slog.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSlogHandler writes to h. Levels map one to one, as Level uses slog's values.
|
||||||
|
func NewSlogHandler(h slog.Handler) Handler {
|
||||||
|
return slogHandler{h: h}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
return s.h.Enabled(ctx, slog.Level(level))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
rec := slog.NewRecord(r.Time, slog.Level(r.Level), r.Message, r.PC)
|
||||||
|
rec.AddAttrs(toSlogAttrs(r.Attrs)...)
|
||||||
|
return s.h.Handle(ctx, rec)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
return slogHandler{h: s.h.WithAttrs(toSlogAttrs(attrs))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toSlogAttrs(attrs []Attr) []slog.Attr {
|
||||||
|
out := make([]slog.Attr, len(attrs))
|
||||||
|
for i, a := range attrs {
|
||||||
|
out[i] = slog.Any(a.Key, a.Value)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"golang.org/x/term"
|
||||||
|
)
|
||||||
|
|
||||||
|
func checkIfTerminal(w io.Writer) bool {
|
||||||
|
if f, ok := w.(*os.File); ok {
|
||||||
|
return term.IsTerminal(int(f.Fd()))
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
// checkIfTerminal also turns on virtual terminal processing, which the console needs to show colors.
|
||||||
|
func checkIfTerminal(w io.Writer) bool {
|
||||||
|
f, ok := w.(*os.File)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
handle := windows.Handle(f.Fd())
|
||||||
|
var mode uint32
|
||||||
|
if err := windows.GetConsoleMode(handle, &mode); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return windows.SetConsoleMode(handle, mode|windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == nil
|
||||||
|
}
|
||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
|
|
||||||
type interfaceIP struct {
|
type interfaceIP struct {
|
||||||
net.IP
|
net.IP
|
||||||
net.Interface
|
Name string
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetOutboundIP returns an outbound IP address of this machine.
|
// GetOutboundIP returns an outbound IP address of this machine.
|
||||||
@@ -43,7 +43,7 @@ func GetOutboundIP() net.IP {
|
|||||||
ip = v.IP
|
ip = v.IP
|
||||||
}
|
}
|
||||||
if ip.IsGlobalUnicast() {
|
if ip.IsGlobalUnicast() {
|
||||||
ips = append(ips, interfaceIP{ip, i})
|
ips = append(ips, interfaceIP{ip, i.Name})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -59,29 +59,25 @@ func preferredIP(ips []interfaceIP) net.IP {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
sort.Slice(ips, func(i, j int) bool {
|
sort.Slice(ips, func(i, j int) bool {
|
||||||
ifi := ips[i].Interface
|
first, second := ips[i], ips[j]
|
||||||
ifj := ips[j].Interface
|
|
||||||
|
|
||||||
// ethernet is preferred
|
// ethernet is preferred
|
||||||
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
|
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
|
||||||
return vi
|
return vi
|
||||||
}
|
}
|
||||||
|
|
||||||
ipi := ips[i].IP
|
|
||||||
ipj := ips[j].IP
|
|
||||||
|
|
||||||
// IPv4 is preferred
|
// IPv4 is preferred
|
||||||
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
|
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
|
||||||
return vi
|
return vi
|
||||||
}
|
}
|
||||||
|
|
||||||
// en0 is preferred to en1
|
// en0 is preferred to en1
|
||||||
if ifi.Name != ifj.Name {
|
if first.Name != second.Name {
|
||||||
return ifi.Name < ifj.Name
|
return first.Name < second.Name
|
||||||
}
|
}
|
||||||
|
|
||||||
// fallback
|
// fallback
|
||||||
return ipi.String() < ipj.String()
|
return first.String() < second.String()
|
||||||
})
|
})
|
||||||
return ips[0].IP
|
return ips[0].IP
|
||||||
}
|
}
|
||||||
+244
-13
@@ -7,20 +7,33 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"maps"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
"github.com/distribution/reference"
|
"github.com/distribution/reference"
|
||||||
"github.com/docker/cli/cli/config"
|
|
||||||
"github.com/moby/moby/api/types/registry"
|
"github.com/moby/moby/api/types/registry"
|
||||||
)
|
)
|
||||||
|
|
||||||
func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfig, error) {
|
func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfig, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
// config.LoadDefaultConfigFile panics on nil io.Writer when the config
|
cfg, err := loadDockerConfig(ctx)
|
||||||
// file is malformed; use config.Load to route errors through the logger.
|
|
||||||
cfg, err := config.Load(config.Dir())
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf("Could not load docker config: %v", err)
|
logger.Warnf("Could not load docker config: %v", err)
|
||||||
return registry.AuthConfig{}, err
|
return registry.AuthConfig{}, err
|
||||||
@@ -34,32 +47,27 @@ func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
authConfig, err := cfg.GetAuthConfig(registryKey)
|
authConfig, err := cfg.authConfig(ctx, registryKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf("Could not get auth config from docker config: %v", err)
|
logger.Warnf("Could not get auth config from docker config: %v", err)
|
||||||
return registry.AuthConfig{}, err
|
return registry.AuthConfig{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return registry.AuthConfig(authConfig), nil
|
return authConfig, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadDockerAuthConfigs(ctx context.Context) map[string]registry.AuthConfig {
|
func LoadDockerAuthConfigs(ctx context.Context) map[string]registry.AuthConfig {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
cfg, err := config.Load(config.Dir())
|
cfg, err := loadDockerConfig(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf("Could not load docker config: %v", err)
|
logger.Warnf("Could not load docker config: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
creds, err := cfg.GetAllCredentials()
|
authConfigs, err := cfg.allAuthConfigs(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf("Could not get docker auth configs: %v", err)
|
logger.Warnf("Could not get docker auth configs: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
authConfigs := make(map[string]registry.AuthConfig, len(creds))
|
|
||||||
for k, v := range creds {
|
|
||||||
authConfigs[k] = registry.AuthConfig(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return authConfigs
|
return authConfigs
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,3 +77,226 @@ func registryAuthConfigKey(domainName string) string {
|
|||||||
}
|
}
|
||||||
return domainName
|
return domainName
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type dockerConfig struct {
|
||||||
|
Auths map[string]registry.AuthConfig `json:"auths"`
|
||||||
|
CredsStore string `json:"credsStore"`
|
||||||
|
CredHelpers map[string]string `json:"credHelpers"`
|
||||||
|
env map[string]registry.AuthConfig // from DOCKER_AUTH_CONFIG, nil when unset or unusable
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadDockerConfig(ctx context.Context) (*dockerConfig, error) {
|
||||||
|
dir := os.Getenv("DOCKER_CONFIG")
|
||||||
|
if dir == "" {
|
||||||
|
home, _ := os.UserHomeDir()
|
||||||
|
if home == "" && runtime.GOOS != "windows" {
|
||||||
|
if current, err := user.Current(); err == nil {
|
||||||
|
home = current.HomeDir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dir = filepath.Join(home, ".docker")
|
||||||
|
}
|
||||||
|
cfg := &dockerConfig{}
|
||||||
|
filename := filepath.Join(dir, "config.json")
|
||||||
|
file, err := os.Open(filename)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("loading config file: %w", err)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
defer file.Close()
|
||||||
|
if err := json.NewDecoder(file).Decode(cfg); err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for address, auth := range cfg.Auths {
|
||||||
|
if auth.Auth != "" {
|
||||||
|
if auth.Username, auth.Password, err = decodeAuth(auth.Auth); err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
auth.Auth, auth.ServerAddress = "", address
|
||||||
|
cfg.Auths[address] = auth
|
||||||
|
}
|
||||||
|
if value := os.Getenv("DOCKER_AUTH_CONFIG"); value != "" {
|
||||||
|
if cfg.env, err = parseDockerAuthConfigEnv(value); err != nil {
|
||||||
|
common.Logger(ctx).Warnf("Failed to create credential store from DOCKER_AUTH_CONFIG: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAuth(auth string) (string, string, error) {
|
||||||
|
decoded, err := base64.StdEncoding.DecodeString(auth)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
username, password, ok := strings.Cut(string(decoded), ":")
|
||||||
|
if !ok || username == "" {
|
||||||
|
return "", "", errors.New("invalid auth configuration file")
|
||||||
|
}
|
||||||
|
return username, strings.Trim(password, "\x00"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDockerAuthConfigEnv(value string) (map[string]registry.AuthConfig, error) {
|
||||||
|
var env struct {
|
||||||
|
Auths map[string]struct {
|
||||||
|
Auth string `json:"auth"`
|
||||||
|
} `json:"auths"`
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(value))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(&env); err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if decoder.More() {
|
||||||
|
return nil, errors.New("DOCKER_AUTH_CONFIG does not support more than one JSON object")
|
||||||
|
}
|
||||||
|
auths := make(map[string]registry.AuthConfig, len(env.Auths))
|
||||||
|
for address, entry := range env.Auths {
|
||||||
|
if entry.Auth == "" {
|
||||||
|
return nil, fmt.Errorf("DOCKER_AUTH_CONFIG environment variable is missing key `auth` for %s", address)
|
||||||
|
}
|
||||||
|
username, password, err := decodeAuth(entry.Auth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
auths[address] = registry.AuthConfig{Username: username, Password: password, ServerAddress: address}
|
||||||
|
}
|
||||||
|
return auths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dockerConfig) authConfig(ctx context.Context, key string) (registry.AuthConfig, error) {
|
||||||
|
if auth, ok := c.env[key]; ok {
|
||||||
|
return auth, nil
|
||||||
|
}
|
||||||
|
helper := c.CredsStore
|
||||||
|
if named, ok := c.CredHelpers[key]; ok {
|
||||||
|
helper = named
|
||||||
|
}
|
||||||
|
if helper == "" {
|
||||||
|
return c.fileAuth(key), nil
|
||||||
|
}
|
||||||
|
auth := c.fileAuth(key)
|
||||||
|
stored, err := helperCredentials(ctx, helper, key)
|
||||||
|
if err != nil {
|
||||||
|
return auth, err
|
||||||
|
}
|
||||||
|
auth.Username, auth.Password, auth.IdentityToken, auth.ServerAddress = stored.Username, stored.Password, stored.IdentityToken, stored.ServerAddress
|
||||||
|
return auth, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dockerConfig) fileAuth(key string) registry.AuthConfig {
|
||||||
|
if auth, ok := c.Auths[key]; ok {
|
||||||
|
return auth
|
||||||
|
}
|
||||||
|
for address, auth := range c.Auths {
|
||||||
|
if registryHostname(address) == key {
|
||||||
|
return auth
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return registry.AuthConfig{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dockerConfig) allAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
|
||||||
|
var auths map[string]registry.AuthConfig
|
||||||
|
if c.CredsStore == "" {
|
||||||
|
auths = maps.Clone(c.Auths)
|
||||||
|
} else {
|
||||||
|
var err error
|
||||||
|
if auths, err = c.helperAuthConfigs(ctx); err != nil {
|
||||||
|
if c.env == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
common.Logger(ctx).Warnf("Could not list the credentials of docker-credential-%s: %v", c.CredsStore, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if auths == nil {
|
||||||
|
auths = map[string]registry.AuthConfig{}
|
||||||
|
}
|
||||||
|
maps.Copy(auths, c.env)
|
||||||
|
if c.CredsStore == "" {
|
||||||
|
c.Auths = maps.Clone(auths) // docker's file store holds the DOCKER_AUTH_CONFIG entries after listing, and credHelpers set to "" read it
|
||||||
|
}
|
||||||
|
for address := range c.CredHelpers {
|
||||||
|
auth, err := c.authConfig(ctx, registryAuthConfigKey(address))
|
||||||
|
if err != nil {
|
||||||
|
common.Logger(ctx).Warnf("Failed to get credentials for registry: %s: %v", address, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
auths[address] = auth
|
||||||
|
}
|
||||||
|
return auths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dockerConfig) helperAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
|
||||||
|
out, err := runCredentialHelper(ctx, c.CredsStore, "list", "unused")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error listing credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
var addresses map[string]string
|
||||||
|
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&addresses); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
auths := make(map[string]registry.AuthConfig, len(addresses))
|
||||||
|
for address := range addresses {
|
||||||
|
stored, err := helperCredentials(ctx, c.CredsStore, address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
auth := c.Auths[address]
|
||||||
|
auth.Username, auth.Password, auth.IdentityToken = stored.Username, stored.Password, stored.IdentityToken
|
||||||
|
if auth.ServerAddress == "" {
|
||||||
|
auth.ServerAddress = stored.ServerAddress
|
||||||
|
}
|
||||||
|
auths[address] = auth
|
||||||
|
}
|
||||||
|
return auths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func helperCredentials(ctx context.Context, helper, address string) (registry.AuthConfig, error) {
|
||||||
|
out, err := runCredentialHelper(ctx, helper, "get", address)
|
||||||
|
if err != nil {
|
||||||
|
if strings.TrimSpace(string(out)) == "credentials not found in native keychain" {
|
||||||
|
return registry.AuthConfig{}, nil
|
||||||
|
}
|
||||||
|
return registry.AuthConfig{}, fmt.Errorf("error getting credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
var stored struct{ Username, Secret string }
|
||||||
|
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&stored); err != nil {
|
||||||
|
return registry.AuthConfig{}, err
|
||||||
|
}
|
||||||
|
if stored.Username == "<token>" {
|
||||||
|
return registry.AuthConfig{IdentityToken: stored.Secret, ServerAddress: address}, nil
|
||||||
|
}
|
||||||
|
return registry.AuthConfig{Username: stored.Username, Password: stored.Secret, ServerAddress: address}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCredentialHelper(ctx context.Context, helper, action, input string) ([]byte, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, "docker-credential-"+helper, action)
|
||||||
|
cmd.Stdin = strings.NewReader(input)
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
return cmd.Output()
|
||||||
|
}
|
||||||
|
|
||||||
|
func registryHostname(address string) string {
|
||||||
|
if _, rest, ok := strings.Cut(address, "://"); ok {
|
||||||
|
if u, err := url.Parse(address); err == nil && u.Hostname() != "" {
|
||||||
|
if u.Port() == "" {
|
||||||
|
return u.Hostname()
|
||||||
|
}
|
||||||
|
return net.JoinHostPort(u.Hostname(), u.Port())
|
||||||
|
}
|
||||||
|
if host, _, _ := strings.Cut(rest, "/"); host != "" {
|
||||||
|
if colon := strings.LastIndex(host, ":"); strings.Count(host, ":") > 1 && !strings.HasPrefix(host, "[") && isDigits(host[colon+1:]) {
|
||||||
|
return net.JoinHostPort(host[:colon], host[colon+1:])
|
||||||
|
}
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
host, _, _ := strings.Cut(address, "/")
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDigits(s string) bool {
|
||||||
|
return s != "" && strings.Trim(s, "0123456789") == ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/moby/moby/api/types/registry"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadDockerAuthConfigFromFileHelpersAndEnv(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "docker-credential-fake"), []byte("#!/bin/sh\nread -r server\n[ \"$1\" = get ] && [ \"$server\" = helper.example.com ] && exec echo '{\"Username\":\"helped\",\"Secret\":\"pw\"}'\necho 'credentials not found in native keychain'\nexit 1\n"), 0o755))
|
||||||
|
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
auth := func(credentials string) string { return base64.StdEncoding.EncodeToString([]byte(credentials)) }
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths": {"https://legacy.example.com/v1/": {"auth": "`+auth("legacy:pw")+`"}}, "credHelpers": {"helper.example.com": "fake", "gone.example.com": "fake", "env-legacy.example.com": ""}}`), 0o600))
|
||||||
|
t.Setenv("DOCKER_CONFIG", dir)
|
||||||
|
t.Setenv("DOCKER_AUTH_CONFIG", `{"auths": {"env.example.com": {"auth": "`+auth("env:pw")+`"}, "https://env-legacy.example.com/v1/": {"auth": "`+auth("env:pw")+`"}}}`)
|
||||||
|
|
||||||
|
for image, want := range map[string]registry.AuthConfig{
|
||||||
|
"legacy.example.com/app": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
|
||||||
|
"helper.example.com/app": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
|
||||||
|
"env.example.com/app": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
|
||||||
|
"gone.example.com/app": {},
|
||||||
|
} {
|
||||||
|
got, err := LoadDockerAuthConfig(t.Context(), image)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, got, image)
|
||||||
|
}
|
||||||
|
assert.Equal(t, map[string]registry.AuthConfig{
|
||||||
|
"https://legacy.example.com/v1/": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
|
||||||
|
"helper.example.com": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
|
||||||
|
"gone.example.com": {},
|
||||||
|
"env.example.com": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
|
||||||
|
"https://env-legacy.example.com/v1/": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
|
||||||
|
"env-legacy.example.com": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
|
||||||
|
}, LoadDockerAuthConfigs(t.Context()))
|
||||||
|
}
|
||||||
+140
-14
@@ -7,15 +7,18 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"archive/tar"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
"github.com/moby/go-archive"
|
|
||||||
"github.com/moby/go-archive/compression"
|
|
||||||
"github.com/moby/moby/client"
|
"github.com/moby/moby/client"
|
||||||
"github.com/moby/patternmatcher"
|
"github.com/moby/patternmatcher"
|
||||||
"github.com/moby/patternmatcher/ignorefile"
|
"github.com/moby/patternmatcher/ignorefile"
|
||||||
@@ -43,9 +46,8 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
logger.Debugf("Building image from '%v'", input.ContextDir)
|
logger.Debugf("Building image from '%v'", input.ContextDir)
|
||||||
|
|
||||||
tags := []string{input.ImageTag}
|
|
||||||
options := client.ImageBuildOptions{
|
options := client.ImageBuildOptions{
|
||||||
Tags: tags,
|
Tags: []string{input.ImageTag},
|
||||||
Remove: true,
|
Remove: true,
|
||||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||||
Dockerfile: input.Dockerfile,
|
Dockerfile: input.Dockerfile,
|
||||||
@@ -72,12 +74,10 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
||||||
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
||||||
|
|
||||||
err = logDockerResponse(logger, resp.Body, err != nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return logDockerResponse(logger, resp.Body, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,14 +115,140 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
|||||||
includes = append(includes, ".dockerignore", relDockerfile)
|
includes = append(includes, ".dockerignore", relDockerfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
buildCtx, err := archive.TarWithOptions(contextDir, &archive.TarOptions{
|
pm, err := patternmatcher.New(excludes)
|
||||||
Compression: compression.None,
|
|
||||||
ExcludePatterns: excludes,
|
|
||||||
IncludeFiles: includes,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
reader, writer := io.Pipe()
|
||||||
return buildCtx, nil
|
go func() { writer.CloseWithError(writeBuildContext(ctx, writer, contextDir, includes, pm)) }()
|
||||||
|
return reader, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||||
|
tw := tar.NewWriter(w)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
hardlinks := map[uint64]string{}
|
||||||
|
buffer := make([]byte, 32*1024)
|
||||||
|
for _, include := range includes {
|
||||||
|
include = filepath.FromSlash(include)
|
||||||
|
var parents []parentMatch
|
||||||
|
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
||||||
|
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return leaveOut(ctx, file, err)
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(contextDir, file)
|
||||||
|
if err != nil || rel == "." {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if rel != include {
|
||||||
|
for len(parents) > 0 && !strings.HasPrefix(rel, parents[len(parents)-1].dir+string(filepath.Separator)) {
|
||||||
|
parents = parents[:len(parents)-1]
|
||||||
|
}
|
||||||
|
var parentInfo patternmatcher.MatchInfo
|
||||||
|
if len(parents) > 0 {
|
||||||
|
parentInfo = parents[len(parents)-1].info
|
||||||
|
}
|
||||||
|
skip, info, err := pm.MatchesUsingParentResults(rel, parentInfo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
parents = append(parents, parentMatch{rel, info})
|
||||||
|
}
|
||||||
|
if skip {
|
||||||
|
if entry.IsDir() && !hasExceptionWithin(pm, rel) {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if seen[rel] {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen[rel] = true
|
||||||
|
return addBuildContextEntry(ctx, tw, file, rel, entry, hardlinks, buffer)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tw.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
type parentMatch struct {
|
||||||
|
dir string
|
||||||
|
info patternmatcher.MatchInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasExceptionWithin(pm *patternmatcher.PatternMatcher, dir string) bool {
|
||||||
|
dirPrefix := dir + string(filepath.Separator)
|
||||||
|
for _, pattern := range pm.Patterns() {
|
||||||
|
if pattern.Exclusion() && strings.HasPrefix(pattern.String()+string(filepath.Separator), dirPrefix) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func leaveOut(ctx context.Context, file string, err error) error {
|
||||||
|
common.Logger(ctx).Warnf("Leaving %s out of the build context: %v", file, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string, entry fs.DirEntry, hardlinks map[uint64]string, buffer []byte) error {
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return leaveOut(ctx, file, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&fs.ModeSocket != 0 {
|
||||||
|
return nil // tar has no socket type
|
||||||
|
}
|
||||||
|
var link string
|
||||||
|
if info.Mode()&fs.ModeSymlink != 0 {
|
||||||
|
if link, err = os.Readlink(file); err != nil {
|
||||||
|
return leaveOut(ctx, file, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
header.Name = filepath.ToSlash(rel)
|
||||||
|
inode, linked := hardlinkInode(info)
|
||||||
|
if target, ok := hardlinks[inode]; linked && ok {
|
||||||
|
header.Typeflag, header.Linkname, header.Size = tar.TypeLink, target, 0
|
||||||
|
}
|
||||||
|
var content *os.File
|
||||||
|
if header.Typeflag == tar.TypeReg && header.Size > 0 {
|
||||||
|
if content, err = os.Open(file); err != nil {
|
||||||
|
return leaveOut(ctx, file, err)
|
||||||
|
}
|
||||||
|
defer content.Close()
|
||||||
|
}
|
||||||
|
if linked && header.Typeflag != tar.TypeLink {
|
||||||
|
hardlinks[inode] = header.Name
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
header.Name += "/"
|
||||||
|
}
|
||||||
|
header.Format = tar.FormatPAX
|
||||||
|
header.ModTime = header.ModTime.Truncate(time.Second)
|
||||||
|
header.AccessTime, header.ChangeTime = time.Time{}, time.Time{}
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
header.Mode = header.Mode&0o755 | 0o111 // Windows has no exec bit, so mark everything executable like docker build
|
||||||
|
}
|
||||||
|
if err := tw.WriteHeader(header); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if content != nil {
|
||||||
|
_, err = io.CopyBuffer(tw, struct{ io.Reader }{content}, buffer) // hides os.File's WriteTo, which would allocate a buffer per file
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type noNameLookups struct{ fs.FileInfo } // leaves Uname and Gname empty like docker build, so packing never waits on NSS
|
||||||
|
|
||||||
|
func (noNameLookups) Uname() (string, error) { return "", nil }
|
||||||
|
|
||||||
|
func (noNameLookups) Gname() (string, error) { return "", nil }
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||||
|
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
writer.Header().Set("API-Version", "1.47")
|
||||||
|
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||||
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
|
writer.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(daemon.Close)
|
||||||
|
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||||
|
t.Setenv("DOCKER_CONFIG", t.TempDir())
|
||||||
|
t.Setenv("DOCKER_AUTH_CONFIG", "")
|
||||||
|
|
||||||
|
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||||
|
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||||
|
})(t.Context()), "build rejected")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, content := range map[string]string{
|
||||||
|
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
||||||
|
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
||||||
|
} {
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||||
|
}
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
||||||
|
|
||||||
|
contexts := []string{dir}
|
||||||
|
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||||
|
contexts = append(contexts, link+string(filepath.Separator))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, contextDir := range contexts {
|
||||||
|
buildContext, err := createBuildContext(t.Context(), contextDir, "Dockerfile")
|
||||||
|
require.NoError(t, err)
|
||||||
|
var names []string
|
||||||
|
for reader := tar.NewReader(buildContext); ; {
|
||||||
|
header, err := reader.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
require.NoError(t, err)
|
||||||
|
names = append(names, header.Name)
|
||||||
|
}
|
||||||
|
require.NoError(t, buildContext.Close())
|
||||||
|
require.Equal(t, []string{".dockerignore", "a.txt", "build/keep.txt", "empty/", "Dockerfile"}, names, contextDir)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func hardlinkInode(info fs.FileInfo) (uint64, bool) {
|
||||||
|
if stat, ok := info.Sys().(*syscall.Stat_t); ok && !info.IsDir() && stat.Nlink > 1 {
|
||||||
|
return stat.Ino, true
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCreateBuildContextKeepsHardlinkedSpecialFilesLinked(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "first"), 0o644))
|
||||||
|
require.NoError(t, os.Link(filepath.Join(dir, "first"), filepath.Join(dir, "second")))
|
||||||
|
|
||||||
|
buildContext, err := createBuildContext(t.Context(), dir, "Dockerfile")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer buildContext.Close()
|
||||||
|
reader := tar.NewReader(buildContext)
|
||||||
|
first, err := reader.Next()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, byte(tar.TypeFifo), first.Typeflag)
|
||||||
|
second, err := reader.Next()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, byte(tar.TypeLink), second.Typeflag)
|
||||||
|
assert.Equal(t, first.Name, second.Linkname)
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import "io/fs"
|
||||||
|
|
||||||
|
func hardlinkInode(_info fs.FileInfo) (uint64, bool) { return 0, false }
|
||||||
+23
-60
@@ -4,11 +4,12 @@
|
|||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||||
|
|
||||||
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||||
// * appended with license information
|
// * appended with license information
|
||||||
// * regexp and loader.ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
|
// * regexp and ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
|
||||||
|
// * cdiDeviceName in place of the container-device-interface parser's IsQualifiedName
|
||||||
// * jsontext in place of json.Compact for the seccomp profile
|
// * jsontext in place of json.Compact for the seccomp profile
|
||||||
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
|
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for nat port types
|
||||||
//
|
//
|
||||||
// docker/cli is licensed under the Apache License, Version 2.0.
|
// docker/cli is licensed under the Apache License, Version 2.0.
|
||||||
// See DOCKER_LICENSE for the full license text.
|
// See DOCKER_LICENSE for the full license text.
|
||||||
@@ -33,14 +34,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
"github.com/docker/cli/cli/compose/loader"
|
|
||||||
"github.com/docker/cli/opts"
|
"github.com/docker/cli/opts"
|
||||||
"github.com/docker/go-connections/nat"
|
"github.com/docker/go-connections/nat"
|
||||||
"github.com/moby/moby/api/types/container"
|
"github.com/moby/moby/api/types/container"
|
||||||
"github.com/moby/moby/api/types/mount"
|
"github.com/moby/moby/api/types/mount"
|
||||||
"github.com/moby/moby/api/types/network"
|
"github.com/moby/moby/api/types/network"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
cdi "tags.cncf.io/container-device-interface/pkg/parser"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -155,9 +154,6 @@ type containerOptions struct {
|
|||||||
init bool
|
init bool
|
||||||
umask opts.UmaskOpt
|
umask opts.UmaskOpt
|
||||||
annotations *opts.MapOpts
|
annotations *opts.MapOpts
|
||||||
|
|
||||||
Image string
|
|
||||||
Args []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// addFlags adds all command line flags that will be used by parse to the FlagSet
|
// addFlags adds all command line flags that will be used by parse to the FlagSet
|
||||||
@@ -388,7 +384,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
volumes := copts.volumes.GetMap()
|
volumes := copts.volumes.GetMap()
|
||||||
// add any bind targets to the list of container volumes
|
// add any bind targets to the list of container volumes
|
||||||
for bind := range volumes {
|
for bind := range volumes {
|
||||||
parsed, err := loader.ParseVolume(bind)
|
parsed, err := ParseVolume(bind)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -396,7 +392,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
if parsed.Source != "" {
|
if parsed.Source != "" {
|
||||||
toBind := bind
|
toBind := bind
|
||||||
|
|
||||||
if parsed.Type == string(mount.TypeBind) {
|
if parsed.Type == mount.TypeBind {
|
||||||
if hostPart, targetPath, ok := strings.Cut(bind, ":"); ok {
|
if hostPart, targetPath, ok := strings.Cut(bind, ":"); ok {
|
||||||
if !filepath.IsAbs(hostPart) && strings.HasPrefix(hostPart, ".") {
|
if !filepath.IsAbs(hostPart) && strings.HasPrefix(hostPart, ".") {
|
||||||
if absHostPart, err := filepath.Abs(hostPart); err == nil {
|
if absHostPart, err := filepath.Abs(hostPart); err == nil {
|
||||||
@@ -423,12 +419,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
tmpfs[k] = v
|
tmpfs[k] = v
|
||||||
}
|
}
|
||||||
|
|
||||||
var runCmd, entrypoint []string
|
var entrypoint []string
|
||||||
|
|
||||||
if len(copts.Args) > 0 {
|
|
||||||
runCmd = copts.Args
|
|
||||||
}
|
|
||||||
|
|
||||||
if copts.entrypoint != "" {
|
if copts.entrypoint != "" {
|
||||||
entrypoint = []string{copts.entrypoint}
|
entrypoint = []string{copts.entrypoint}
|
||||||
} else if flags.Changed("entrypoint") {
|
} else if flags.Changed("entrypoint") {
|
||||||
@@ -449,36 +440,15 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
portBindings := network.PortMap{}
|
portBindings, err := convertPortMap(natPortBindings)
|
||||||
for port, bindings := range natPortBindings {
|
if err != nil {
|
||||||
p, err := network.ParsePort(string(port))
|
return nil, err
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
portBindings[p] = []network.PortBinding{}
|
|
||||||
for _, b := range bindings {
|
|
||||||
var hostIP netip.Addr
|
|
||||||
if b.HostIP != "" {
|
|
||||||
hostIP, err = netip.ParseAddr(b.HostIP)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
portBindings[p] = append(portBindings[p], network.PortBinding{
|
|
||||||
HostIP: hostIP,
|
|
||||||
HostPort: b.HostPort,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add published ports as exposed ports.
|
// Add published ports as exposed ports.
|
||||||
exposedPorts := network.PortSet{}
|
exposedPorts, err := convertPortSet(ports)
|
||||||
for port := range ports {
|
if err != nil {
|
||||||
p, err := network.ParsePort(string(port))
|
return nil, err
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
exposedPorts[p] = struct{}{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Merge in exposed ports to the map of published ports
|
// Merge in exposed ports to the map of published ports
|
||||||
@@ -504,7 +474,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
deviceMappings := make([]container.DeviceMapping, 0, len(devices))
|
deviceMappings := make([]container.DeviceMapping, 0, len(devices))
|
||||||
cdiDeviceNames := make([]string, 0, len(devices))
|
cdiDeviceNames := make([]string, 0, len(devices))
|
||||||
for _, device := range devices {
|
for _, device := range devices {
|
||||||
if cdi.IsQualifiedName(device) {
|
if cdiDeviceName.MatchString(device) {
|
||||||
cdiDeviceNames = append(cdiDeviceNames, device)
|
cdiDeviceNames = append(cdiDeviceNames, device)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -619,11 +589,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
|
|
||||||
deviceRequests := copts.gpus.Value()
|
deviceRequests := copts.gpus.Value()
|
||||||
if len(cdiDeviceNames) > 0 {
|
if len(cdiDeviceNames) > 0 {
|
||||||
cdiDeviceRequest := container.DeviceRequest{
|
deviceRequests = append(deviceRequests, container.DeviceRequest{
|
||||||
Driver: "cdi",
|
Driver: "cdi",
|
||||||
DeviceIDs: cdiDeviceNames,
|
DeviceIDs: cdiDeviceNames,
|
||||||
}
|
})
|
||||||
deviceRequests = append(deviceRequests, cdiDeviceRequest)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resources := container.Resources{
|
resources := container.Resources{
|
||||||
@@ -669,8 +638,6 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
AttachStdout: attachStdout,
|
AttachStdout: attachStdout,
|
||||||
AttachStderr: attachStderr,
|
AttachStderr: attachStderr,
|
||||||
Env: envVariables,
|
Env: envVariables,
|
||||||
Cmd: runCmd,
|
|
||||||
Image: copts.Image,
|
|
||||||
Volumes: volumes,
|
Volumes: volumes,
|
||||||
Entrypoint: entrypoint,
|
Entrypoint: entrypoint,
|
||||||
WorkingDir: copts.workingDir,
|
WorkingDir: copts.workingDir,
|
||||||
@@ -811,10 +778,8 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
|||||||
// For backward compatibility: if no custom options are provided for the network,
|
// For backward compatibility: if no custom options are provided for the network,
|
||||||
// and only a single network is specified, omit the endpoint-configuration
|
// and only a single network is specified, omit the endpoint-configuration
|
||||||
// on the client (the daemon will still create it when creating the container)
|
// on the client (the daemon will still create it when creating the container)
|
||||||
if i == 0 && len(copts.netMode.Value()) == 1 {
|
if i == 0 && len(copts.netMode.Value()) == 1 && reflect.ValueOf(*ep).IsZero() {
|
||||||
if ep == nil || reflect.ValueOf(*ep).IsZero() {
|
continue
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
endpoints[n.Target] = ep
|
endpoints[n.Target] = ep
|
||||||
}
|
}
|
||||||
@@ -845,14 +810,12 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
|||||||
return invalidParameter(errors.New("conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses"))
|
return invalidParameter(errors.New("conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses"))
|
||||||
}
|
}
|
||||||
if copts.aliases.Len() > 0 {
|
if copts.aliases.Len() > 0 {
|
||||||
n.Aliases = make([]string, copts.aliases.Len())
|
n.Aliases = slices.Clone(copts.aliases.GetSlice())
|
||||||
copy(n.Aliases, copts.aliases.GetSlice())
|
|
||||||
}
|
}
|
||||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||||
// for the default bridge it defines a legacy-link.
|
// for the default bridge it defines a legacy-link.
|
||||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
||||||
n.Links = make([]string, copts.links.Len())
|
n.Links = slices.Clone(copts.links.GetSlice())
|
||||||
copy(n.Links, copts.links.GetSlice())
|
|
||||||
}
|
}
|
||||||
if copts.ipv4Address != nil {
|
if copts.ipv4Address != nil {
|
||||||
if ipv4, ok := netip.AddrFromSlice(copts.ipv4Address.To4()); ok {
|
if ipv4, ok := netip.AddrFromSlice(copts.ipv4Address.To4()); ok {
|
||||||
@@ -891,7 +854,6 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
|||||||
}
|
}
|
||||||
epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
|
epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
|
||||||
if len(ep.DriverOpts) > 0 {
|
if len(ep.DriverOpts) > 0 {
|
||||||
epConfig.DriverOpts = make(map[string]string)
|
|
||||||
epConfig.DriverOpts = ep.DriverOpts
|
epConfig.DriverOpts = ep.DriverOpts
|
||||||
}
|
}
|
||||||
if len(ep.Links) > 0 {
|
if len(ep.Links) > 0 {
|
||||||
@@ -935,11 +897,10 @@ func convertToStandardNotation(ports []string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
|
func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
|
||||||
loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts)
|
|
||||||
if loggingDriver == "none" && len(loggingOpts) > 0 {
|
if loggingDriver == "none" && len(loggingOpts) > 0 {
|
||||||
return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver)
|
return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver)
|
||||||
}
|
}
|
||||||
return loggingOptsMap, nil
|
return opts.ConvertKVStringsToMap(loggingOpts), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// takes a local seccomp daemon, reads the file contents for sending to the daemon
|
// takes a local seccomp daemon, reads the file contents for sending to the daemon
|
||||||
@@ -1197,6 +1158,8 @@ func convertPortSet(ports nat.PortSet) (network.PortSet, error) {
|
|||||||
return converted, nil
|
return converted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var cdiDeviceName = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?/[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?=[a-zA-Z0-9]([a-zA-Z0-9_.:-]*[a-zA-Z0-9])?$`)
|
||||||
|
|
||||||
func convertPortMap(bindings nat.PortMap) (network.PortMap, error) {
|
func convertPortMap(bindings nat.PortMap) (network.PortMap, error) {
|
||||||
converted := make(network.PortMap, len(bindings))
|
converted := make(network.PortMap, len(bindings))
|
||||||
for port, portBindings := range bindings {
|
for port, portBindings := range bindings {
|
||||||
|
|||||||
+219
-360
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
// This file is exact copy of https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
|
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
|
||||||
// * appended with license information
|
// * appended with license information
|
||||||
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
|
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
|
||||||
//
|
//
|
||||||
@@ -25,13 +25,11 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp/cmpopts"
|
|
||||||
"github.com/moby/moby/api/types/container"
|
"github.com/moby/moby/api/types/container"
|
||||||
networktypes "github.com/moby/moby/api/types/network"
|
networktypes "github.com/moby/moby/api/types/network"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"gotest.tools/v3/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
is "gotest.tools/v3/assert/cmp"
|
"github.com/stretchr/testify/require"
|
||||||
"gotest.tools/v3/skip"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func mustParseMAC(s string) networktypes.HardwareAddr {
|
func mustParseMAC(s string) networktypes.HardwareAddr {
|
||||||
@@ -71,7 +69,6 @@ func parseRun(args []string) (*container.Config, *container.HostConfig, *network
|
|||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
// TODO(dnephin): fix tests to accept ContainerConfig; see https://github.com/moby/moby/pull/31621
|
|
||||||
containerCfg, err := parse(flags, copts, runtime.GOOS)
|
containerCfg, err := parse(flags, copts, runtime.GOOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
@@ -83,14 +80,13 @@ func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
|
|||||||
flags := pflag.NewFlagSet("run", pflag.ContinueOnError)
|
flags := pflag.NewFlagSet("run", pflag.ContinueOnError)
|
||||||
flags.SetOutput(io.Discard)
|
flags.SetOutput(io.Discard)
|
||||||
flags.Usage = nil
|
flags.Usage = nil
|
||||||
copts := addFlags(flags)
|
return flags, addFlags(flags)
|
||||||
return flags, copts
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
|
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
config, hostConfig, nwConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash"))
|
config, hostConfig, nwConfig, err := parseRun(strings.Split(args, " "))
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
return config, hostConfig, nwConfig
|
return config, hostConfig, nwConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,12 +152,12 @@ func TestParseRunLinks(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, hostConfig, netConfig := mustParse(t, tc.input)
|
_, hostConfig, netConfig := mustParse(t, tc.input)
|
||||||
assert.Check(t, is.DeepEqual(hostConfig.Links, tc.expHostConfigLinks))
|
assert.Equal(t, tc.expHostConfigLinks, hostConfig.Links)
|
||||||
assert.Check(t, is.Len(netConfig.EndpointsConfig, len(tc.expNetConfigLinks)))
|
assert.Len(t, netConfig.EndpointsConfig, len(tc.expNetConfigLinks))
|
||||||
for netName, expLinks := range tc.expNetConfigLinks {
|
for netName, expLinks := range tc.expNetConfigLinks {
|
||||||
nc, ok := netConfig.EndpointsConfig[netName]
|
nc, ok := netConfig.EndpointsConfig[netName]
|
||||||
assert.Assert(t, ok)
|
require.True(t, ok)
|
||||||
assert.Check(t, is.DeepEqual(nc.Links, expLinks))
|
assert.Equal(t, expLinks, nc.Links)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -169,65 +165,33 @@ func TestParseRunLinks(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseRunWithoutUmask(t *testing.T) {
|
func TestParseRunWithoutUmask(t *testing.T) {
|
||||||
_, hostConfig, _, err := parseRun([]string{"ubuntu", "bash"})
|
_, hostConfig, _, err := parseRun([]string{"ubuntu", "bash"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Assert(t, hostConfig.Umask == nil)
|
require.Nil(t, hostConfig.Umask)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunUmask(t *testing.T) {
|
func TestParseRunUmask(t *testing.T) {
|
||||||
_, hostConfig, _, err := parseRun([]string{"--umask", "0022", "ubuntu", "bash"})
|
_, hostConfig, _, err := parseRun([]string{"--umask", "0022", "ubuntu", "bash"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Assert(t, hostConfig.Umask != nil)
|
require.NotNil(t, hostConfig.Umask)
|
||||||
assert.Equal(t, uint32(0o22), *hostConfig.Umask)
|
require.Equal(t, uint32(0o22), *hostConfig.Umask)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunAttach(t *testing.T) {
|
func TestParseRunAttach(t *testing.T) {
|
||||||
tests := []struct {
|
for _, tc := range []struct {
|
||||||
input string
|
input string
|
||||||
expected container.Config
|
stdin, stdout, stderr bool
|
||||||
}{
|
}{
|
||||||
{
|
{"", false, true, true},
|
||||||
input: "",
|
{"-i", true, true, true},
|
||||||
expected: container.Config{
|
{"-a stdin", true, false, false},
|
||||||
AttachStdout: true,
|
{"-a stdin -a stdout", true, true, false},
|
||||||
AttachStderr: true,
|
{"-a stdin -a stdout -a stderr", true, true, true},
|
||||||
},
|
} {
|
||||||
},
|
|
||||||
{
|
|
||||||
input: "-i",
|
|
||||||
expected: container.Config{
|
|
||||||
AttachStdin: true,
|
|
||||||
AttachStdout: true,
|
|
||||||
AttachStderr: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
input: "-a stdin",
|
|
||||||
expected: container.Config{
|
|
||||||
AttachStdin: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
input: "-a stdin -a stdout",
|
|
||||||
expected: container.Config{
|
|
||||||
AttachStdin: true,
|
|
||||||
AttachStdout: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
input: "-a stdin -a stdout -a stderr",
|
|
||||||
expected: container.Config{
|
|
||||||
AttachStdin: true,
|
|
||||||
AttachStdout: true,
|
|
||||||
AttachStderr: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.input, func(t *testing.T) {
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
config, _, _ := mustParse(t, tc.input)
|
config, _, _ := mustParse(t, tc.input)
|
||||||
assert.Equal(t, config.AttachStdin, tc.expected.AttachStdin)
|
require.Equal(t, tc.stdin, config.AttachStdin)
|
||||||
assert.Equal(t, config.AttachStdout, tc.expected.AttachStdout)
|
require.Equal(t, tc.stdout, config.AttachStdout)
|
||||||
assert.Equal(t, config.AttachStderr, tc.expected.AttachStderr)
|
require.Equal(t, tc.stderr, config.AttachStderr)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -273,109 +237,52 @@ func TestParseRunWithInvalidArgs(t *testing.T) {
|
|||||||
flags, _ := setupRunFlags()
|
flags, _ := setupRunFlags()
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||||
assert.Error(t, flags.Parse(tc.args), tc.error)
|
require.EqualError(t, flags.Parse(tc.args), tc.error)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseWithVolumes(t *testing.T) { //nolint:gocyclo // verbatim copy from docker/cli tests
|
func TestParseWithVolumes(t *testing.T) {
|
||||||
// A single volume
|
for _, tc := range []struct {
|
||||||
arr, tryit := setupPlatformVolume([]string{`/tmp`}, []string{`c:\tmp`})
|
unix, windows []string
|
||||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
binds int
|
||||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
}{
|
||||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
{[]string{`/tmp`}, []string{`c:\tmp`}, 0},
|
||||||
t.Fatalf("Error parsing volume flags, %q is missing from volumes. Received %v", tryit, config.Volumes)
|
{[]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`}, 0},
|
||||||
}
|
{[]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`}, 1},
|
||||||
|
{[]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`}, 2},
|
||||||
// Two volumes
|
{[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`}, []string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`}, 2},
|
||||||
arr, tryit = setupPlatformVolume([]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`})
|
{[]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, nil, 2},
|
||||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
{[]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, nil, 2},
|
||||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
{[]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`}, 1},
|
||||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
{nil, []string{os.Getenv("SystemDrive") + `\:d:`}, 1},
|
||||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[0], config.Volumes)
|
} {
|
||||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
volumes := tc.unix
|
||||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[1], config.Volumes)
|
if runtime.GOOS == "windows" {
|
||||||
}
|
volumes = tc.windows
|
||||||
|
|
||||||
// A single bind mount
|
|
||||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`})
|
|
||||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || hostConfig.Binds[0] != arr[0] {
|
|
||||||
t.Fatalf("Error parsing volume flags, %q should mount-bind the path before the colon into the path after the colon. Received %v %v", arr[0], hostConfig.Binds, config.Volumes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two bind mounts.
|
|
||||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`})
|
|
||||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
|
||||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two bind mounts, first read-only, second read-write.
|
|
||||||
// TODO Windows: The Windows version uses read-write as that's the only mode it supports. Can change this post TP4
|
|
||||||
arr, tryit = setupPlatformVolume(
|
|
||||||
[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`},
|
|
||||||
[]string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`})
|
|
||||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
|
||||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Similar to previous test but with alternate modes which are only supported by Linux
|
|
||||||
if runtime.GOOS != "windows" {
|
|
||||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, []string{})
|
|
||||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
|
||||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
|
||||||
}
|
}
|
||||||
|
if len(volumes) == 0 {
|
||||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, []string{})
|
continue
|
||||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
|
||||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
|
||||||
}
|
}
|
||||||
|
t.Run(strings.Join(volumes, ","), func(t *testing.T) {
|
||||||
|
var args []string
|
||||||
|
for _, volume := range volumes {
|
||||||
|
args = append(args, "-v", volume)
|
||||||
|
}
|
||||||
|
config, hostConfig, _, err := parseRun(args)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if tc.binds == 0 {
|
||||||
|
assert.Nil(t, hostConfig.Binds)
|
||||||
|
} else {
|
||||||
|
assert.ElementsMatch(t, volumes[:tc.binds], hostConfig.Binds)
|
||||||
|
}
|
||||||
|
expectedVolumes := map[string]struct{}{}
|
||||||
|
for _, volume := range volumes[tc.binds:] {
|
||||||
|
expectedVolumes[volume] = struct{}{}
|
||||||
|
}
|
||||||
|
assert.Equal(t, expectedVolumes, config.Volumes)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// One bind mount and one volume
|
|
||||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`})
|
|
||||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] {
|
|
||||||
t.Fatalf("Error parsing volume flags, %s and %s should only one and only one bind mount %s. Received %s", arr[0], arr[1], arr[0], hostConfig.Binds)
|
|
||||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
|
||||||
t.Fatalf("Error parsing volume flags %s and %s. %s is missing from volumes. Received %v", arr[0], arr[1], arr[1], config.Volumes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Root to non-c: drive letter (Windows specific)
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
arr, tryit = setupPlatformVolume([]string{}, []string{os.Getenv("SystemDrive") + `\:d:`})
|
|
||||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] || len(config.Volumes) != 0 {
|
|
||||||
t.Fatalf("Error parsing %s. Should have a single bind mount and no volumes", arr[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setupPlatformVolume takes two arrays of volume specs - a Unix style
|
|
||||||
// spec and a Windows style spec. Depending on the platform being unit tested,
|
|
||||||
// it returns one of them, along with a volume string that would be passed
|
|
||||||
// on the docker CLI (e.g. -v /bar -v /foo).
|
|
||||||
func setupPlatformVolume(u, w []string) ([]string, string) {
|
|
||||||
var a []string
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
a = w
|
|
||||||
} else {
|
|
||||||
a = u
|
|
||||||
}
|
|
||||||
s := ""
|
|
||||||
for _, v := range a {
|
|
||||||
s = s + "-v " + v + " "
|
|
||||||
}
|
|
||||||
return a, s
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if (a == c && b == d) || (a == d && b == c)
|
|
||||||
// because maps are randomized
|
|
||||||
func compareRandomizedStrings(a, b, c, d string) error {
|
|
||||||
if a == c && b == d {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if a == d && b == c {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errors.New("strings don't match")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Simple parse with MacAddress validation
|
// Simple parse with MacAddress validation
|
||||||
@@ -396,66 +303,49 @@ func TestRunFlagsParseWithMemory(t *testing.T) {
|
|||||||
flags, _ := setupRunFlags()
|
flags, _ := setupRunFlags()
|
||||||
args := []string{"--memory=invalid", "img", "cmd"}
|
args := []string{"--memory=invalid", "img", "cmd"}
|
||||||
err := flags.Parse(args)
|
err := flags.Parse(args)
|
||||||
assert.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
|
require.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
|
||||||
|
|
||||||
_, hostconfig, _ := mustParse(t, "--memory=1G")
|
_, hostconfig, _ := mustParse(t, "--memory=1G")
|
||||||
assert.Check(t, is.Equal(int64(1073741824), hostconfig.Memory))
|
assert.Equal(t, int64(1073741824), hostconfig.Memory)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseWithMemorySwap(t *testing.T) {
|
func TestParseWithMemorySwap(t *testing.T) {
|
||||||
flags, _ := setupRunFlags()
|
flags, _ := setupRunFlags()
|
||||||
args := []string{"--memory-swap=invalid", "img", "cmd"}
|
args := []string{"--memory-swap=invalid", "img", "cmd"}
|
||||||
err := flags.Parse(args)
|
err := flags.Parse(args)
|
||||||
assert.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
|
require.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
|
||||||
|
|
||||||
_, hostconfig, _ := mustParse(t, "--memory-swap=1G")
|
_, hostconfig, _ := mustParse(t, "--memory-swap=1G")
|
||||||
assert.Check(t, is.Equal(int64(1073741824), hostconfig.MemorySwap))
|
assert.Equal(t, int64(1073741824), hostconfig.MemorySwap)
|
||||||
|
|
||||||
_, hostconfig, _ = mustParse(t, "--memory-swap=-1")
|
_, hostconfig, _ = mustParse(t, "--memory-swap=-1")
|
||||||
assert.Check(t, is.Equal(int64(-1), hostconfig.MemorySwap))
|
assert.Equal(t, int64(-1), hostconfig.MemorySwap)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseHostname(t *testing.T) {
|
func TestParseHostname(t *testing.T) {
|
||||||
validHostnames := map[string]string{
|
for _, tc := range []struct {
|
||||||
"hostname": "hostname",
|
input, hostname, domainname string
|
||||||
"host-name": "host-name",
|
}{
|
||||||
"hostname123": "hostname123",
|
{"--hostname=hostname", "hostname", ""},
|
||||||
"123hostname": "123hostname",
|
{"--hostname=host-name", "host-name", ""},
|
||||||
"hostname-of-63-bytes-long-should-be-valid-and-without-any-error": "hostname-of-63-bytes-long-should-be-valid-and-without-any-error",
|
{"--hostname=hostname123", "hostname123", ""},
|
||||||
}
|
{"--hostname=123hostname", "123hostname", ""},
|
||||||
hostnameWithDomain := "--hostname=hostname.domainname"
|
{"--hostname=hostname-of-63-bytes-long-should-be-valid-and-without-any-error", "hostname-of-63-bytes-long-should-be-valid-and-without-any-error", ""},
|
||||||
hostnameWithDomainTld := "--hostname=hostname.domainname.tld"
|
{"--hostname=hostname.domainname", "hostname.domainname", ""},
|
||||||
for hostname, expectedHostname := range validHostnames {
|
{"--hostname=hostname.domainname.tld", "hostname.domainname.tld", ""},
|
||||||
if config, _, _ := mustParse(t, "--hostname="+hostname); config.Hostname != expectedHostname {
|
{"--domainname=domainname", "", "domainname"},
|
||||||
t.Fatalf("Expected the config to have 'hostname' as %q, got %q", expectedHostname, config.Hostname)
|
{"--domainname=domain-name", "", "domain-name"},
|
||||||
}
|
{"--domainname=domainname123", "", "domainname123"},
|
||||||
}
|
{"--domainname=123domainname", "", "123domainname"},
|
||||||
if config, _, _ := mustParse(t, hostnameWithDomain); config.Hostname != "hostname.domainname" || config.Domainname != "" {
|
{"--domainname=domainname-63-bytes-long-should-be-valid-and-without-any-errors", "", "domainname-63-bytes-long-should-be-valid-and-without-any-errors"},
|
||||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname, got %q", config.Hostname)
|
{"--hostname=some.prefix --domainname=domainname", "some.prefix", "domainname"},
|
||||||
}
|
{"--hostname=another-prefix --domainname=domainname.tld", "another-prefix", "domainname.tld"},
|
||||||
if config, _, _ := mustParse(t, hostnameWithDomainTld); config.Hostname != "hostname.domainname.tld" || config.Domainname != "" {
|
} {
|
||||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname.tld, got %q", config.Hostname)
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
}
|
config, _, _ := mustParse(t, tc.input)
|
||||||
}
|
require.Equal(t, tc.hostname, config.Hostname)
|
||||||
|
require.Equal(t, tc.domainname, config.Domainname)
|
||||||
func TestParseHostnameDomainname(t *testing.T) {
|
})
|
||||||
validDomainnames := map[string]string{
|
|
||||||
"domainname": "domainname",
|
|
||||||
"domain-name": "domain-name",
|
|
||||||
"domainname123": "domainname123",
|
|
||||||
"123domainname": "123domainname",
|
|
||||||
"domainname-63-bytes-long-should-be-valid-and-without-any-errors": "domainname-63-bytes-long-should-be-valid-and-without-any-errors",
|
|
||||||
}
|
|
||||||
for domainname, expectedDomainname := range validDomainnames {
|
|
||||||
if config, _, _ := mustParse(t, "--domainname="+domainname); config.Domainname != expectedDomainname {
|
|
||||||
t.Fatalf("Expected the config to have 'domainname' as %q, got %q", expectedDomainname, config.Domainname)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if config, _, _ := mustParse(t, "--hostname=some.prefix --domainname=domainname"); config.Hostname != "some.prefix" || config.Domainname != "domainname" {
|
|
||||||
t.Fatalf("Expected the config to have 'hostname' as 'some.prefix' and 'domainname' as 'domainname', got %q and %q", config.Hostname, config.Domainname)
|
|
||||||
}
|
|
||||||
if config, _, _ := mustParse(t, "--hostname=another-prefix --domainname=domainname.tld"); config.Hostname != "another-prefix" || config.Domainname != "domainname.tld" {
|
|
||||||
t.Fatalf("Expected the config to have 'hostname' as 'another-prefix' and 'domainname' as 'domainname.tld', got %q and %q", config.Hostname, config.Domainname)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -474,7 +364,7 @@ func TestParseWithExpose(t *testing.T) {
|
|||||||
for expose, expectedError := range tests {
|
for expose, expectedError := range tests {
|
||||||
t.Run(expose, func(t *testing.T) {
|
t.Run(expose, func(t *testing.T) {
|
||||||
_, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
_, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
||||||
assert.Error(t, err, expectedError)
|
require.EqualError(t, err, expectedError)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -489,30 +379,38 @@ func TestParseWithExpose(t *testing.T) {
|
|||||||
for expose, exposedPorts := range tests {
|
for expose, exposedPorts := range tests {
|
||||||
t.Run(expose, func(t *testing.T) {
|
t.Run(expose, func(t *testing.T) {
|
||||||
config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
for _, port := range exposedPorts {
|
for _, port := range exposedPorts {
|
||||||
_, ok := config.ExposedPorts[port]
|
_, ok := config.ExposedPorts[port]
|
||||||
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("merge with published", func(t *testing.T) {
|
t.Run("merge with published", func(t *testing.T) {
|
||||||
// Merge with actual published port
|
config, hostConfig, _, err := parseRun([]string{"--publish=80", "--publish=127.0.0.1:8080:80", "--publish=[::1]:8081:80", "--expose=80-81/tcp"})
|
||||||
config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"})
|
require.NoError(t, err)
|
||||||
assert.NilError(t, err)
|
assert.Equal(t, networktypes.PortMap{
|
||||||
assert.Check(t, is.Len(config.ExposedPorts, 2))
|
networktypes.MustParsePort("80/tcp"): {
|
||||||
|
{},
|
||||||
|
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "8080"},
|
||||||
|
{HostIP: netip.MustParseAddr("::1"), HostPort: "8081"},
|
||||||
|
},
|
||||||
|
}, hostConfig.PortBindings)
|
||||||
|
assert.Len(t, config.ExposedPorts, 2)
|
||||||
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")}
|
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")}
|
||||||
for _, port := range ports {
|
for _, port := range ports {
|
||||||
_, ok := config.ExposedPorts[port]
|
_, ok := config.ExposedPorts[port]
|
||||||
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseDevice(t *testing.T) {
|
func TestParseDevice(t *testing.T) {
|
||||||
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("Windows and macOS validate server-side")
|
||||||
|
}
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
devices []string
|
devices []string
|
||||||
deviceMapping *container.DeviceMapping
|
deviceMapping *container.DeviceMapping
|
||||||
@@ -586,17 +484,17 @@ func TestParseDevice(t *testing.T) {
|
|||||||
|
|
||||||
_, hostconfig, _, err := parseRun(args)
|
_, hostconfig, _, err := parseRun(args)
|
||||||
|
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
if tc.deviceMapping != nil {
|
if tc.deviceMapping != nil {
|
||||||
if assert.Check(t, is.Len(hostconfig.Devices, 1)) {
|
if assert.Len(t, hostconfig.Devices, 1) {
|
||||||
assert.Check(t, is.DeepEqual(*tc.deviceMapping, hostconfig.Devices[0]))
|
assert.Equal(t, *tc.deviceMapping, hostconfig.Devices[0])
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
assert.Check(t, is.Len(hostconfig.Devices, 0))
|
assert.Empty(t, hostconfig.Devices)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Check(t, is.DeepEqual(tc.deviceRequests, hostconfig.DeviceRequests))
|
assert.Equal(t, tc.deviceRequests, hostconfig.DeviceRequests)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -665,11 +563,11 @@ func TestParseDeviceByServerOS(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
got, err := parseDevice(tc.device, tc.serverOS)
|
got, err := parseDevice(tc.device, tc.serverOS)
|
||||||
if tc.wantErr != "" {
|
if tc.wantErr != "" {
|
||||||
assert.Error(t, err, tc.wantErr)
|
require.EqualError(t, err, tc.wantErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, got, tc.want)
|
require.Equal(t, tc.want, got)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -848,13 +746,13 @@ func TestParseNetworkConfig(t *testing.T) {
|
|||||||
_, hConfig, nwConfig, err := parseRun(tc.flags)
|
_, hConfig, nwConfig, err := parseRun(tc.flags)
|
||||||
|
|
||||||
if tc.expectedErr != "" {
|
if tc.expectedErr != "" {
|
||||||
assert.Error(t, err, tc.expectedErr)
|
require.EqualError(t, err, tc.expectedErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.DeepEqual(t, hConfig.NetworkMode, tc.expectedHostCfg.NetworkMode)
|
require.Equal(t, tc.expectedHostCfg.NetworkMode, hConfig.NetworkMode)
|
||||||
assert.DeepEqual(t, nwConfig.EndpointsConfig, tc.expected, cmpopts.EquateComparable(netip.Addr{}))
|
require.Equal(t, tc.expected, nwConfig.EndpointsConfig)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -863,24 +761,24 @@ func TestParseModes(t *testing.T) {
|
|||||||
// pid ko
|
// pid ko
|
||||||
flags, copts := setupRunFlags()
|
flags, copts := setupRunFlags()
|
||||||
args := []string{"--pid=container:", "img", "cmd"}
|
args := []string{"--pid=container:", "img", "cmd"}
|
||||||
assert.NilError(t, flags.Parse(args))
|
require.NoError(t, flags.Parse(args))
|
||||||
_, err := parse(flags, copts, runtime.GOOS)
|
_, err := parse(flags, copts, runtime.GOOS)
|
||||||
assert.ErrorContains(t, err, "--pid: invalid PID mode")
|
require.ErrorContains(t, err, "--pid: invalid PID mode")
|
||||||
|
|
||||||
// pid ok
|
// pid ok
|
||||||
_, hostconfig, _, err := parseRun([]string{"--pid=host", "img", "cmd"})
|
_, hostconfig, _, err := parseRun([]string{"--pid=host", "img", "cmd"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
if !hostconfig.PidMode.Valid() {
|
if !hostconfig.PidMode.Valid() {
|
||||||
t.Fatalf("Expected a valid PidMode, got %v", hostconfig.PidMode)
|
t.Fatalf("Expected a valid PidMode, got %v", hostconfig.PidMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// uts ko
|
// uts ko
|
||||||
_, _, _, err = parseRun([]string{"--uts=container:", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
_, _, _, err = parseRun([]string{"--uts=container:", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
||||||
assert.ErrorContains(t, err, "--uts: invalid UTS mode")
|
require.ErrorContains(t, err, "--uts: invalid UTS mode")
|
||||||
|
|
||||||
// uts ok
|
// uts ok
|
||||||
_, hostconfig, _, err = parseRun([]string{"--uts=host", "img", "cmd"})
|
_, hostconfig, _, err = parseRun([]string{"--uts=host", "img", "cmd"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
if !hostconfig.UTSMode.Valid() {
|
if !hostconfig.UTSMode.Valid() {
|
||||||
t.Fatalf("Expected a valid UTSMode, got %v", hostconfig.UTSMode)
|
t.Fatalf("Expected a valid UTSMode, got %v", hostconfig.UTSMode)
|
||||||
}
|
}
|
||||||
@@ -892,11 +790,11 @@ func TestRunFlagsParseShmSize(t *testing.T) {
|
|||||||
args := []string{"--shm-size=a128m", "img", "cmd"}
|
args := []string{"--shm-size=a128m", "img", "cmd"}
|
||||||
expectedErr := `invalid argument "a128m" for "--shm-size" flag:`
|
expectedErr := `invalid argument "a128m" for "--shm-size" flag:`
|
||||||
err := flags.Parse(args)
|
err := flags.Parse(args)
|
||||||
assert.ErrorContains(t, err, expectedErr)
|
require.ErrorContains(t, err, expectedErr)
|
||||||
|
|
||||||
// shm-size ok
|
// shm-size ok
|
||||||
_, hostconfig, _, err := parseRun([]string{"--shm-size=128m", "img", "cmd"})
|
_, hostconfig, _, err := parseRun([]string{"--shm-size=128m", "img", "cmd"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
if hostconfig.ShmSize != 134217728 {
|
if hostconfig.ShmSize != 134217728 {
|
||||||
t.Fatalf("Expected a valid ShmSize, got %d", hostconfig.ShmSize)
|
t.Fatalf("Expected a valid ShmSize, got %d", hostconfig.ShmSize)
|
||||||
}
|
}
|
||||||
@@ -967,11 +865,11 @@ func TestParseRestartPolicy(t *testing.T) {
|
|||||||
t.Run(tc.input, func(t *testing.T) {
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
_, hostConfig, _, err := parseRun([]string{"--restart=" + tc.input, "img", "cmd"})
|
_, hostConfig, _, err := parseRun([]string{"--restart=" + tc.input, "img", "cmd"})
|
||||||
if tc.expectedErr != "" {
|
if tc.expectedErr != "" {
|
||||||
assert.Check(t, is.Error(err, tc.expectedErr))
|
assert.Nil(t, hostConfig)
|
||||||
assert.Check(t, is.Nil(hostConfig))
|
assert.EqualError(t, err, tc.expectedErr)
|
||||||
} else {
|
} else {
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Check(t, is.DeepEqual(hostConfig.RestartPolicy, tc.expected))
|
assert.Equal(t, tc.expected, hostConfig.RestartPolicy)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -980,45 +878,33 @@ func TestParseRestartPolicy(t *testing.T) {
|
|||||||
func TestParseRestartPolicyAutoRemove(t *testing.T) {
|
func TestParseRestartPolicyAutoRemove(t *testing.T) {
|
||||||
_, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
_, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
||||||
const expected = "conflicting options: cannot specify both --restart and --rm"
|
const expected = "conflicting options: cannot specify both --restart and --rm"
|
||||||
assert.Check(t, is.Error(err, expected))
|
assert.EqualError(t, err, expected)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseHealth(t *testing.T) {
|
func TestParseHealth(t *testing.T) {
|
||||||
checkOk := func(args ...string) *container.HealthConfig {
|
for _, tc := range []struct {
|
||||||
config, _, _, err := parseRun(args)
|
args []string
|
||||||
if err != nil {
|
expected *container.HealthConfig
|
||||||
t.Fatalf("%#v: %v", args, err)
|
wantErr string
|
||||||
}
|
}{
|
||||||
return config.Healthcheck
|
{[]string{"--no-healthcheck", "img", "cmd"}, &container.HealthConfig{Test: []string{"NONE"}}, ""},
|
||||||
}
|
{[]string{"--health-cmd=/check.sh -q", "img", "cmd"}, &container.HealthConfig{Test: []string{"CMD-SHELL", "/check.sh -q"}}, ""},
|
||||||
checkError := func(expected string, args ...string) {
|
{[]string{"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd"}, nil, "--no-healthcheck conflicts with --health-* options"},
|
||||||
config, _, _, err := parseRun(args)
|
{
|
||||||
if err == nil {
|
[]string{"--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd"},
|
||||||
t.Fatalf("Expected error, but got %#v", config)
|
&container.HealthConfig{Timeout: 2 * time.Second, Retries: 3, Interval: 4500 * time.Millisecond, StartPeriod: 5 * time.Second, StartInterval: time.Second},
|
||||||
}
|
"",
|
||||||
if err.Error() != expected {
|
},
|
||||||
t.Fatalf("Expected %#v, got %#v", expected, err)
|
} {
|
||||||
}
|
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||||
}
|
config, _, _, err := parseRun(tc.args)
|
||||||
health := checkOk("--no-healthcheck", "img", "cmd")
|
if tc.wantErr != "" {
|
||||||
if health == nil || len(health.Test) != 1 || health.Test[0] != "NONE" {
|
require.EqualError(t, err, tc.wantErr)
|
||||||
t.Fatalf("--no-healthcheck failed: %#v", health)
|
} else {
|
||||||
}
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tc.expected, config.Healthcheck)
|
||||||
health = checkOk("--health-cmd=/check.sh -q", "img", "cmd")
|
}
|
||||||
if len(health.Test) != 2 || health.Test[0] != "CMD-SHELL" || health.Test[1] != "/check.sh -q" {
|
})
|
||||||
t.Fatalf("--health-cmd: got %#v", health.Test)
|
|
||||||
}
|
|
||||||
if health.Timeout != 0 {
|
|
||||||
t.Fatalf("--health-cmd: timeout = %s", health.Timeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkError("--no-healthcheck conflicts with --health-* options",
|
|
||||||
"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
|
|
||||||
|
|
||||||
health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd")
|
|
||||||
if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second || health.StartInterval != 1*time.Second {
|
|
||||||
t.Fatalf("--health-*: got %#v", health)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1037,93 +923,66 @@ func TestParseLoggingOpts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
func TestParseFileVariables(t *testing.T) {
|
||||||
expErr := "--env-file: open nonexistent: no such file or directory"
|
for _, tc := range []struct {
|
||||||
if runtime.GOOS == "windows" {
|
flag, file, key string
|
||||||
expErr = "--env-file: open nonexistent: The system cannot find the file specified."
|
}{
|
||||||
}
|
{"env", "valid.env", "ENV"},
|
||||||
// env ko
|
{"label", "valid.label", "LABEL"},
|
||||||
if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
} {
|
||||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
t.Run(tc.flag, func(t *testing.T) {
|
||||||
}
|
expectedError := "no such file or directory"
|
||||||
// env ok
|
if runtime.GOOS == "windows" {
|
||||||
config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"})
|
expectedError = "The system cannot find the file specified."
|
||||||
if err != nil {
|
}
|
||||||
t.Fatal(err)
|
_, _, _, err := parseRun([]string{"--" + tc.flag + "-file=nonexistent"})
|
||||||
}
|
require.EqualError(t, err, "--"+tc.flag+"-file: open nonexistent: "+expectedError)
|
||||||
if len(config.Env) != 1 || config.Env[0] != "ENV1=value1" {
|
for _, extra := range []bool{false, true} {
|
||||||
t.Fatalf("Expected a config with [ENV1=value1], got %v", config.Env)
|
args := []string{"--" + tc.flag + "-file=testdata/" + tc.file}
|
||||||
}
|
expected := []string{tc.key + "1=value1"}
|
||||||
config, _, _, err = parseRun([]string{"--env-file=testdata/valid.env", "--env=ENV2=value2", "img", "cmd"})
|
if extra {
|
||||||
if err != nil {
|
args = append(args, "--"+tc.flag+"="+tc.key+"2=value2")
|
||||||
t.Fatal(err)
|
expected = append(expected, tc.key+"2=value2")
|
||||||
}
|
}
|
||||||
if len(config.Env) != 2 || config.Env[0] != "ENV1=value1" || config.Env[1] != "ENV2=value2" {
|
config, _, _, err := parseRun(args)
|
||||||
t.Fatalf("Expected a config with [ENV1=value1 ENV2=value2], got %v", config.Env)
|
require.NoError(t, err)
|
||||||
|
if tc.flag == "env" {
|
||||||
|
assert.Equal(t, expected, config.Env)
|
||||||
|
} else {
|
||||||
|
assert.Len(t, config.Labels, len(expected))
|
||||||
|
for _, label := range expected {
|
||||||
|
key, value, _ := strings.Cut(label, "=")
|
||||||
|
assert.Equal(t, value, config.Labels[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
|
func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
|
||||||
// UTF8 with BOM
|
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env"})
|
||||||
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env", "img", "cmd"})
|
require.NoError(t, err)
|
||||||
if err != nil {
|
assert.Equal(t, []string{"FOO=BAR", "HELLO=您好", "BAR=FOO"}, config.Env)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
env := []string{"FOO=BAR", "HELLO=" + string([]byte{0xe6, 0x82, 0xa8, 0xe5, 0xa5, 0xbd}), "BAR=FOO"}
|
|
||||||
if len(config.Env) != len(env) {
|
|
||||||
t.Fatalf("Expected a config with %d env variables, got %v: %v", len(env), len(config.Env), config.Env)
|
|
||||||
}
|
|
||||||
for i, v := range env {
|
|
||||||
if config.Env[i] != v {
|
|
||||||
t.Fatalf("Expected a config with [%s], got %v", v, []byte(config.Env[i]))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// UTF16 with BOM
|
for _, file := range []string{"utf16.env", "utf16be.env"} {
|
||||||
e := "invalid utf8 bytes at line"
|
t.Run(file, func(t *testing.T) {
|
||||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
_, _, _, err := parseRun([]string{"--env-file=testdata/" + file})
|
||||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
require.ErrorContains(t, err, "invalid utf8 bytes at line")
|
||||||
}
|
})
|
||||||
// UTF16BE with BOM
|
|
||||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16be.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
|
||||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
|
||||||
expErr := "--label-file: open nonexistent: no such file or directory"
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
expErr = "--label-file: open nonexistent: The system cannot find the file specified."
|
|
||||||
}
|
|
||||||
// label ko
|
|
||||||
if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
|
||||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
|
||||||
}
|
|
||||||
// label ok
|
|
||||||
config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(config.Labels) != 1 || config.Labels["LABEL1"] != "value1" {
|
|
||||||
t.Fatalf("Expected a config with [LABEL1:value1], got %v", config.Labels)
|
|
||||||
}
|
|
||||||
config, _, _, err = parseRun([]string{"--label-file=testdata/valid.label", "--label=LABEL2=value2", "img", "cmd"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(config.Labels) != 2 || config.Labels["LABEL1"] != "value1" || config.Labels["LABEL2"] != "value2" {
|
|
||||||
t.Fatalf("Expected a config with [LABEL1:value1 LABEL2:value2], got %v", config.Labels)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEntryPoint(t *testing.T) {
|
func TestParseEntryPoint(t *testing.T) {
|
||||||
config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"})
|
config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"})
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Check(t, is.DeepEqual(config.Entrypoint, []string{"anything"}))
|
assert.Equal(t, []string{"anything"}, config.Entrypoint)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDevice(t *testing.T) {
|
func TestValidateDevice(t *testing.T) {
|
||||||
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("Windows and macOS validate server-side")
|
||||||
|
}
|
||||||
valid := []string{
|
valid := []string{
|
||||||
"/home",
|
"/home",
|
||||||
"/home:/home",
|
"/home:/home",
|
||||||
@@ -1222,22 +1081,22 @@ func TestValidateDeviceByServerOS(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
got, err := validateDevice(tc.value, tc.serverOS)
|
got, err := validateDevice(tc.value, tc.serverOS)
|
||||||
if tc.wantError != "" {
|
if tc.wantError != "" {
|
||||||
assert.Error(t, err, tc.wantError)
|
require.EqualError(t, err, tc.wantError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, got, tc.want)
|
require.Equal(t, tc.want, got)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
|
func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
|
||||||
got, err := validateDeviceCgroupRule("c 1:3 rwm")
|
got, err := validateDeviceCgroupRule("c 1:3 rwm")
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, got, "c 1:3 rwm")
|
require.Equal(t, "c 1:3 rwm", got)
|
||||||
|
|
||||||
_, err = validateDeviceCgroupRule("invalid")
|
_, err = validateDeviceCgroupRule("invalid")
|
||||||
assert.Error(t, err, "invalid device cgroup format 'invalid'")
|
require.EqualError(t, err, "invalid device cgroup format 'invalid'")
|
||||||
|
|
||||||
if invalidParameter(nil) != nil {
|
if invalidParameter(nil) != nil {
|
||||||
t.Fatal("invalidParameter(nil) should be nil")
|
t.Fatal("invalidParameter(nil) should be nil")
|
||||||
@@ -1245,9 +1104,9 @@ func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
|
|||||||
cause := errors.New("bad input")
|
cause := errors.New("bad input")
|
||||||
err = invalidParameter(cause)
|
err = invalidParameter(cause)
|
||||||
var invalid interface{ InvalidParameter() }
|
var invalid interface{ InvalidParameter() }
|
||||||
assert.Assert(t, errors.As(err, &invalid))
|
require.ErrorAs(t, err, &invalid)
|
||||||
assert.Assert(t, errors.Is(err, cause))
|
require.ErrorIs(t, err, cause)
|
||||||
assert.Equal(t, invalidParameter(err), err) // already invalid, so not wrapped twice
|
require.Condition(t, func() bool { return invalidParameter(err) == err })
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseSystemPaths(t *testing.T) {
|
func TestParseSystemPaths(t *testing.T) {
|
||||||
@@ -1298,9 +1157,9 @@ func TestParseSystemPaths(t *testing.T) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
securityOpts, maskedPaths, readonlyPaths := parseSystemPaths(tc.in)
|
securityOpts, maskedPaths, readonlyPaths := parseSystemPaths(tc.in)
|
||||||
assert.DeepEqual(t, securityOpts, tc.out)
|
require.Equal(t, tc.out, securityOpts)
|
||||||
assert.DeepEqual(t, maskedPaths, tc.masked)
|
require.Equal(t, tc.masked, maskedPaths)
|
||||||
assert.DeepEqual(t, readonlyPaths, tc.readonly)
|
require.Equal(t, tc.readonly, readonlyPaths)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1322,9 +1181,9 @@ func TestConvertToStandardNotation(t *testing.T) {
|
|||||||
for key, ports := range valid {
|
for key, ports := range valid {
|
||||||
convertedPorts, err := convertToStandardNotation(ports)
|
convertedPorts, err := convertToStandardNotation(ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
assert.NilError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
assert.DeepEqual(t, strings.Split(key, " "), convertedPorts)
|
require.Equal(t, strings.Split(key, " "), convertedPorts)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ports := range invalid {
|
for _, ports := range invalid {
|
||||||
|
|||||||
@@ -8,10 +8,10 @@ package container
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
"github.com/moby/moby/client"
|
"github.com/moby/moby/client"
|
||||||
|
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ImageExistsLocally returns a boolean indicating if an image with the
|
// ImageExistsLocally returns a boolean indicating if an image with the
|
||||||
@@ -30,11 +30,23 @@ func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool,
|
|||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform {
|
if platform == "" || platform == "any" {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, nil
|
requested, err := parsePlatform(platform)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return platformMatches(requested, &specs.Platform{OS: inspectImage.Os, Architecture: inspectImage.Architecture, Variant: inspectImage.Variant}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func platformMatches(requested, image *specs.Platform) bool {
|
||||||
|
variant := image.Variant
|
||||||
|
if variant == "" {
|
||||||
|
variant = map[string]string{"amd64": "v1", "arm": "v7", "arm64": "v8"}[image.Architecture]
|
||||||
|
}
|
||||||
|
return image.OS == requested.OS && image.Architecture == requested.Architecture && (requested.Variant == "" || requested.Variant == variant)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemoveImage removes image from local store, the function is used to run different
|
// RemoveImage removes image from local store, the function is used to run different
|
||||||
@@ -46,18 +58,14 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
|
|||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
|
|
||||||
if _, err := cli.ImageInspect(ctx, imageName); cerrdefs.IsNotFound(err) {
|
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
||||||
|
Force: force,
|
||||||
|
PruneChildren: pruneChildren,
|
||||||
|
}); cerrdefs.IsNotFound(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
|
||||||
Force: force,
|
|
||||||
PruneChildren: pruneChildren,
|
|
||||||
}); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -5,65 +5,74 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
|
mobyclient "github.com/moby/moby/client"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
log.SetLevel(log.DebugLevel)
|
logging.SetLevel(logging.LevelDebug)
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
|
func loadScratchImage(t *testing.T, platform string) string {
|
||||||
// network or emulation since there is nothing to run) and returns its tag, removing it after
|
|
||||||
// the test.
|
|
||||||
func buildScratchImage(t *testing.T, platform string) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
tag := fmt.Sprintf("act-test-exists-%s:latest", strings.TrimPrefix(platform, "linux/"))
|
tag := fmt.Sprintf("act-test-exists-%s-%d:latest", strings.TrimPrefix(platform, "linux/"), time.Now().UnixNano())
|
||||||
cmd := exec.Command("docker", "build", "--platform", platform, "-t", tag, "-")
|
var archive bytes.Buffer
|
||||||
cmd.Stdin = strings.NewReader("FROM scratch\nLABEL act-test=1\n")
|
require.NoError(t, writeFilesTar(t.Context(), &archive, 0, 0,
|
||||||
// Force BuildKit: it records the requested architecture in the image config for a
|
&FileEntry{Name: "config.json", Mode: 0o600, Body: fmt.Sprintf(`{"architecture":%q,"os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`, strings.TrimPrefix(platform, "linux/"))},
|
||||||
// FROM-scratch build, whereas the classic builder ignores --platform and tags it with the
|
&FileEntry{Name: "manifest.json", Mode: 0o600, Body: fmt.Sprintf(`[{"Config":"config.json","RepoTags":[%q],"Layers":[]}]`, tag)},
|
||||||
// host arch, which would break the per-platform existence assertions below.
|
))
|
||||||
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
|
cli, err := GetDockerClient(t.Context())
|
||||||
out, err := cmd.CombinedOutput()
|
require.NoError(t, err)
|
||||||
require.NoError(t, err, string(out))
|
t.Cleanup(func() { _ = cli.Close() })
|
||||||
t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() })
|
response, err := cli.ImageLoad(t.Context(), &archive)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer response.Close()
|
||||||
|
require.NoError(t, logDockerResponse(common.Logger(t.Context()), response, false))
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = cli.ImageRemove(context.WithoutCancel(t.Context()), tag, mobyclient.ImageRemoveOptions{Force: true})
|
||||||
|
})
|
||||||
return tag
|
return tag
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageExistsLocally(t *testing.T) {
|
func TestImageExistsLocally(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireDocker(t)
|
requireDocker(t)
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
|
||||||
// a non-existent image is reported absent
|
|
||||||
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, missing)
|
assert.False(t, missing)
|
||||||
|
|
||||||
// Build tiny images for two architectures locally so per-platform existence can be checked
|
amd64Ref := loadScratchImage(t, "linux/amd64")
|
||||||
// offline (formerly pulled node:24-bookworm-slim for amd64 and arm64 over the network).
|
arm64Ref := loadScratchImage(t, "linux/arm64")
|
||||||
amd64Ref := buildScratchImage(t, "linux/amd64")
|
|
||||||
arm64Ref := buildScratchImage(t, "linux/arm64")
|
|
||||||
|
|
||||||
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, amd64Exists)
|
assert.True(t, amd64Exists)
|
||||||
|
|
||||||
// a non-host architecture image is detected for its own architecture
|
|
||||||
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, arm64Exists)
|
assert.True(t, arm64Exists)
|
||||||
|
|
||||||
// a present image is reported absent for a different platform
|
|
||||||
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, wrongPlatform)
|
assert.False(t, wrongPlatform)
|
||||||
|
|
||||||
|
removed, err := RemoveImage(ctx, amd64Ref, false, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, removed)
|
||||||
|
removed, err = RemoveImage(ctx, amd64Ref, false, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, removed)
|
||||||
}
|
}
|
||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
type dockerMessage struct {
|
type dockerMessage struct {
|
||||||
@@ -26,25 +26,17 @@ type dockerMessage struct {
|
|||||||
Progress string `json:"progress"`
|
Progress string `json:"progress"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error {
|
func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isError bool) error {
|
||||||
if dockerResponse == nil {
|
if dockerResponse == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
defer dockerResponse.Close()
|
defer dockerResponse.Close()
|
||||||
|
|
||||||
scanner := bufio.NewScanner(dockerResponse)
|
scanner := bufio.NewScanner(dockerResponse)
|
||||||
msg := dockerMessage{}
|
scanner.Buffer(nil, 64*1024*1024) // a build echoes long instructions in one record
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Bytes()
|
line := scanner.Bytes()
|
||||||
|
var msg dockerMessage
|
||||||
msg.ID = ""
|
|
||||||
msg.Stream = ""
|
|
||||||
msg.Error = ""
|
|
||||||
msg.ErrorDetail.Message = ""
|
|
||||||
msg.Status = ""
|
|
||||||
msg.Progress = ""
|
|
||||||
|
|
||||||
if err := json.Unmarshal(line, &msg); err != nil {
|
if err := json.Unmarshal(line, &msg); err != nil {
|
||||||
writeLog(logger, false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
|
writeLog(logger, false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
|
||||||
continue
|
continue
|
||||||
@@ -74,10 +66,10 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return scanner.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...any) {
|
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
||||||
if isError {
|
if isError {
|
||||||
logger.Errorf(format, args...)
|
logger.Errorf(format, args...)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -53,7 +53,9 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID,
|
|||||||
networks:
|
networks:
|
||||||
for _, n := range networks.Items {
|
for _, n := range networks.Items {
|
||||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||||
if err != nil {
|
if cerrdefs.IsNotFound(err) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
|
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -71,7 +73,9 @@ networks:
|
|||||||
continue networks
|
continue networks
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); cerrdefs.IsNotFound(err) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
|
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -81,14 +85,14 @@ networks:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return createDockerNetwork(ctx, cli, name, opts)
|
return createDockerNetwork(ctx, cli, name, opts)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||||
@@ -153,7 +157,7 @@ func isAddressPoolExhausted(err error) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -163,7 +167,7 @@ func NewDockerNetworkConnectExecutor(networkName, container string) common.Execu
|
|||||||
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -178,14 +182,14 @@ func connectNetwork(ctx context.Context, cli client.APIClient, networkName, cont
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return disconnectNetwork(ctx, cli, networkName, container)
|
return disconnectNetwork(ctx, cli, networkName, container)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -197,42 +201,41 @@ func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, c
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
|
return removeNetwork(ctx, cli, name)
|
||||||
|
}).IfNot(common.Dryrun)
|
||||||
|
}
|
||||||
|
|
||||||
// Make sure that all network of the specified name are removed
|
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
||||||
// cli.NetworkRemove refuses to remove a network if there are duplicates
|
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{}) // NetworkRemove by name refuses duplicates, so remove each by ID
|
||||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
if err != nil {
|
||||||
if err != nil {
|
return err
|
||||||
|
}
|
||||||
|
var errs []error
|
||||||
|
for _, n := range networks.Items {
|
||||||
|
if n.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||||
|
if cerrdefs.IsNotFound(err) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// For Gitea, reduce log noise
|
if len(result.Network.Containers) != 0 {
|
||||||
// common.Logger(ctx).Debugf("%v", networks)
|
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the orphan cleanup reclaims it once they are gone", name)
|
||||||
var errs []error
|
continue
|
||||||
for _, n := range networks.Items {
|
}
|
||||||
if n.Name == name {
|
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
|
||||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// it holds a subnet out of the daemon's pool until something reclaims it
|
|
||||||
if len(result.Network.Containers) != 0 {
|
|
||||||
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the idle cleanup reclaims it once they are gone", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
|
||||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
|
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestIsolatedCacheContainer(t *testing.T) {
|
func TestIsolatedCacheContainer(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
|
cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
|
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
|
||||||
@@ -58,7 +58,7 @@ func TestIsAddressPoolExhausted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
|
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
bridgeOptions, want map[string]string
|
bridgeOptions, want map[string]string
|
||||||
}{
|
}{
|
||||||
@@ -87,7 +87,7 @@ func (m *mockDockerClient) NetworkDisconnect(ctx context.Context, id string, opt
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestConnectAndDisconnectNetwork(t *testing.T) {
|
func TestConnectAndDisconnectNetwork(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
connect := func(answer error) error {
|
connect := func(answer error) error {
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
|
client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
|
||||||
@@ -116,8 +116,8 @@ func TestConnectAndDisconnectNetwork(t *testing.T) {
|
|||||||
// Of this runner's networks, only the ones nothing but its own cache container is attached to
|
// Of this runner's networks, only the ones nothing but its own cache container is attached to
|
||||||
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
|
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
|
||||||
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
|
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
|
||||||
func TestRemoveOrphanNetworks(t *testing.T) {
|
func TestRemoveNetworks(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
|
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
|
||||||
@@ -128,11 +128,13 @@ func TestRemoveOrphanNetworks(t *testing.T) {
|
|||||||
{ID: "starting"},
|
{ID: "starting"},
|
||||||
{ID: "cached"},
|
{ID: "cached"},
|
||||||
{ID: "shared"},
|
{ID: "shared"},
|
||||||
|
{ID: "gone"},
|
||||||
|
{ID: "raced"},
|
||||||
}}, nil)
|
}}, nil)
|
||||||
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
|
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
|
||||||
Return(mobyclient.NetworkInspectResult{}, nil)
|
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
|
||||||
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
|
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
|
||||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil).Twice()
|
||||||
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
|
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
|
||||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
|
||||||
Network: network.Network{Created: cutoff.Add(time.Second)},
|
Network: network.Network{Created: cutoff.Add(time.Second)},
|
||||||
@@ -142,13 +144,28 @@ func TestRemoveOrphanNetworks(t *testing.T) {
|
|||||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
|
||||||
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
|
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
|
||||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
|
||||||
|
client.On("NetworkInspect", ctx, "gone", mobyclient.NetworkInspectOptions{}).
|
||||||
|
Return(mobyclient.NetworkInspectResult{}, cerrdefs.ErrNotFound).Twice()
|
||||||
|
client.On("NetworkInspect", ctx, "raced", mobyclient.NetworkInspectOptions{}).
|
||||||
|
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
|
||||||
|
client.On("NetworkRemove", ctx, "raced", mobyclient.NetworkRemoveOptions{}).
|
||||||
|
Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrNotFound).Twice()
|
||||||
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
|
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
|
||||||
Return(mobyclient.NetworkRemoveResult{}, nil)
|
Return(mobyclient.NetworkRemoveResult{}, nil).Twice()
|
||||||
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
|
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
|
||||||
Return(nil).Once()
|
Return(nil).Once()
|
||||||
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
|
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
|
||||||
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
|
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
|
||||||
|
|
||||||
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
|
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
|
||||||
|
|
||||||
|
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
|
||||||
|
{ID: "orphan", Name: "job"},
|
||||||
|
{ID: "busy", Name: "job"},
|
||||||
|
{ID: "gone", Name: "job"},
|
||||||
|
{ID: "raced", Name: "job"},
|
||||||
|
{ID: "shared", Name: "other"},
|
||||||
|
}}, nil)
|
||||||
|
require.NoError(t, removeNetwork(ctx, client, "job"))
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
@@ -6,6 +6,7 @@ package container
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -17,32 +18,17 @@ func TestParsePlatform(t *testing.T) {
|
|||||||
assert.Nil(t, got)
|
assert.Nil(t, got)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("os/arch", func(t *testing.T) {
|
for input, want := range map[string]specs.Platform{
|
||||||
got, err := parsePlatform("linux/amd64")
|
"linux/amd64": {OS: "linux", Architecture: "amd64"},
|
||||||
require.NoError(t, err)
|
"linux/arm/v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
|
||||||
require.NotNil(t, got)
|
"Linux/AMD64/V8": {OS: "linux", Architecture: "amd64", Variant: "v8"},
|
||||||
assert.Equal(t, "linux", got.OS)
|
} {
|
||||||
assert.Equal(t, "amd64", got.Architecture)
|
t.Run(input, func(t *testing.T) {
|
||||||
assert.Empty(t, got.Variant)
|
got, err := parsePlatform(input)
|
||||||
})
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, &want, got)
|
||||||
t.Run("os/arch/variant", func(t *testing.T) {
|
})
|
||||||
got, err := parsePlatform("linux/arm/v7")
|
}
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, got)
|
|
||||||
assert.Equal(t, "linux", got.OS)
|
|
||||||
assert.Equal(t, "arm", got.Architecture)
|
|
||||||
assert.Equal(t, "v7", got.Variant)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("input is lowercased", func(t *testing.T) {
|
|
||||||
got, err := parsePlatform("Linux/AMD64/V8")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, got)
|
|
||||||
assert.Equal(t, "linux", got.OS)
|
|
||||||
assert.Equal(t, "amd64", got.Architecture)
|
|
||||||
assert.Equal(t, "v8", got.Variant)
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, bad := range []string{
|
for _, bad := range []string{
|
||||||
"amd64",
|
"amd64",
|
||||||
@@ -61,3 +47,26 @@ func TestParsePlatform(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPlatformMatches(t *testing.T) {
|
||||||
|
for _, testcase := range []struct {
|
||||||
|
requested, image string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"linux/amd64", "linux/amd64", true},
|
||||||
|
{"linux/amd64/v1", "linux/amd64", true},
|
||||||
|
{"linux/arm64", "linux/amd64", false},
|
||||||
|
{"linux/arm64/v8", "linux/arm64", true},
|
||||||
|
{"linux/arm/v7", "linux/arm/v7", true},
|
||||||
|
{"linux/arm", "linux/arm/v7", true},
|
||||||
|
{"linux/arm/v6", "linux/arm/v7", false},
|
||||||
|
{"linux/arm", "linux/arm/v6", true},
|
||||||
|
{"windows/arm/v7", "linux/arm/v7", false},
|
||||||
|
} {
|
||||||
|
requested, err := parsePlatform(testcase.requested)
|
||||||
|
require.NoError(t, err)
|
||||||
|
image, err := parsePlatform(testcase.image)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, testcase.want, platformMatches(requested, image), "%s on %s", testcase.requested, testcase.image)
|
||||||
|
}
|
||||||
|
}
|
||||||
+117
-24
@@ -37,10 +37,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
jobLabel = "com.gitea.runner.job"
|
jobLabel = "com.gitea.runner.job."
|
||||||
maxCreateBody = 8 << 20
|
maxCreateBody = 8 << 20
|
||||||
|
|
||||||
dockerProxyProbeTimeout = 5 * time.Second
|
dockerProxyProbeTimeout = 5 * time.Second
|
||||||
|
dockerProxyProbeWorkerTimeout = time.Minute
|
||||||
|
dockerProxyProbeTarget = "/gitea-runner-probe"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -48,13 +50,27 @@ var (
|
|||||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var dockerProxyProbes sync.Map
|
||||||
|
|
||||||
|
type dockerProxyDirectory struct {
|
||||||
|
dir, daemonDir string
|
||||||
|
socket os.FileInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
type dockerProxyProbe struct {
|
||||||
|
directory dockerProxyDirectory
|
||||||
|
done chan struct{}
|
||||||
|
seen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||||
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
cli, err := GetDockerClient(probeCtx)
|
cli, err := client.New(client.FromEnv) // negotiates on first request, so a cached probe costs no round trip
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -63,37 +79,78 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if info, err := os.Stat(daemonSocket); err != nil || info.Mode()&os.ModeSocket == 0 {
|
socket, err := os.Stat(daemonSocket)
|
||||||
|
if err != nil || socket.Mode()&os.ModeSocket == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
dir, daemonDir := runnerContainerWorkdir(probeCtx, cli)
|
workdir, _ := os.Getwd()
|
||||||
seen := daemonDir != ""
|
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||||
if !seen {
|
probe, err := probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||||
if dir, err = filepath.Abs(os.TempDir()); err == nil {
|
|
||||||
daemonDir = dir
|
|
||||||
seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if !seen {
|
if !probe.seen {
|
||||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
proxy, err := StartDockerProxy(daemonSocket, dir, job)
|
proxy, err := StartDockerProxy(daemonSocket, probe.directory.dir, job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
|
proxy.Socket = probe.directory.daemonDir + strings.TrimPrefix(proxy.Socket, probe.directory.dir)
|
||||||
return proxy
|
return proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (*dockerProxyProbe, error) {
|
||||||
|
for {
|
||||||
|
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
|
||||||
|
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
|
||||||
|
if loaded {
|
||||||
|
probe, _ = actual.(*dockerProxyProbe)
|
||||||
|
if !os.SameFile(probe.directory.socket, socket) { // daemon replacement invalidates cached paths
|
||||||
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
go func() { // outlives callers' deadlines so a slow daemon still gets its result cached
|
||||||
|
defer cli.Close()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeWorkerTimeout)
|
||||||
|
defer cancel()
|
||||||
|
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||||
|
probe.seen = probe.directory.daemonDir != ""
|
||||||
|
if !probe.seen {
|
||||||
|
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
|
||||||
|
probe.directory.daemonDir = probe.directory.dir
|
||||||
|
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if probe.seen {
|
||||||
|
dockerProxyProbes.LoadOrStore(key, probe)
|
||||||
|
} else {
|
||||||
|
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||||
|
}
|
||||||
|
close(probe.done)
|
||||||
|
removeStaleProbeContainers(cli)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return &dockerProxyProbe{}, ctx.Err()
|
||||||
|
case <-probe.done:
|
||||||
|
}
|
||||||
|
if ctx.Err() == nil && errors.Is(probe.err, context.DeadlineExceeded) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return probe, probe.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
||||||
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
||||||
workdir, err := os.Getwd()
|
workdir, err := os.Getwd()
|
||||||
@@ -144,19 +201,25 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
|||||||
if len(images.Items) == 0 {
|
if len(images.Items) == 0 {
|
||||||
return false, errors.New("no image available for the docker proxy probe")
|
return false, errors.New("no image available for the docker proxy probe")
|
||||||
}
|
}
|
||||||
created, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
var created client.ContainerCreateResult
|
||||||
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
|
for _, image := range images.Items {
|
||||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
|
||||||
}},
|
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||||
})
|
{Type: mount.TypeBind, Source: daemonDir, Target: dockerProxyProbeTarget, ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if !cerrdefs.IsNotFound(err) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
if cerrdefs.IsInvalidArgument(err) {
|
if cerrdefs.IsInvalidArgument(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, dockerProxyProbeTarget)
|
||||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||||
@@ -165,6 +228,36 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
|||||||
return seen, err
|
return seen, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeStaleProbeContainers removes probe containers left by abandoned creates once any probe in flight when listed has ended.
|
||||||
|
func removeStaleProbeContainers(cli client.APIClient) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||||
|
defer cancel()
|
||||||
|
listed, err := cli.ContainerList(ctx, client.ContainerListOptions{
|
||||||
|
All: true,
|
||||||
|
Filters: make(client.Filters).Add("status", string(container.StateCreated)).Add("volume", dockerProxyProbeTarget),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
common.Logger(ctx).Warnf("listing stale docker proxy probe containers failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
listed.Items = slices.DeleteFunc(listed.Items, func(stale container.Summary) bool {
|
||||||
|
return stale.State != container.StateCreated || !slices.ContainsFunc(stale.Mounts, func(mountPoint container.MountPoint) bool {
|
||||||
|
return mountPoint.Type == mount.TypeBind && !mountPoint.RW && mountPoint.Destination == dockerProxyProbeTarget
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if len(listed.Items) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(dockerProxyProbeWorkerTimeout + dockerProxyProbeTimeout)
|
||||||
|
for _, stale := range listed.Items {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||||
|
if err := (&containerReference{cli: cli, id: stale.ID}).remove()(ctx); err != nil {
|
||||||
|
common.Logger(ctx).Warnf("removing stale docker proxy probe container failed: %v", err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||||
info, err := os.Stat(daemonSocket)
|
info, err := os.Stat(daemonSocket)
|
||||||
@@ -302,7 +395,7 @@ func rewriteCreate(r *http.Request, job string, mounts map[string]string) error
|
|||||||
if config.Labels == nil {
|
if config.Labels == nil {
|
||||||
config.Labels = make(map[string]string)
|
config.Labels = make(map[string]string)
|
||||||
}
|
}
|
||||||
config.Labels[jobLabel] = job
|
config.Labels[jobLabel+job] = "" // per-job key, so nested proxies keep each other's
|
||||||
if fields["Labels"], err = json.Marshal(config.Labels); err != nil {
|
if fields["Labels"], err = json.Marshal(config.Labels); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -560,7 +653,7 @@ func RemoveDockerJobResources(ctx context.Context, job string) error {
|
|||||||
|
|
||||||
func removeLabelled(ctx context.Context, cli client.APIClient, job string) error {
|
func removeLabelled(ctx context.Context, cli client.APIClient, job string) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
filters := make(client.Filters).Add("label", jobLabel+"="+job)
|
filters := make(client.Filters).Add("label", jobLabel+job)
|
||||||
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
|
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
|
||||||
errs := []error{err}
|
errs := []error{err}
|
||||||
for _, c := range containers.Items {
|
for _, c := range containers.Items {
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
@@ -68,6 +70,7 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
bodies := make(chan []byte, 8)
|
bodies := make(chan []byte, 8)
|
||||||
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
|
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
|
||||||
streamsDone := make(chan struct{}, 2)
|
streamsDone := make(chan struct{}, 2)
|
||||||
|
var probes atomic.Int32
|
||||||
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
|
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
|
||||||
listener, err := net.Listen("unix", daemonSocket)
|
listener, err := net.Listen("unix", daemonSocket)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -75,12 +78,30 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
switch {
|
switch {
|
||||||
case strings.HasSuffix(r.URL.Path, "/create"):
|
case strings.HasSuffix(r.URL.Path, "/create"):
|
||||||
body, _ := io.ReadAll(r.Body)
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
if bytes.Contains(body, []byte(`"Image":"probe-image"`)) {
|
||||||
|
_, _ = w.Write([]byte(`{"Id":"probe"}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
if r.URL.RawQuery == "wait" {
|
if r.URL.RawQuery == "wait" {
|
||||||
close(createStarted)
|
close(createStarted)
|
||||||
<-releaseCreate
|
<-releaseCreate
|
||||||
}
|
}
|
||||||
bodies <- body
|
bodies <- body
|
||||||
w.WriteHeader(http.StatusCreated)
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/images/json"):
|
||||||
|
if probes.Add(1) == 1 {
|
||||||
|
http.Error(w, "probe unavailable", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
_, _ = w.Write([]byte(`[]`))
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||||
|
http.NotFound(w, r)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||||
|
w.Header().Set("X-Docker-Container-Path-Stat", "e30=")
|
||||||
|
case r.Method == http.MethodDelete:
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
|
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
|
||||||
if r.URL.Path == "/exec/error/start" {
|
if r.URL.Path == "/exec/error/start" {
|
||||||
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
|
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
|
||||||
@@ -146,52 +167,52 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
t.Run("labels creates", func(t *testing.T) {
|
t.Run("labels creates", func(t *testing.T) {
|
||||||
for _, testCase := range []struct{ path, body, want string }{
|
for _, testCase := range []struct{ path, body, want string }{
|
||||||
{
|
{
|
||||||
"/v1.47/containers/create", `{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job":"other"}}`,
|
"/v1.47/containers/create", `{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":""}}`,
|
||||||
`{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job":"job-1"}}`,
|
`{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":"","com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/containers/create", `{"HostConfig":{"Binds":["/workspace/o/r:/src:ro","/workspace/o/r/../r/sub/:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2"}}`,
|
"/containers/create", `{"HostConfig":{"Binds":["/workspace/o/r:/src:ro","/workspace/o/r/../r/sub/:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2"}}`,
|
||||||
`{"HostConfig":{"Binds":["/volumes/job/_data:/src:ro","/volumes/job/_data/sub:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2","com.gitea.runner.job":"job-1"}}`,
|
`{"HostConfig":{"Binds":["/volumes/job/_data:/src:ro","/volumes/job/_data/sub:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2","com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/var/run/docker.sock"},{"type":"bind","source":"/workspace/o/r/sub/","target":"/m"},{"Type":"bind","Source":"/workspace/o/r/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/workspace/o/r/../r","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]}}`,
|
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/var/run/docker.sock"},{"type":"bind","source":"/workspace/o/r/sub/","target":"/m"},{"Type":"bind","Source":"/workspace/o/r/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/workspace/o/r/../r","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]}}`,
|
||||||
`{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/tmp/p/docker.sock","Target":"/var/run/docker.sock"},{"Type":"bind","Source":"/volumes/job/_data/sub/","target":"/m"},{"Type":"bind","Source":"/volumes/job/_data/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/volumes/job/_data","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]},"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/tmp/p/docker.sock","Target":"/var/run/docker.sock"},{"Type":"bind","Source":"/volumes/job/_data/sub/","target":"/m"},{"Type":"bind","Source":"/volumes/job/_data/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/volumes/job/_data","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/workspace/o/r/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]}}`,
|
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/workspace/o/r/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]}}`,
|
||||||
`{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/volumes/job/_data/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]},"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/volumes/job/_data/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/containers/create", `{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null}`,
|
"/containers/create", `{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null}`,
|
||||||
`{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null,"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null,"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/networks/create", `{"Name":"n"}`,
|
"/networks/create", `{"Name":"n"}`,
|
||||||
`{"Name":"n","Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"Name":"n","Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", `{"Name":"v","labels":null,"DriverOpts":{"type":"none","o":"rbind,ro","device":"/workspace/o/r/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
"/volumes/create", `{"Name":"v","labels":null,"DriverOpts":{"type":"none","o":"rbind,ro","device":"/workspace/o/r/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
||||||
`{"Name":"v","Labels":{"com.gitea.runner.job":"job-1"},"DriverOpts":{"type":"none","o":"rbind,ro","device":"/volumes/job/_data/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
`{"Name":"v","Labels":{"com.gitea.runner.job.job-1":""},"DriverOpts":{"type":"none","o":"rbind,ro","device":"/volumes/job/_data/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", `{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"}}`,
|
"/volumes/create", `{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"}}`,
|
||||||
`{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", `{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"}}`,
|
"/volumes/create", `{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"}}`,
|
||||||
`{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", "",
|
"/volumes/create", "",
|
||||||
`{"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", "null",
|
"/volumes/create", "null",
|
||||||
`{"Labels":{"com.gitea.runner.job":"job-1"}}`,
|
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"/volumes/create", `{"Labels":{"discard":"1"},"labels":null,"LABELS":{"own":"1"},"LABELS":{"extra":"2"}}`,
|
"/volumes/create", `{"Labels":{"discard":"1"},"labels":null,"LABELS":{"own":"1"},"LABELS":{"extra":"2"}}`,
|
||||||
`{"Labels":{"own":"1","extra":"2","com.gitea.runner.job":"job-1"}}`,
|
`{"Labels":{"own":"1","extra":"2","com.gitea.runner.job.job-1":""}}`,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
resp, err := client.Post("http://docker"+testCase.path, "application/json", strings.NewReader(testCase.body))
|
resp, err := client.Post("http://docker"+testCase.path, "application/json", strings.NewReader(testCase.body))
|
||||||
@@ -211,6 +232,30 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
|
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("discovery caches only successes", func(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("automatic proxy discovery requires Linux")
|
||||||
|
}
|
||||||
|
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||||
|
tempDir := shortTempDir(t)
|
||||||
|
t.Setenv("TMPDIR", tempDir)
|
||||||
|
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||||
|
for _, job := range []string{"first", "second"} {
|
||||||
|
proxy := NewDockerProxy(t.Context(), job)
|
||||||
|
require.NotNil(t, proxy)
|
||||||
|
require.NoError(t, proxy.Close(context.Background()))
|
||||||
|
}
|
||||||
|
assert.EqualValues(t, 2, probes.Load())
|
||||||
|
workdir, err := os.Getwd()
|
||||||
|
require.NoError(t, err)
|
||||||
|
cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
|
||||||
|
cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
|
||||||
|
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||||
|
require.NotNil(t, proxy)
|
||||||
|
require.NoError(t, proxy.Close(context.Background()))
|
||||||
|
assert.EqualValues(t, 3, probes.Load())
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
|
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
|
||||||
for _, stream := range []struct{ path, upgrade string }{
|
for _, stream := range []struct{ path, upgrade string }{
|
||||||
{path: "/v1.47/exec/abc/start"},
|
{path: "/v1.47/exec/abc/start"},
|
||||||
@@ -244,7 +289,7 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, resp.Body.Close())
|
require.NoError(t, resp.Body.Close())
|
||||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||||
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job":"job-1"}}`, string(<-bodies))
|
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -276,7 +321,7 @@ func TestDockerProxy(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, resp.Body.Close())
|
require.NoError(t, resp.Body.Close())
|
||||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||||
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job":"job-1"}}`, string(<-bodies))
|
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
|
||||||
require.NoError(t, <-closed)
|
require.NoError(t, <-closed)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -285,8 +330,8 @@ func TestRemoveLabelledRemovesContainersNetworksAndVolumes(t *testing.T) {
|
|||||||
containerFailure := errors.New("container removal failed")
|
containerFailure := errors.New("container removal failed")
|
||||||
listFailure := errors.New("listing failed")
|
listFailure := errors.New("listing failed")
|
||||||
volumeFailure := errors.New("volume removal failed")
|
volumeFailure := errors.New("volume removal failed")
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
filters := make(mobyclient.Filters).Add("label", jobLabel+"=job-1")
|
filters := make(mobyclient.Filters).Add("label", jobLabel+"job-1")
|
||||||
cli := &mockDockerClient{}
|
cli := &mockDockerClient{}
|
||||||
cli.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true, Filters: filters}).
|
cli.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true, Filters: filters}).
|
||||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil).Once()
|
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil).Once()
|
||||||
@@ -312,7 +357,8 @@ func TestRemoveLabelledRemovesContainersNetworksAndVolumes(t *testing.T) {
|
|||||||
|
|
||||||
func TestDockerProxyWithDaemon(t *testing.T) {
|
func TestDockerProxyWithDaemon(t *testing.T) {
|
||||||
requireDocker(t)
|
requireDocker(t)
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
cleanupCtx := context.WithoutCancel(ctx)
|
||||||
require.NoError(t, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
|
require.NoError(t, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
|
||||||
direct, err := GetDockerClient(ctx)
|
direct, err := GetDockerClient(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -327,6 +373,9 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
|||||||
|
|
||||||
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||||
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
||||||
|
if os.Geteuid() != 0 && errors.Is(err, os.ErrPermission) {
|
||||||
|
t.Skipf("non-root runners that cannot take the daemon socket's ownership get it directly: %v", err)
|
||||||
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
||||||
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
||||||
@@ -337,22 +386,24 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
|||||||
Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "300"}, Labels: map[string]string{"own": "1"}},
|
Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "300"}, Labels: map[string]string{"own": "1"}},
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _, _ = direct.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true}) })
|
t.Cleanup(func() {
|
||||||
|
_, _ = direct.ContainerRemove(cleanupCtx, created.ID, mobyclient.ContainerRemoveOptions{Force: true})
|
||||||
|
})
|
||||||
_, err = viaProxy.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
|
_, err = viaProxy.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
net, err := viaProxy.NetworkCreate(ctx, job, mobyclient.NetworkCreateOptions{})
|
net, err := viaProxy.NetworkCreate(ctx, job, mobyclient.NetworkCreateOptions{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _, _ = direct.NetworkRemove(ctx, net.ID, mobyclient.NetworkRemoveOptions{}) })
|
t.Cleanup(func() { _, _ = direct.NetworkRemove(cleanupCtx, net.ID, mobyclient.NetworkRemoveOptions{}) })
|
||||||
_, err = viaProxy.VolumeCreate(ctx, mobyclient.VolumeCreateOptions{Name: job})
|
_, err = viaProxy.VolumeCreate(ctx, mobyclient.VolumeCreateOptions{Name: job})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _, _ = direct.VolumeRemove(ctx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
|
t.Cleanup(func() { _, _ = direct.VolumeRemove(cleanupCtx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
|
||||||
|
|
||||||
inspected, err := direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
inspected, err := direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, map[string]string{"own": "1", jobLabel: job}, inspected.Container.Config.Labels)
|
assert.Subset(t, inspected.Container.Config.Labels, map[string]string{"own": "1", jobLabel + job: ""})
|
||||||
vol, err := direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
|
vol, err := direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, job, vol.Volume.Labels[jobLabel])
|
assert.Contains(t, vol.Volume.Labels, jobLabel+job)
|
||||||
|
|
||||||
exec, err := viaProxy.ExecCreate(ctx, created.ID, mobyclient.ExecCreateOptions{Cmd: []string{"cat"}, AttachStdin: true, AttachStdout: true, TTY: true})
|
exec, err := viaProxy.ExecCreate(ctx, created.ID, mobyclient.ExecCreateOptions{Cmd: []string{"cat"}, AttachStdin: true, AttachStdout: true, TTY: true})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -426,10 +477,23 @@ func BenchmarkDockerProxy(b *testing.B) {
|
|||||||
|
|
||||||
type probeClient struct {
|
type probeClient struct {
|
||||||
mobyclient.APIClient
|
mobyclient.APIClient
|
||||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
create func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||||
stat func(string) error
|
stat func(string) error
|
||||||
mounts []container.MountPoint
|
mounts []container.MountPoint
|
||||||
|
containers []container.Summary
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
|
||||||
|
return mobyclient.ContainerListResult{Items: c.containers}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||||
|
return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||||
@@ -437,11 +501,11 @@ func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.Conta
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
||||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "probe-image"}}}, nil
|
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
return c.create(opts)
|
return c.create(ctx, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
@@ -452,9 +516,193 @@ func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobycl
|
|||||||
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||||
|
type result struct {
|
||||||
|
seen bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
for name, createErr := range map[string]error{"unavailable directory": nil, "daemon error": errors.New("probe failed")} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
key := [3]string{"daemon", dir, dir}
|
||||||
|
release := make(chan struct{})
|
||||||
|
var calls, removals atomic.Int32
|
||||||
|
visible := false
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
<-release
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
|
||||||
|
},
|
||||||
|
stat: func(_path string) error {
|
||||||
|
if visible {
|
||||||
|
return nil
|
||||||
|
} else {
|
||||||
|
return cerrdefs.ErrNotFound
|
||||||
|
}
|
||||||
|
},
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removals.Add(1)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
results := make(chan result, 8)
|
||||||
|
for range cap(results) {
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
results <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
synctest.Wait()
|
||||||
|
assert.EqualValues(t, 1, calls.Load())
|
||||||
|
replacement, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
replaced := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||||
|
replaced <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
close(release)
|
||||||
|
for range cap(results) {
|
||||||
|
result := <-results
|
||||||
|
require.ErrorIs(t, result.err, createErr)
|
||||||
|
assert.False(t, result.seen)
|
||||||
|
}
|
||||||
|
replacementResult := <-replaced
|
||||||
|
require.ErrorIs(t, replacementResult.err, createErr)
|
||||||
|
assert.False(t, replacementResult.seen)
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
if createErr == nil {
|
||||||
|
assert.EqualValues(t, 2, removals.Load())
|
||||||
|
} else {
|
||||||
|
assert.Zero(t, removals.Load())
|
||||||
|
}
|
||||||
|
createErr, visible = nil, true
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, probe.seen)
|
||||||
|
assert.Equal(t, dir, probe.directory.dir)
|
||||||
|
assert.Equal(t, dir, probe.directory.daemonDir)
|
||||||
|
assert.EqualValues(t, 3, calls.Load())
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, entries)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
key := [3]string{"daemon", dir, dir}
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
release := make(chan struct{})
|
||||||
|
var calls, removals atomic.Int32
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
<-release
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
|
||||||
|
},
|
||||||
|
stat: func(_path string) error { return nil },
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removals.Add(1)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||||
|
leader <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
follower <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
cancel()
|
||||||
|
leaderResult := <-leader
|
||||||
|
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||||
|
assert.False(t, leaderResult.seen)
|
||||||
|
go func() {
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||||
|
late <- result{probe.seen, err}
|
||||||
|
}()
|
||||||
|
synctest.Wait()
|
||||||
|
close(release)
|
||||||
|
for _, results := range []chan result{follower, late} {
|
||||||
|
result := <-results
|
||||||
|
require.NoError(t, result.err)
|
||||||
|
assert.True(t, result.seen)
|
||||||
|
}
|
||||||
|
assert.EqualValues(t, 1, calls.Load())
|
||||||
|
assert.EqualValues(t, 1, removals.Load())
|
||||||
|
_, cached := dockerProxyProbes.Load(key)
|
||||||
|
assert.True(t, cached)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
socket, err := os.Stat(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
var calls atomic.Int32
|
||||||
|
cli := &probeClient{
|
||||||
|
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
<-ctx.Done()
|
||||||
|
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||||
|
}
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||||
|
},
|
||||||
|
stat: func(_path string) error { return nil },
|
||||||
|
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, probe.seen)
|
||||||
|
assert.EqualValues(t, 2, calls.Load())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveStaleProbeContainers(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
|
||||||
|
writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
|
||||||
|
var removed []string
|
||||||
|
cli := &probeClient{
|
||||||
|
containers: []container.Summary{
|
||||||
|
{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
|
||||||
|
{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
|
||||||
|
{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
|
||||||
|
},
|
||||||
|
remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||||
|
removed = append(removed, id)
|
||||||
|
return mobyclient.ContainerRemoveResult{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
removeStaleProbeContainers(cli)
|
||||||
|
assert.Equal(t, []string{"abandoned"}, removed)
|
||||||
|
assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestDaemonSeesDir(t *testing.T) {
|
func TestDaemonSeesDir(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
markers := make(map[string]bool)
|
markers := make(map[string]bool)
|
||||||
|
|
||||||
for _, testCase := range []struct {
|
for _, testCase := range []struct {
|
||||||
name string
|
name string
|
||||||
daemonDir string
|
daemonDir string
|
||||||
@@ -471,9 +719,14 @@ func TestDaemonSeesDir(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(t.Context())
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
removed := false
|
removed := false
|
||||||
|
var attemptedImages []string
|
||||||
cli := &probeClient{
|
cli := &probeClient{
|
||||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||||
|
attemptedImages = append(attemptedImages, opts.Config.Image)
|
||||||
|
if opts.Config.Image == "removed-image" {
|
||||||
|
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrNotFound
|
||||||
|
}
|
||||||
cancel()
|
cancel()
|
||||||
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
||||||
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
||||||
@@ -501,8 +754,10 @@ func TestDaemonSeesDir(t *testing.T) {
|
|||||||
}
|
}
|
||||||
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
||||||
require.ErrorIs(t, err, testCase.removeErr)
|
require.ErrorIs(t, err, testCase.removeErr)
|
||||||
|
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
|
||||||
assert.Equal(t, testCase.seen, seen)
|
assert.Equal(t, testCase.seen, seen)
|
||||||
assert.Equal(t, testCase.removed, removed)
|
assert.Equal(t, testCase.removed, removed)
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
entries, err := os.ReadDir(dir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Empty(t, entries)
|
assert.Empty(t, entries)
|
||||||
|
|||||||
@@ -114,7 +114,7 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return imagePullOptions, err
|
return imagePullOptions, err
|
||||||
}
|
}
|
||||||
if authConfig.Username == "" && authConfig.Password == "" {
|
if authConfig.Username == "" && authConfig.Password == "" && authConfig.IdentityToken == "" && authConfig.RegistryToken == "" {
|
||||||
return imagePullOptions, nil
|
return imagePullOptions, nil
|
||||||
}
|
}
|
||||||
logger.Info("using DockerAuthConfig authentication for docker pull")
|
logger.Info("using DockerAuthConfig authentication for docker pull")
|
||||||
|
|||||||
@@ -5,20 +5,24 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"io"
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/iotest"
|
||||||
|
|
||||||
"github.com/docker/cli/cli/config"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
"github.com/sirupsen/logrus/hooks/test"
|
|
||||||
|
"github.com/moby/moby/api/pkg/authconfig"
|
||||||
|
"github.com/moby/moby/api/types/registry"
|
||||||
assert "github.com/stretchr/testify/assert"
|
assert "github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
log.SetLevel(log.DebugLevel)
|
logging.SetLevel(logging.LevelDebug)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCleanImage(t *testing.T) {
|
func TestCleanImage(t *testing.T) {
|
||||||
@@ -36,18 +40,15 @@ func TestCleanImage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
imageOut := cleanImage(context.Background(), table.imageIn)
|
imageOut := cleanImage(t.Context(), table.imageIn)
|
||||||
assert.Equal(t, table.imageOut, imageOut)
|
assert.Equal(t, table.imageOut, imageOut)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetImagePullOptions(t *testing.T) {
|
func TestGetImagePullOptions(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
|
||||||
orig := config.Dir()
|
t.Setenv("DOCKER_CONFIG", "/non-existent/docker")
|
||||||
t.Cleanup(func() { config.SetDir(orig) })
|
|
||||||
|
|
||||||
config.SetDir("/non-existent/docker")
|
|
||||||
|
|
||||||
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{})
|
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{})
|
||||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
@@ -61,13 +62,28 @@ func TestGetImagePullOptions(t *testing.T) {
|
|||||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZCJ9", options.RegistryAuth, "Username and Password should be provided")
|
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZCJ9", options.RegistryAuth, "Username and Password should be provided")
|
||||||
|
|
||||||
config.SetDir("testdata/docker-pull-options")
|
t.Setenv("DOCKER_CONFIG", "testdata/docker-pull-options")
|
||||||
|
|
||||||
options, err = getImagePullOptions(ctx, NewDockerPullExecutorInput{
|
options, err = getImagePullOptions(ctx, NewDockerPullExecutorInput{
|
||||||
Image: "nektos/act",
|
Image: "nektos/act",
|
||||||
})
|
})
|
||||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths":{"identity.example.com":{"identitytoken":"identity-token"},"registry.example.com":{"registrytoken":"registry-token"}}}`), 0o600))
|
||||||
|
t.Setenv("DOCKER_CONFIG", dir)
|
||||||
|
for host, want := range map[string]registry.AuthConfig{
|
||||||
|
"identity.example.com": {IdentityToken: "identity-token", ServerAddress: "identity.example.com"},
|
||||||
|
"registry.example.com": {RegistryToken: "registry-token", ServerAddress: "registry.example.com"},
|
||||||
|
} {
|
||||||
|
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{Image: host + "/app"})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, options.RegistryAuth, host)
|
||||||
|
got, err := authconfig.Decode(options.RegistryAuth)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, *got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A digest-pinned image is immutable, so its local copy is always current.
|
// A digest-pinned image is immutable, so its local copy is always current.
|
||||||
@@ -79,11 +95,15 @@ func TestIsPinnedImage(t *testing.T) {
|
|||||||
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
|
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
|
||||||
// whichever of the two shapes the daemon used.
|
// whichever of the two shapes the daemon used.
|
||||||
func TestLogDockerResponseError(t *testing.T) {
|
func TestLogDockerResponseError(t *testing.T) {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
streamErr := func(line string) error {
|
streamErr := func(line string) error {
|
||||||
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
|
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
|
||||||
}
|
}
|
||||||
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
||||||
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
||||||
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
||||||
|
require.NoError(t, streamErr(`{"stream":"`+strings.Repeat("x", 70000)+`"}`))
|
||||||
|
require.ErrorIs(t, logDockerResponse(logger, io.NopCloser(io.MultiReader(
|
||||||
|
strings.NewReader(`{"status":"Downloading"}`+"\n"), iotest.ErrReader(io.ErrUnexpectedEOF),
|
||||||
|
)), false), io.ErrUnexpectedEOF)
|
||||||
}
|
}
|
||||||
+186
-199
@@ -13,6 +13,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -25,13 +26,11 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
"dario.cat/mergo"
|
|
||||||
"github.com/bmatcuk/doublestar/v4"
|
"github.com/bmatcuk/doublestar/v4"
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
"github.com/docker/cli/cli/compose/loader"
|
"github.com/docker/cli/opts"
|
||||||
"github.com/docker/cli/cli/connhelper"
|
|
||||||
"github.com/joho/godotenv"
|
|
||||||
"github.com/moby/moby/api/pkg/stdcopy"
|
"github.com/moby/moby/api/pkg/stdcopy"
|
||||||
"github.com/moby/moby/api/types/container"
|
"github.com/moby/moby/api/types/container"
|
||||||
"github.com/moby/moby/api/types/mount"
|
"github.com/moby/moby/api/types/mount"
|
||||||
@@ -40,7 +39,6 @@ import (
|
|||||||
"github.com/moby/moby/client"
|
"github.com/moby/moby/client"
|
||||||
"github.com/moby/moby/client/pkg/versions"
|
"github.com/moby/moby/client/pkg/versions"
|
||||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// drainGracePeriod bounds how long we wait for an output-copy goroutine to
|
// drainGracePeriod bounds how long we wait for an output-copy goroutine to
|
||||||
@@ -52,15 +50,12 @@ const drainGracePeriod = 2 * time.Second
|
|||||||
|
|
||||||
// NewContainer creates a reference to a container
|
// NewContainer creates a reference to a container
|
||||||
func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
|
func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||||
cr := new(containerReference)
|
|
||||||
cr.input = input
|
|
||||||
// Resolved up front because the image pull runs before the container is created.
|
// Resolved up front because the image pull runs before the container is created.
|
||||||
cf := createFlagsFromOptions(input.allOptions())
|
cf := createFlagsFromOptions(input.allOptions())
|
||||||
if cf.platform != "" {
|
if cf.platform != "" {
|
||||||
cr.input.Platform = cf.platform
|
input.Platform = cf.platform
|
||||||
}
|
}
|
||||||
cr.pullPolicy = cf.pull
|
return &containerReference{input: input, pullPolicy: cf.pull}
|
||||||
return cr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// supportsContainerImagePlatform reports whether the Docker server API version
|
// supportsContainerImagePlatform reports whether the Docker server API version
|
||||||
@@ -95,8 +90,8 @@ func (cr *containerReference) Start(attach bool) common.Executor {
|
|||||||
cr.attach().IfBool(attach),
|
cr.attach().IfBool(attach),
|
||||||
cr.start(),
|
cr.start(),
|
||||||
cr.wait().IfBool(attach),
|
cr.wait().IfBool(attach),
|
||||||
cr.tryReadUID(),
|
cr.tryReadID("-u", func(id int) { cr.UID = id }),
|
||||||
cr.tryReadGID(),
|
cr.tryReadID("-g", func(id int) { cr.GID = id }),
|
||||||
func(ctx context.Context) error {
|
func(ctx context.Context) error {
|
||||||
// If this fails, then folders have wrong permissions on non root container
|
// If this fails, then folders have wrong permissions on non root container
|
||||||
if cr.UID != 0 || cr.GID != 0 {
|
if cr.UID != 0 || cr.GID != 0 {
|
||||||
@@ -210,18 +205,30 @@ func (cr *containerReference) DumpLogs(ctx context.Context) error {
|
|||||||
if cr.id == "" {
|
if cr.id == "" {
|
||||||
return cr.missingContainerError("read its logs")
|
return cr.missingContainerError("read its logs")
|
||||||
}
|
}
|
||||||
|
tty, err := cr.containerTTY(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer logs.Close()
|
defer logs.Close()
|
||||||
return cr.copyOutput(logs)
|
return cr.copyOutput(logs, tty)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cr *containerReference) containerTTY(ctx context.Context) (bool, error) {
|
||||||
|
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("failed to inspect container: %w", err)
|
||||||
|
}
|
||||||
|
return inspect.Container.Config != nil && inspect.Container.Config.Tty, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// copyOutput writes a container stream to the writers the container was created with,
|
// copyOutput writes a container stream to the writers the container was created with,
|
||||||
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
|
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
|
||||||
func (cr *containerReference) copyOutput(stream io.Reader) error {
|
func (cr *containerReference) copyOutput(stream io.Reader, tty bool) error {
|
||||||
outWriter := cr.input.Stdout
|
outWriter := cr.input.Stdout
|
||||||
if outWriter == nil {
|
if outWriter == nil {
|
||||||
outWriter = os.Stdout
|
outWriter = os.Stdout
|
||||||
@@ -232,7 +239,7 @@ func (cr *containerReference) copyOutput(stream io.Reader) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if !cr.input.AllocatePTY || os.Getenv("NORAW") != "" {
|
if !tty || os.Getenv("NORAW") != "" {
|
||||||
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
|
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
|
||||||
} else {
|
} else {
|
||||||
_, err = io.Copy(outWriter, stream)
|
_, err = io.Copy(outWriter, stream)
|
||||||
@@ -301,11 +308,29 @@ func containerInfoFromInspect(inspect container.InspectResponse) *Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun)
|
return parseEnvFile(cr, srcPath, env, false).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||||
return cr.extractFromImageEnv(env).IfNot(common.Dryrun)
|
envMap := *env
|
||||||
|
return common.Executor(func(ctx context.Context) error {
|
||||||
|
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
common.Logger(ctx).Error(err)
|
||||||
|
return fmt.Errorf("inspect container: %w", err)
|
||||||
|
}
|
||||||
|
if inspect.Container.Config == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for key, value := range opts.ConvertKVStringsToMap(inspect.Container.Config.Env) {
|
||||||
|
if envMap[key] == "" {
|
||||||
|
envMap[key] = value
|
||||||
|
} else if key == "PATH" {
|
||||||
|
envMap[key] += ":" + value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor {
|
func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor {
|
||||||
@@ -354,15 +379,13 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
|||||||
dockerHost := os.Getenv("DOCKER_HOST")
|
dockerHost := os.Getenv("DOCKER_HOST")
|
||||||
|
|
||||||
if strings.HasPrefix(dockerHost, "ssh://") {
|
if strings.HasPrefix(dockerHost, "ssh://") {
|
||||||
var helper *connhelper.ConnectionHelper
|
var args []string
|
||||||
|
if args, err = sshArgs(dockerHost); err != nil {
|
||||||
helper, err = connhelper.GetConnectionHelper(dockerHost)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
cli, err = client.New(
|
cli, err = client.New(
|
||||||
client.WithHost(helper.Host),
|
client.WithHost("http://docker.example.com"), // the dialer below ignores the address
|
||||||
client.WithDialContext(helper.Dialer),
|
client.WithDialContext(func(context.Context, string, string) (net.Conn, error) { return dialCommand("ssh", args...) }),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
cli, err = client.New(client.FromEnv)
|
cli, err = client.New(client.FromEnv)
|
||||||
@@ -380,17 +403,16 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
|||||||
return cli, nil
|
return cli, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetHostInfo(ctx context.Context) (info system.Info, err error) {
|
func GetHostInfo(ctx context.Context) (system.Info, error) {
|
||||||
var cli client.APIClient
|
cli, err := GetDockerClient(ctx)
|
||||||
cli, err = GetDockerClient(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return info, err
|
return system.Info{}, err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
|
|
||||||
result, err := cli.Info(ctx, client.InfoOptions{})
|
result, err := cli.Info(ctx, client.InfoOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return info, err
|
return system.Info{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.Info, nil
|
return result.Info, nil
|
||||||
@@ -462,22 +484,23 @@ func (cr *containerReference) find() common.Executor {
|
|||||||
}
|
}
|
||||||
cr.id = ""
|
cr.id = ""
|
||||||
}
|
}
|
||||||
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{
|
result, err := cr.cli.ContainerInspect(ctx, cr.input.Name, client.ContainerInspectOptions{})
|
||||||
All: true,
|
if err == nil && result.Container.Name == "/"+cr.input.Name {
|
||||||
})
|
cr.id = result.Container.ID
|
||||||
|
return nil
|
||||||
|
} else if cerrdefs.IsNotFound(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{All: true}) // inspect also matches ids, and podman fails it with a 500 for a container removed mid-request
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to list containers: %w", err)
|
return fmt.Errorf("failed to list containers: %w", err)
|
||||||
}
|
}
|
||||||
|
for _, candidate := range containers.Items {
|
||||||
for _, c := range containers.Items {
|
if slices.Contains(candidate.Names, "/"+cr.input.Name) {
|
||||||
for _, name := range c.Names {
|
cr.id = candidate.ID
|
||||||
if name[1:] == cr.input.Name {
|
break
|
||||||
cr.id = c.ID
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -488,7 +511,7 @@ func (cr *containerReference) remove() common.Executor {
|
|||||||
if idOrName == "" && cr.input != nil {
|
if idOrName == "" && cr.input != nil {
|
||||||
idOrName = cr.input.Name
|
idOrName = cr.input.Name
|
||||||
}
|
}
|
||||||
if idOrName == "" {
|
if idOrName == "" || cr.cli == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -544,46 +567,32 @@ func (input *NewContainerInput) allOptions() string {
|
|||||||
return strings.TrimSpace(input.RunnerOptions + " " + input.WorkflowOptions)
|
return strings.TrimSpace(input.RunnerOptions + " " + input.WorkflowOptions)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
|
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
options := cr.input.allOptions()
|
options := cr.input.allOptions()
|
||||||
|
|
||||||
if options == "" {
|
if options == "" {
|
||||||
return config, hostConfig, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// For Gitea, checked here because the parse below is what would read those files
|
// For Gitea, checked here because the parse below is what would read those files
|
||||||
if err := rejectHostReadingOptions(cr.input.WorkflowOptions); err != nil {
|
if err := rejectHostReadingOptions(cr.input.WorkflowOptions); err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse configuration from CLI container.options
|
// parse configuration from CLI container.options
|
||||||
flags, copts, cf, err := parseContainerOptions(options)
|
flags, copts, cf, err := parseContainerOptions(options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cf.validate(); err != nil {
|
if err := cf.validate(); err != nil {
|
||||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment.
|
|
||||||
// In the old fork version, the code is
|
|
||||||
// if len(copts.netMode.Value()) == 0 {
|
|
||||||
// if err = copts.netMode.Set("host"); err != nil {
|
|
||||||
// return nil, nil, fmt.Errorf("cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// And it has been commented with:
|
|
||||||
// If a service container's network is set to `host`, the container will not be able to
|
|
||||||
// connect to the specified network created for the job container and the service containers.
|
|
||||||
// So comment out the following code.
|
|
||||||
// Not the if it's necessary to comment it in the new version,
|
|
||||||
// since it's cr.input.NetworkMode now.
|
|
||||||
|
|
||||||
if len(copts.netMode.Value()) == 0 {
|
if len(copts.netMode.Value()) == 0 {
|
||||||
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
|
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
|
||||||
return nil, nil, fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
return fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -596,7 +605,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
|||||||
|
|
||||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||||
}
|
}
|
||||||
// workflow aliases join the runner's own, deduped so a re-create cannot grow the input
|
// workflow aliases join the runner's own, deduped so a re-create cannot grow the input
|
||||||
for _, endpoint := range containerConfig.NetworkingConfig.EndpointsConfig {
|
for _, endpoint := range containerConfig.NetworkingConfig.EndpointsConfig {
|
||||||
@@ -611,30 +620,24 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
|||||||
if !hostConfig.Privileged {
|
if !hostConfig.Privileged {
|
||||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig, trusted)
|
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig, trusted)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config)
|
logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config)
|
||||||
|
|
||||||
err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice)
|
overlay(reflect.ValueOf(config).Elem(), reflect.ValueOf(containerConfig.Config).Elem(), true)
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("cannot merge container.Config options: '%s': '%w'", options, err)
|
|
||||||
}
|
|
||||||
logger.Debugf("Merged container.Config ==> %+v", config)
|
logger.Debugf("Merged container.Config ==> %+v", config)
|
||||||
|
|
||||||
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
|
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
|
||||||
|
|
||||||
cr.sanitizeConfig(ctx, nil, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
|
cr.sanitizeConfig(ctx, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
|
||||||
overlayVolumes(hostConfig, containerConfig.HostConfig)
|
overlayVolumes(hostConfig, containerConfig.HostConfig)
|
||||||
binds := hostConfig.Binds
|
binds := hostConfig.Binds
|
||||||
mounts := hostConfig.Mounts
|
mounts := hostConfig.Mounts
|
||||||
networkMode := hostConfig.NetworkMode
|
networkMode := hostConfig.NetworkMode
|
||||||
err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride)
|
overlay(reflect.ValueOf(hostConfig).Elem(), reflect.ValueOf(containerConfig.HostConfig).Elem(), false)
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("cannot merge container.HostConfig options: '%s': '%w'", options, err)
|
|
||||||
}
|
|
||||||
hostConfig.Binds = binds
|
hostConfig.Binds = binds
|
||||||
hostConfig.Mounts = mounts
|
hostConfig.Mounts = mounts
|
||||||
if cf.name != "" {
|
if cf.name != "" {
|
||||||
@@ -644,10 +647,13 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
|||||||
if flags.Changed("network") || flags.Changed("net") {
|
if flags.Changed("network") || flags.Changed("net") {
|
||||||
logger.Warn("--network and --net in the options will be ignored.")
|
logger.Warn("--network and --net in the options will be ignored.")
|
||||||
}
|
}
|
||||||
|
if len(cr.input.Entrypoint) != 0 && flags.Changed("entrypoint") {
|
||||||
|
logger.Warn("--entrypoint in the options will be ignored.")
|
||||||
|
}
|
||||||
hostConfig.NetworkMode = networkMode
|
hostConfig.NetworkMode = networkMode
|
||||||
logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
|
logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
|
||||||
|
|
||||||
return config, hostConfig, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||||
@@ -673,17 +679,11 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
|||||||
ExposedPorts: exposedPorts,
|
ExposedPorts: exposedPorts,
|
||||||
Tty: input.AllocatePTY,
|
Tty: input.AllocatePTY,
|
||||||
}
|
}
|
||||||
// For Gitea, reduce log noise
|
|
||||||
// logger.Debugf("Common container.Config ==> %+v", config)
|
|
||||||
|
|
||||||
if len(input.Cmd) != 0 {
|
if len(input.Cmd) != 0 {
|
||||||
config.Cmd = input.Cmd
|
config.Cmd = input.Cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(input.Entrypoint) != 0 {
|
|
||||||
config.Entrypoint = input.Entrypoint
|
|
||||||
}
|
|
||||||
|
|
||||||
mounts := make([]mount.Mount, 0)
|
mounts := make([]mount.Mount, 0)
|
||||||
for mountSource, mountTarget := range input.Mounts {
|
for mountSource, mountTarget := range input.Mounts {
|
||||||
mounts = append(mounts, mount.Mount{
|
mounts = append(mounts, mount.Mount{
|
||||||
@@ -718,29 +718,25 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
|||||||
PortBindings: portBindings,
|
PortBindings: portBindings,
|
||||||
AutoRemove: input.AutoRemove,
|
AutoRemove: input.AutoRemove,
|
||||||
}
|
}
|
||||||
// For Gitea, reduce log noise
|
|
||||||
// logger.Debugf("Common container.HostConfig ==> %+v", hostConfig)
|
|
||||||
|
|
||||||
config, hostConfig, err = cr.mergeContainerConfigs(ctx, config, hostConfig)
|
if err := cr.mergeContainerConfigs(ctx, config, hostConfig); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(input.Entrypoint) != 0 {
|
||||||
|
config.Entrypoint = input.Entrypoint
|
||||||
|
}
|
||||||
|
|
||||||
// For Gitea
|
// For Gitea
|
||||||
config, hostConfig = cr.sanitizeConfig(ctx, config, hostConfig)
|
cr.sanitizeConfig(ctx, hostConfig)
|
||||||
|
|
||||||
var networkingConfig *network.NetworkingConfig
|
var networkingConfig *network.NetworkingConfig
|
||||||
// For Gitea, reduce log noise
|
|
||||||
// logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases)
|
|
||||||
n := hostConfig.NetworkMode
|
n := hostConfig.NetworkMode
|
||||||
// IsUserDefined and IsHost are broken on windows
|
// IsUserDefined and IsHost are broken on windows
|
||||||
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
||||||
endpointConfig := &network.EndpointSettings{
|
|
||||||
Aliases: input.NetworkAliases,
|
|
||||||
}
|
|
||||||
networkingConfig = &network.NetworkingConfig{
|
networkingConfig = &network.NetworkingConfig{
|
||||||
EndpointsConfig: map[string]*network.EndpointSettings{
|
EndpointsConfig: map[string]*network.EndpointSettings{
|
||||||
input.NetworkMode: endpointConfig,
|
input.NetworkMode: {Aliases: input.NetworkAliases},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -764,44 +760,6 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) extractFromImageEnv(env *map[string]string) common.Executor {
|
|
||||||
envMap := *env
|
|
||||||
return func(ctx context.Context) error {
|
|
||||||
logger := common.Logger(ctx)
|
|
||||||
|
|
||||||
inspect, err := cr.cli.ImageInspect(ctx, cr.input.Image)
|
|
||||||
if err != nil {
|
|
||||||
logger.Error(err)
|
|
||||||
return fmt.Errorf("inspect image: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if inspect.Config == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
imageEnv, err := godotenv.Unmarshal(strings.Join(inspect.Config.Env, "\n"))
|
|
||||||
if err != nil {
|
|
||||||
logger.Error(err)
|
|
||||||
return fmt.Errorf("unmarshal image env: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for k, v := range imageEnv {
|
|
||||||
if k == "PATH" {
|
|
||||||
if envMap[k] == "" {
|
|
||||||
envMap[k] = v
|
|
||||||
} else {
|
|
||||||
envMap[k] += `:` + v
|
|
||||||
}
|
|
||||||
} else if envMap[k] == "" {
|
|
||||||
envMap[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
env = &envMap
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (cr *containerReference) exec(cmd []string, env map[string]string, user, workdir string) common.Executor {
|
func (cr *containerReference) exec(cmd []string, env map[string]string, user, workdir string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
if cr.id == "" {
|
if cr.id == "" {
|
||||||
@@ -819,20 +777,11 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
|||||||
|
|
||||||
logger.Debugf("Exec command '%s'", cmd)
|
logger.Debugf("Exec command '%s'", cmd)
|
||||||
isTerminal := cr.input.AllocatePTY
|
isTerminal := cr.input.AllocatePTY
|
||||||
envList := make([]string, 0)
|
wd := cr.input.WorkingDir
|
||||||
for k, v := range env {
|
if strings.HasPrefix(workdir, "/") {
|
||||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
wd = workdir
|
||||||
}
|
} else if workdir != "" {
|
||||||
|
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
||||||
var wd string
|
|
||||||
if workdir != "" {
|
|
||||||
if strings.HasPrefix(workdir, "/") {
|
|
||||||
wd = workdir
|
|
||||||
} else {
|
|
||||||
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
wd = cr.input.WorkingDir
|
|
||||||
}
|
}
|
||||||
logger.Debugf("Working directory '%s'", wd)
|
logger.Debugf("Working directory '%s'", wd)
|
||||||
|
|
||||||
@@ -840,7 +789,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
|||||||
User: user,
|
User: user,
|
||||||
Cmd: cmd,
|
Cmd: cmd,
|
||||||
WorkingDir: wd,
|
WorkingDir: wd,
|
||||||
Env: envList,
|
Env: getEnvListFromMap(env),
|
||||||
TTY: isTerminal,
|
TTY: isTerminal,
|
||||||
AttachStderr: true,
|
AttachStderr: true,
|
||||||
AttachStdout: true,
|
AttachStdout: true,
|
||||||
@@ -857,8 +806,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
|||||||
}
|
}
|
||||||
defer resp.Close()
|
defer resp.Close()
|
||||||
|
|
||||||
err = cr.waitForCommand(ctx, resp.HijackedResponse, idResp, user, workdir)
|
if err := cr.waitForCommand(ctx, resp.HijackedResponse); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -907,15 +855,7 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) tryReadUID() common.Executor {
|
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse) error {
|
||||||
return cr.tryReadID("-u", func(id int) { cr.UID = id })
|
|
||||||
}
|
|
||||||
|
|
||||||
func (cr *containerReference) tryReadGID() common.Executor {
|
|
||||||
return cr.tryReadID("-g", func(id int) { cr.GID = id })
|
|
||||||
}
|
|
||||||
|
|
||||||
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse, _ client.ExecCreateResult, _, _ string) error {
|
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
|
|
||||||
// Buffered so the copy goroutine never blocks on send if the grace-period
|
// Buffered so the copy goroutine never blocks on send if the grace-period
|
||||||
@@ -923,7 +863,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
cmdResponse := make(chan error, 1)
|
cmdResponse := make(chan error, 1)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
cmdResponse <- cr.copyOutput(resp.Reader)
|
cmdResponse <- cr.copyOutput(resp.Reader, cr.input.AllocatePTY)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -939,7 +879,10 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
// truncate the tail of the log. The goroutine exits once the hijacked
|
// truncate the tail of the log. The goroutine exits once the hijacked
|
||||||
// stream is closed by resp.Close() in the caller's defer.
|
// stream is closed by resp.Close() in the caller's defer.
|
||||||
select {
|
select {
|
||||||
case <-cmdResponse:
|
case err := <-cmdResponse:
|
||||||
|
if err != nil {
|
||||||
|
logger.Debugf("Failed to drain command output after cancellation: %v", err)
|
||||||
|
}
|
||||||
case <-time.After(drainGracePeriod):
|
case <-time.After(drainGracePeriod):
|
||||||
logger.Warn("Timed out draining command output after cancellation")
|
logger.Warn("Timed out draining command output after cancellation")
|
||||||
}
|
}
|
||||||
@@ -948,15 +891,12 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
|||||||
// from executing
|
// from executing
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case err := <-cmdResponse:
|
case err := <-cmdResponse:
|
||||||
if err != nil {
|
return err
|
||||||
logger.Error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||||
|
dstPath = path.Clean(dstPath)
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
if cr.id == "" {
|
if cr.id == "" {
|
||||||
return cr.missingContainerError("copy directory to %s", dstPath)
|
return cr.missingContainerError("copy directory to %s", dstPath)
|
||||||
@@ -967,34 +907,27 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
||||||
defer func(tarFile *os.File) {
|
defer func() {
|
||||||
name := tarFile.Name()
|
if err := tarFile.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||||
err := tarFile.Close()
|
|
||||||
if !errors.Is(err, os.ErrClosed) {
|
|
||||||
logger.Error(err)
|
logger.Error(err)
|
||||||
}
|
}
|
||||||
err = os.Remove(name)
|
if err := os.Remove(tarFile.Name()); err != nil {
|
||||||
if err != nil {
|
|
||||||
logger.Error(err)
|
logger.Error(err)
|
||||||
}
|
}
|
||||||
}(tarFile)
|
}()
|
||||||
if err := writeDirTar(ctx, tarFile, dstPath, srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
extractionRoot := "/"
|
||||||
|
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||||
|
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||||
|
}
|
||||||
|
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath)
|
logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath)
|
||||||
_, err = tarFile.Seek(0, 0)
|
if _, err := tarFile.Seek(0, 0); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to seek tar archive: %w", err)
|
return fmt.Errorf("failed to seek tar archive: %w", err)
|
||||||
}
|
}
|
||||||
_, err = cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
|
return cr.upload(ctx, extractionRoot, tarFile)
|
||||||
DestinationPath: "/",
|
|
||||||
Content: tarFile,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to copy content to container: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1009,19 +942,23 @@ func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) c
|
|||||||
}
|
}
|
||||||
|
|
||||||
common.Logger(ctx).Debugf("Extracting content to '%s'", dstPath)
|
common.Logger(ctx).Debugf("Extracting content to '%s'", dstPath)
|
||||||
_, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
|
return cr.upload(ctx, dstPath, &buf)
|
||||||
DestinationPath: dstPath,
|
|
||||||
Content: &buf,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to copy content to container: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cr *containerReference) upload(ctx context.Context, dstPath string, content io.Reader) error {
|
||||||
|
if _, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{DestinationPath: dstPath, Content: content}); err != nil {
|
||||||
|
return fmt.Errorf("failed to copy content to container: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (cr *containerReference) attach() common.Executor {
|
func (cr *containerReference) attach() common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
|
tty, err := cr.containerTTY(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
||||||
Stream: true,
|
Stream: true,
|
||||||
Stdout: true,
|
Stdout: true,
|
||||||
@@ -1032,9 +969,12 @@ func (cr *containerReference) attach() common.Executor {
|
|||||||
}
|
}
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
cr.attachDone = done
|
cr.attachDone = done
|
||||||
|
stopClose := context.AfterFunc(ctx, out.Close)
|
||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
if copyErr := cr.copyOutput(out.Reader); copyErr != nil {
|
defer out.Close()
|
||||||
|
defer stopClose()
|
||||||
|
if copyErr := cr.copyOutput(out.Reader, tty); copyErr != nil && ctx.Err() == nil {
|
||||||
common.Logger(ctx).Error(copyErr)
|
common.Logger(ctx).Error(copyErr)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -1069,6 +1009,9 @@ func (cr *containerReference) wait() common.Executor {
|
|||||||
return fmt.Errorf("failed to wait for container: %w", err)
|
return fmt.Errorf("failed to wait for container: %w", err)
|
||||||
}
|
}
|
||||||
case status := <-waitResult.Result:
|
case status := <-waitResult.Result:
|
||||||
|
if status.Error != nil {
|
||||||
|
return fmt.Errorf("failed to wait for container: %s", status.Error.Message)
|
||||||
|
}
|
||||||
statusCode = status.StatusCode
|
statusCode = status.StatusCode
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1098,7 +1041,7 @@ func (cr *containerReference) wait() common.Executor {
|
|||||||
// sanitizeOptionsHostConfig takes back everything a workflow could escape the container with,
|
// sanitizeOptionsHostConfig takes back everything a workflow could escape the container with,
|
||||||
// setting each field to trusted, which is what the runner's own options parse to on their own.
|
// setting each field to trusted, which is what the runner's own options parse to on their own.
|
||||||
// Only for unprivileged mode, since privileged mode grants host access anyway.
|
// Only for unprivileged mode, since privileged mode grants host access anyway.
|
||||||
func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *container.HostConfig) {
|
func sanitizeOptionsHostConfig(logger *logging.Logger, hostConfig, trusted *container.HostConfig) {
|
||||||
resetOption(logger, "--pid", &hostConfig.PidMode, trusted.PidMode)
|
resetOption(logger, "--pid", &hostConfig.PidMode, trusted.PidMode)
|
||||||
resetOption(logger, "--ipc", &hostConfig.IpcMode, trusted.IpcMode)
|
resetOption(logger, "--ipc", &hostConfig.IpcMode, trusted.IpcMode)
|
||||||
resetOption(logger, "--uts", &hostConfig.UTSMode, trusted.UTSMode)
|
resetOption(logger, "--uts", &hostConfig.UTSMode, trusted.UTSMode)
|
||||||
@@ -1137,7 +1080,7 @@ func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *c
|
|||||||
|
|
||||||
// resetOption puts a field back to the runner's own value. It compares the values rather than
|
// resetOption puts a field back to the runner's own value. It compares the values rather than
|
||||||
// the flags, so a field that more than one option feeds cannot slip through.
|
// the flags, so a field that more than one option feeds cannot slip through.
|
||||||
func resetOption[T any](logger logrus.FieldLogger, option string, field *T, trusted T) {
|
func resetOption[T any](logger *logging.Logger, option string, field *T, trusted T) {
|
||||||
if reflect.DeepEqual(*field, trusted) {
|
if reflect.DeepEqual(*field, trusted) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1159,9 +1102,7 @@ func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
|||||||
return containerConfig.HostConfig, nil
|
return containerConfig.HostConfig, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// For Gitea
|
func (cr *containerReference) sanitizeConfig(ctx context.Context, hostConfig *container.HostConfig) {
|
||||||
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
|
|
||||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
|
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||||
|
|
||||||
@@ -1169,7 +1110,7 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
|||||||
// sanitize binds
|
// sanitize binds
|
||||||
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
|
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
|
||||||
for _, bind := range hostConfig.Binds {
|
for _, bind := range hostConfig.Binds {
|
||||||
parsed, err := loader.ParseVolume(bind)
|
parsed, err := ParseVolume(bind)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf("parse volume [%s] error: %v", bind, err)
|
logger.Warnf("parse volume [%s] error: %v", bind, err)
|
||||||
continue
|
continue
|
||||||
@@ -1179,8 +1120,8 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
|||||||
sanitizedBinds = append(sanitizedBinds, bind)
|
sanitizedBinds = append(sanitizedBinds, bind)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type), parsed.ReadOnly) {
|
if matcher.isValid(parsed.Source, parsed.Type, parsed.ReadOnly) {
|
||||||
if matcher.needsNoCopy(parsed.Source, mount.Type(parsed.Type)) && (parsed.Volume == nil || !parsed.Volume.NoCopy) {
|
if matcher.needsNoCopy(parsed.Source, parsed.Type) && !parsed.NoCopy {
|
||||||
bind += ",nocopy"
|
bind += ",nocopy"
|
||||||
}
|
}
|
||||||
sanitizedBinds = append(sanitizedBinds, bind)
|
sanitizedBinds = append(sanitizedBinds, bind)
|
||||||
@@ -1210,19 +1151,65 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
hostConfig.Mounts = sanitizedMounts
|
hostConfig.Mounts = sanitizedMounts
|
||||||
|
|
||||||
return config, hostConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func SanitizeBinds(ctx context.Context, validVolumes, binds []string) []string {
|
func SanitizeBinds(ctx context.Context, validVolumes, binds []string) []string {
|
||||||
cr := &containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}
|
cr := &containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}
|
||||||
_, hostConfig := cr.sanitizeConfig(ctx, nil, &container.HostConfig{Binds: binds})
|
hostConfig := &container.HostConfig{Binds: binds}
|
||||||
|
cr.sanitizeConfig(ctx, hostConfig)
|
||||||
return hostConfig.Binds
|
return hostConfig.Binds
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func overlay(dst, src reflect.Value, appendSlices bool) {
|
||||||
|
switch dst.Kind() {
|
||||||
|
case reflect.Struct:
|
||||||
|
opaque := true
|
||||||
|
for i := range dst.NumField() {
|
||||||
|
if dst.Type().Field(i).IsExported() {
|
||||||
|
opaque = false
|
||||||
|
overlay(dst.Field(i), src.Field(i), appendSlices)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if opaque {
|
||||||
|
dst.Set(src)
|
||||||
|
}
|
||||||
|
case reflect.Map:
|
||||||
|
if src.IsNil() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if dst.IsNil() {
|
||||||
|
dst.Set(reflect.MakeMap(dst.Type()))
|
||||||
|
}
|
||||||
|
for key, value := range src.Seq2() {
|
||||||
|
dst.SetMapIndex(key, value)
|
||||||
|
}
|
||||||
|
case reflect.Slice:
|
||||||
|
if appendSlices {
|
||||||
|
dst.Set(reflect.AppendSlice(dst, src))
|
||||||
|
} else if src.Len() > 0 {
|
||||||
|
dst.Set(src)
|
||||||
|
}
|
||||||
|
case reflect.Pointer:
|
||||||
|
if src.IsNil() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if dst.IsNil() {
|
||||||
|
dst.Set(src)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
overlay(dst.Elem(), src.Elem(), appendSlices)
|
||||||
|
case reflect.Array:
|
||||||
|
dst.Set(src)
|
||||||
|
default:
|
||||||
|
if !src.IsZero() {
|
||||||
|
dst.Set(src)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
|
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
|
||||||
func bindTarget(bind string) string {
|
func bindTarget(bind string) string {
|
||||||
parsed, err := loader.ParseVolume(bind)
|
parsed, err := ParseVolume(bind)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|||||||
+301
-227
@@ -5,6 +5,7 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"archive/tar"
|
||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
@@ -12,12 +13,18 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
cerrdefs "github.com/containerd/errdefs"
|
cerrdefs "github.com/containerd/errdefs"
|
||||||
"github.com/moby/moby/api/pkg/stdcopy"
|
"github.com/moby/moby/api/pkg/stdcopy"
|
||||||
@@ -25,50 +32,33 @@ import (
|
|||||||
"github.com/moby/moby/api/types/mount"
|
"github.com/moby/moby/api/types/mount"
|
||||||
"github.com/moby/moby/api/types/network"
|
"github.com/moby/moby/api/types/network"
|
||||||
mobyclient "github.com/moby/moby/client"
|
mobyclient "github.com/moby/moby/client"
|
||||||
"github.com/sirupsen/logrus/hooks/test"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/mock"
|
"github.com/stretchr/testify/mock"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDocker(t *testing.T) {
|
func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
|
||||||
requireDocker(t)
|
client := &mockDockerClient{}
|
||||||
ctx := context.Background()
|
client.On("ContainerInspect", mock.Anything, "amd64-job", mobyclient.ContainerInspectOptions{}).
|
||||||
client, err := GetDockerClient(ctx)
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Env: []string{
|
||||||
require.NoError(t, err)
|
"PATH=/opt/node/x64/bin:/usr/bin:/bin", "CONFLICT_VAR=container-value", "SOME_RANDOM_VAR=",
|
||||||
defer client.Close()
|
"ANOTHER_ONE=old-value", "ANOTHER_ONE=BUT_I_HAVE_VALUE", "LITERAL='quoted'=$HOME\nsecond line",
|
||||||
|
}}}}, nil)
|
||||||
dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
|
||||||
ContextDir: "testdata",
|
|
||||||
ImageTag: "envmergetest",
|
|
||||||
})
|
|
||||||
|
|
||||||
err = dockerBuild(ctx)
|
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
cr := &containerReference{
|
|
||||||
cli: client,
|
|
||||||
input: &NewContainerInput{
|
|
||||||
Image: "envmergetest",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin",
|
"PATH": "/custom/bin",
|
||||||
"RANDOM_VAR": "WITH_VALUE",
|
"RANDOM_VAR": "WITH_VALUE",
|
||||||
"ANOTHER_VAR": "",
|
"ANOTHER_VAR": "",
|
||||||
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
|
"CONFLICT_VAR": "step-value",
|
||||||
}
|
}
|
||||||
|
require.NoError(t, (&containerReference{cli: client, id: "amd64-job"}).UpdateFromImageEnv(&env)(t.Context()))
|
||||||
envExecutor := cr.extractFromImageEnv(&env)
|
|
||||||
err = envExecutor(ctx)
|
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
assert.Equal(t, map[string]string{
|
assert.Equal(t, map[string]string{
|
||||||
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin:/this/path/does/not/exists/anywhere:/this/either",
|
"PATH": "/custom/bin:/opt/node/x64/bin:/usr/bin:/bin",
|
||||||
"RANDOM_VAR": "WITH_VALUE",
|
"RANDOM_VAR": "WITH_VALUE",
|
||||||
"ANOTHER_VAR": "",
|
"ANOTHER_VAR": "",
|
||||||
"SOME_RANDOM_VAR": "",
|
"SOME_RANDOM_VAR": "",
|
||||||
"ANOTHER_ONE": "BUT_I_HAVE_VALUE",
|
"ANOTHER_ONE": "BUT_I_HAVE_VALUE",
|
||||||
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
|
"CONFLICT_VAR": "step-value",
|
||||||
|
"LITERAL": "'quoted'=$HOME\nsecond line",
|
||||||
}, env)
|
}, env)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,6 +92,16 @@ func (m *mockDockerClient) ContainerAttach(ctx context.Context, containerID stri
|
|||||||
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
|
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockDockerClient) ContainerLogs(ctx context.Context, containerID string, opts mobyclient.ContainerLogsOptions) (mobyclient.ContainerLogsResult, error) {
|
||||||
|
args := m.Called(ctx, containerID, opts)
|
||||||
|
return args.Get(0).(mobyclient.ContainerLogsResult), args.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockDockerClient) ContainerStart(ctx context.Context, containerID string, opts mobyclient.ContainerStartOptions) (mobyclient.ContainerStartResult, error) {
|
||||||
|
args := m.Called(ctx, containerID, opts)
|
||||||
|
return args.Get(0).(mobyclient.ContainerStartResult), args.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
|
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
|
||||||
args := m.Called(ctx, containerID, opts)
|
args := m.Called(ctx, containerID, opts)
|
||||||
return args.Get(0).(mobyclient.ContainerWaitResult)
|
return args.Get(0).(mobyclient.ContainerWaitResult)
|
||||||
@@ -168,6 +168,7 @@ func (r *interruptReader) Read(_ []byte) (int, error) {
|
|||||||
type mockConn struct {
|
type mockConn struct {
|
||||||
net.Conn
|
net.Conn
|
||||||
mock.Mock
|
mock.Mock
|
||||||
|
closed atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *mockConn) Write(b []byte) (n int, err error) {
|
func (m *mockConn) Write(b []byte) (n int, err error) {
|
||||||
@@ -176,11 +177,12 @@ func (m *mockConn) Write(b []byte) (n int, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *mockConn) Close() (err error) {
|
func (m *mockConn) Close() (err error) {
|
||||||
|
m.closed.Store(true)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerExecAbort(t *testing.T) {
|
func TestDockerExecAbort(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
|
||||||
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
|
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
|
||||||
conn := &mockConn{}
|
conn := &mockConn{}
|
||||||
@@ -196,11 +198,9 @@ func TestDockerExecAbort(t *testing.T) {
|
|||||||
}, nil)
|
}, nil)
|
||||||
|
|
||||||
cr := &containerReference{
|
cr := &containerReference{
|
||||||
id: "123",
|
id: "123",
|
||||||
cli: client,
|
cli: client,
|
||||||
input: &NewContainerInput{
|
input: &NewContainerInput{},
|
||||||
Image: "image",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
channel := make(chan error)
|
channel := make(chan error)
|
||||||
@@ -221,14 +221,12 @@ func TestDockerExecAbort(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerExecFailure(t *testing.T) {
|
func TestDockerExecFailure(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
|
||||||
conn := &mockConn{}
|
|
||||||
|
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||||
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
|
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
|
||||||
Conn: conn,
|
Conn: &mockConn{},
|
||||||
Reader: bufio.NewReader(strings.NewReader("output")),
|
Reader: bufio.NewReader(strings.NewReader("output")),
|
||||||
}, nil)
|
}, nil)
|
||||||
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
|
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
|
||||||
@@ -236,11 +234,9 @@ func TestDockerExecFailure(t *testing.T) {
|
|||||||
}, nil)
|
}, nil)
|
||||||
|
|
||||||
cr := &containerReference{
|
cr := &containerReference{
|
||||||
id: "123",
|
id: "123",
|
||||||
cli: client,
|
cli: client,
|
||||||
input: &NewContainerInput{
|
input: &NewContainerInput{},
|
||||||
Image: "image",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err := cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
|
err := cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
|
||||||
@@ -249,7 +245,21 @@ func TestDockerExecFailure(t *testing.T) {
|
|||||||
assert.Equal(t, ExitCodeError(1), exitErr)
|
assert.Equal(t, ExitCodeError(1), exitErr)
|
||||||
assert.Equal(t, "Process completed with exit code 1.", err.Error())
|
assert.Equal(t, "Process completed with exit code 1.", err.Error())
|
||||||
|
|
||||||
conn.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDockerExecStreamFailure(t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).
|
||||||
|
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||||
|
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).
|
||||||
|
Return(mobyclient.ExecAttachResult{
|
||||||
|
Conn: &mockConn{},
|
||||||
|
Reader: bufio.NewReader(bytes.NewReader(stdcopyFrame(stdcopy.Systemerr, "unable to start exec"))),
|
||||||
|
}, nil)
|
||||||
|
require.ErrorContains(t, (&containerReference{id: "123", cli: client, input: &NewContainerInput{}}).
|
||||||
|
exec([]string{"sh"}, nil, "", "")(ctx), "unable to start exec")
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,86 +274,89 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDockerAttachFlushesTrailingLine verifies that wait() blocks until the
|
|
||||||
// attach() streaming goroutine has drained and flushed the container's output,
|
|
||||||
// so a final line without a trailing newline is not lost.
|
|
||||||
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||||
ctx := context.Background()
|
for _, testcase := range []struct {
|
||||||
|
name string
|
||||||
|
tty bool
|
||||||
|
}{
|
||||||
|
{name: "multiplexed"},
|
||||||
|
{name: "tty from container options", tty: true},
|
||||||
|
} {
|
||||||
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
output := []byte("line one\nlast line without newline")
|
||||||
|
if !testcase.tty {
|
||||||
|
output = stdcopyFrame(stdcopy.Stdout, string(output))
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
logWriter := common.NewLineWriter(func(line string) bool {
|
||||||
|
lines = append(lines, line)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Tty: testcase.tty}}}, nil)
|
||||||
|
conn := &mockConn{}
|
||||||
|
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||||
|
Return(mobyclient.ContainerAttachResult{
|
||||||
|
Conn: conn,
|
||||||
|
Reader: bufio.NewReader(bytes.NewReader(output)),
|
||||||
|
}, nil)
|
||||||
|
client.On("ContainerLogs", ctx, "123", mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}).
|
||||||
|
Return(io.NopCloser(bytes.NewReader(output)), nil)
|
||||||
|
statusCh := make(chan container.WaitResponse, 1)
|
||||||
|
statusCh <- container.WaitResponse{StatusCode: 0}
|
||||||
|
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||||
|
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||||
|
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{Stdout: logWriter, Stderr: logWriter}}
|
||||||
|
|
||||||
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
|
require.NoError(t, cr.attach()(ctx))
|
||||||
|
require.NoError(t, cr.wait()(ctx))
|
||||||
var lines []string
|
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
||||||
logWriter := common.NewLineWriter(func(s string) bool {
|
assert.True(t, conn.closed.Load())
|
||||||
lines = append(lines, s)
|
require.NoError(t, cr.DumpLogs(ctx))
|
||||||
return true
|
assert.Equal(t, []string{"line one\n", "last line without newline", "line one\n", "last line without newline"}, lines)
|
||||||
})
|
client.AssertExpectations(t)
|
||||||
|
|
||||||
client := &mockDockerClient{}
|
|
||||||
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
|
||||||
Return(mobyclient.ContainerAttachResult{
|
|
||||||
Conn: &mockConn{},
|
|
||||||
Reader: bufio.NewReader(framed),
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
statusCh := make(chan container.WaitResponse, 1)
|
|
||||||
statusCh <- container.WaitResponse{StatusCode: 0}
|
|
||||||
errCh := make(chan error, 1)
|
|
||||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
|
||||||
Return(mobyclient.ContainerWaitResult{
|
|
||||||
Result: (<-chan container.WaitResponse)(statusCh),
|
|
||||||
Error: (<-chan error)(errCh),
|
|
||||||
})
|
})
|
||||||
|
|
||||||
cr := &containerReference{
|
|
||||||
id: "123",
|
|
||||||
cli: client,
|
|
||||||
input: &NewContainerInput{
|
|
||||||
Image: "image",
|
|
||||||
Stdout: logWriter,
|
|
||||||
Stderr: logWriter,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
require.NoError(t, cr.attach()(ctx))
|
func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
|
||||||
require.NoError(t, cr.wait()(ctx))
|
conn, peer := net.Pipe()
|
||||||
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
// wait() must have blocked until the goroutine drained AND flushed; the
|
t.Cleanup(func() { _ = peer.Close() })
|
||||||
// trailing, non-newline-terminated line must therefore be present. Reading
|
client := &mockDockerClient{}
|
||||||
// lines here is race-free because wait() synchronizes on attachDone, which
|
client.On("ContainerInspect", mock.Anything, "123", mobyclient.ContainerInspectOptions{}).
|
||||||
// the goroutine closes after the final append.
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{}}}, nil)
|
||||||
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
client.On("ContainerAttach", mock.Anything, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
|
||||||
|
Return(mobyclient.ContainerAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
|
||||||
|
client.On("ContainerStart", mock.Anything, "123", mobyclient.ContainerStartOptions{}).
|
||||||
|
Return(mobyclient.ContainerStartResult{}, errors.New("container start failed"))
|
||||||
|
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{}}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
require.ErrorContains(t, cr.Start(true)(ctx), "container start failed")
|
||||||
|
cancel()
|
||||||
|
<-cr.attachDone
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerWaitFailure(t *testing.T) {
|
func TestDockerWaitFailure(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
|
||||||
statusCh := make(chan container.WaitResponse, 1)
|
statusCh := make(chan container.WaitResponse, 1)
|
||||||
statusCh <- container.WaitResponse{StatusCode: 2}
|
statusCh <- container.WaitResponse{StatusCode: 2}
|
||||||
errCh := make(chan error, 1)
|
|
||||||
|
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||||
Return(mobyclient.ContainerWaitResult{
|
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||||
Result: (<-chan container.WaitResponse)(statusCh),
|
cr := &containerReference{id: "123", cli: client}
|
||||||
Error: (<-chan error)(errCh),
|
|
||||||
})
|
|
||||||
|
|
||||||
cr := &containerReference{
|
|
||||||
id: "123",
|
|
||||||
cli: client,
|
|
||||||
input: &NewContainerInput{
|
|
||||||
Image: "image",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err := cr.wait()(ctx)
|
err := cr.wait()(ctx)
|
||||||
var exitErr ExitCodeError
|
var exitErr ExitCodeError
|
||||||
require.ErrorAs(t, err, &exitErr)
|
require.ErrorAs(t, err, &exitErr)
|
||||||
assert.Equal(t, ExitCodeError(2), exitErr)
|
assert.Equal(t, ExitCodeError(2), exitErr)
|
||||||
assert.Equal(t, "Process completed with exit code 2.", err.Error())
|
assert.Equal(t, "Process completed with exit code 2.", err.Error())
|
||||||
|
statusCh <- container.WaitResponse{Error: &container.WaitExitError{Message: "container wait failed"}}
|
||||||
|
require.EqualError(t, cr.wait()(ctx), "failed to wait for container: container wait failed")
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -368,8 +381,8 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
|||||||
{name: "real failure", err: removeFailure, wantErr: removeFailure},
|
{name: "real failure", err: removeFailure, wantErr: removeFailure},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
logger, hook := test.NewNullLogger()
|
logger, hook := logtest.NewLogger()
|
||||||
ctx := common.WithLogger(context.Background(), logger)
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
|
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
|
||||||
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
|
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
|
||||||
@@ -392,7 +405,7 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
|||||||
} else {
|
} else {
|
||||||
assert.Empty(t, cr.id)
|
assert.Empty(t, cr.id)
|
||||||
}
|
}
|
||||||
assert.Empty(t, hook.AllEntries())
|
assert.Empty(t, hook.AllRecords())
|
||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -402,7 +415,7 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
|||||||
// create() lost its reply, must still be removed rather than leaking with its network. It
|
// create() lost its reply, must still be removed rather than leaking with its network. It
|
||||||
// was never started here, so it is not worth a kill of its own.
|
// was never started here, so it is not worth a kill of its own.
|
||||||
func TestRemoveWithoutIDUsesName(t *testing.T) {
|
func TestRemoveWithoutIDUsesName(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
|
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
|
||||||
Return(mobyclient.ContainerRemoveResult{}, nil)
|
Return(mobyclient.ContainerRemoveResult{}, nil)
|
||||||
@@ -412,70 +425,88 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
|
|||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
// find() must drop a stale cached id so later Copy/Exec don't hit the
|
func TestRemovePreservesClientCreationError(t *testing.T) {
|
||||||
// daemon with a torn-down container.
|
t.Setenv("DOCKER_HOST", "://")
|
||||||
|
|
||||||
|
require.ErrorContains(t, (&containerReference{input: &NewContainerInput{Name: "job-1"}}).
|
||||||
|
Remove()(t.Context()), "failed to connect to docker daemon")
|
||||||
|
}
|
||||||
|
|
||||||
func TestFindRevalidatesStaleID(t *testing.T) {
|
func TestFindRevalidatesStaleID(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
|
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||||
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
|
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
|
||||||
boom := errors.New("daemon unreachable")
|
boom := errors.New("daemon unreachable")
|
||||||
newCR := func(id string) (*containerReference, *mockDockerClient) {
|
for _, tc := range []struct {
|
||||||
client := &mockDockerClient{}
|
name, cachedID string
|
||||||
return &containerReference{id: id, cli: client, input: &NewContainerInput{Name: "job-1"}}, client
|
cachedErr error
|
||||||
|
lookup container.InspectResponse
|
||||||
|
lookupErr error
|
||||||
|
listed []container.Summary
|
||||||
|
wantID string
|
||||||
|
retryList bool
|
||||||
|
}{
|
||||||
|
{name: "stale id cleared, name lookup empty", cachedID: "stale", cachedErr: notFound, lookupErr: notFound},
|
||||||
|
{name: "stale id cleared, name lookup repopulates", cachedID: "stale", cachedErr: notFound, lookup: container.InspectResponse{ID: "fresh", Name: "/job-1"}, wantID: "fresh"},
|
||||||
|
{name: "id match falls back to listing", lookup: container.InspectResponse{ID: "job-1abc", Name: "/somebody-else"}, listed: []container.Summary{{ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh"},
|
||||||
|
{name: "live id kept", cachedID: "live", wantID: "live"},
|
||||||
|
{name: "transient inspect error trusts cache", cachedID: "live", cachedErr: boom, wantID: "live"},
|
||||||
|
{name: "name lookup error falls back to listing", lookupErr: boom, listed: []container.Summary{{ID: "other", Names: []string{"/somebody-else"}}, {ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh", retryList: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
cr := &containerReference{id: tc.cachedID, cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||||
|
if tc.cachedID != "" {
|
||||||
|
client.On("ContainerInspect", ctx, tc.cachedID, inspectOpts).Return(mobyclient.ContainerInspectResult{}, tc.cachedErr)
|
||||||
|
}
|
||||||
|
if tc.cachedID == "" || cerrdefs.IsNotFound(tc.cachedErr) {
|
||||||
|
client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{Container: tc.lookup}, tc.lookupErr)
|
||||||
|
}
|
||||||
|
if tc.listed != nil {
|
||||||
|
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: tc.listed}, nil).Once()
|
||||||
|
}
|
||||||
|
require.NoError(t, cr.find()(ctx))
|
||||||
|
assert.Equal(t, tc.wantID, cr.id)
|
||||||
|
if tc.retryList {
|
||||||
|
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, boom)
|
||||||
|
cr.id = ""
|
||||||
|
require.ErrorIs(t, cr.find()(ctx), boom)
|
||||||
|
}
|
||||||
|
client.AssertExpectations(t)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
listOpts := mobyclient.ContainerListOptions{All: true}
|
}
|
||||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
|
||||||
|
|
||||||
t.Run("stale id cleared, name lookup empty", func(t *testing.T) {
|
func TestRunnerArchCachedPerRun(t *testing.T) {
|
||||||
cr, client := newCR("stale")
|
var infos atomic.Int32
|
||||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, nil)
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
require.NoError(t, cr.find()(ctx))
|
writer.Header().Set("API-Version", "1.47")
|
||||||
assert.Empty(t, cr.id)
|
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||||
client.AssertExpectations(t)
|
infos.Add(1)
|
||||||
})
|
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||||
|
}
|
||||||
t.Run("stale id cleared, name lookup repopulates", func(t *testing.T) {
|
}))
|
||||||
cr, client := newCR("stale")
|
t.Cleanup(daemon.Close)
|
||||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
extensions := &LinuxContainerEnvironmentExtensions{}
|
||||||
{ID: "other", Names: []string{"/somebody-else"}},
|
run := WithRunnerArch(t.Context())
|
||||||
{ID: "fresh", Names: []string{"/job-1"}},
|
var lookups sync.WaitGroup
|
||||||
}}, nil)
|
for range 4 {
|
||||||
require.NoError(t, cr.find()(ctx))
|
lookups.Go(func() { assert.Equal(t, "ARM64", extensions.GetRunnerContext(run)["arch"]) })
|
||||||
assert.Equal(t, "fresh", cr.id)
|
}
|
||||||
client.AssertExpectations(t)
|
lookups.Wait()
|
||||||
})
|
assert.Equal(t, "ARM64", extensions.GetRunnerContext(t.Context())["arch"])
|
||||||
|
assert.EqualValues(t, 2, infos.Load())
|
||||||
t.Run("live id kept", func(t *testing.T) {
|
|
||||||
cr, client := newCR("live")
|
|
||||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, nil)
|
|
||||||
require.NoError(t, cr.find()(ctx))
|
|
||||||
assert.Equal(t, "live", cr.id)
|
|
||||||
client.AssertExpectations(t)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("transient inspect error trusts cache", func(t *testing.T) {
|
|
||||||
cr, client := newCR("live")
|
|
||||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, boom)
|
|
||||||
require.NoError(t, cr.find()(ctx))
|
|
||||||
assert.Equal(t, "live", cr.id)
|
|
||||||
client.AssertExpectations(t)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("list error propagates", func(t *testing.T) {
|
|
||||||
cr, client := newCR("")
|
|
||||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, boom)
|
|
||||||
require.ErrorIs(t, cr.find()(ctx), boom)
|
|
||||||
client.AssertExpectations(t)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Every daemon entry point fails fast with a clear, container-named
|
// Every daemon entry point fails fast with a clear, container-named
|
||||||
// error when no live cr.id is known.
|
// error when no live cr.id is known.
|
||||||
func TestRejectsMissingContainer(t *testing.T) {
|
func TestRejectsMissingContainer(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
client := &mockDockerClient{}
|
client := &mockDockerClient{}
|
||||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
|
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
|
||||||
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
|
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||||
check := func(op string, err error) {
|
check := func(op string, err error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -498,30 +529,47 @@ func TestRejectsMissingContainer(t *testing.T) {
|
|||||||
check("Inspect after removal", err)
|
check("Inspect after removal", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// End-to-end: a stale cr.id is cleared, repopulated from name lookup,
|
|
||||||
// and the Copy completes against the fresh id.
|
|
||||||
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||||
ctx := context.Background()
|
for _, testcase := range []struct {
|
||||||
client := &mockDockerClient{}
|
name, destination, extractionRoot, entry string
|
||||||
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
directory bool
|
||||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
}{
|
||||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
|
{name: "files", destination: "/var/run/act", extractionRoot: "/var/run/act", entry: "x"},
|
||||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
{name: "action below act volume", directory: true, destination: "/var/run/act/actions/checkout/", extractionRoot: "/var/run/act", entry: "actions/checkout/x"},
|
||||||
{ID: "fresh", Names: []string{"/job-1"}},
|
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
|
||||||
}}, nil)
|
} {
|
||||||
client.On("CopyToContainer", ctx, "fresh", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
return opts.DestinationPath == "/var/run/act"
|
hook := logtest.NewWithLevel(logging.LevelError)
|
||||||
})).Return(mobyclient.CopyToContainerResult{}, nil)
|
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
||||||
|
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||||
|
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
|
||||||
|
client.On("CopyToContainer", ctx, "fresh", mock.Anything).
|
||||||
|
Run(func(args mock.Arguments) {
|
||||||
|
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
|
||||||
|
assert.Equal(t, testcase.extractionRoot, opts.DestinationPath)
|
||||||
|
header, err := tar.NewReader(opts.Content).Next()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, testcase.entry, header.Name)
|
||||||
|
}).Return(mobyclient.CopyToContainerResult{}, nil).Once()
|
||||||
|
|
||||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||||
require.NoError(t, cr.Copy("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
if testcase.directory {
|
||||||
assert.Equal(t, "fresh", cr.id)
|
source := t.TempDir()
|
||||||
client.AssertExpectations(t)
|
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||||
|
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
||||||
|
} else {
|
||||||
|
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||||
|
}
|
||||||
|
assert.Equal(t, "fresh", cr.id)
|
||||||
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
client.AssertExpectations(t)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type assert containerReference implements ExecutionsEnvironment
|
|
||||||
var _ ExecutionsEnvironment = &containerReference{}
|
|
||||||
|
|
||||||
func TestCheckVolumes(t *testing.T) {
|
func TestCheckVolumes(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
desc string
|
desc string
|
||||||
@@ -609,17 +657,18 @@ func TestCheckVolumes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.desc, func(t *testing.T) {
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
logger, hook := test.NewNullLogger()
|
logger, hook := logtest.NewLogger()
|
||||||
ctx := common.WithLogger(context.Background(), logger)
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
cr := &containerReference{
|
cr := &containerReference{
|
||||||
input: &NewContainerInput{
|
input: &NewContainerInput{
|
||||||
ValidVolumes: tc.validVolumes,
|
ValidVolumes: tc.validVolumes,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts})
|
hostConf := &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts}
|
||||||
|
cr.sanitizeConfig(ctx, hostConf)
|
||||||
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
||||||
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
||||||
assert.Len(t, hook.AllEntries(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts)) // every drop is warned about
|
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -639,41 +688,35 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
|
|||||||
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
|
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Both of these are read here, on the runner, so a workflow could read the runner's files
|
|
||||||
// and environment with them.
|
|
||||||
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||||
hostFile := filepath.Join(t.TempDir(), "host.env")
|
hostFile := filepath.Join(t.TempDir(), "host.env")
|
||||||
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
||||||
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
||||||
|
|
||||||
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
||||||
|
|
||||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
require.ErrorContains(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}), "not allowed in a workflow")
|
||||||
require.ErrorContains(t, err, "not allowed in a workflow")
|
|
||||||
|
|
||||||
// the runner reading its own files is what those options are for
|
|
||||||
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
|
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
|
||||||
_, _, err = cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// a bare name is no longer resolved from the runner's environment, for either source
|
logger := logging.New(logtest.New())
|
||||||
logger, _ := test.NewNullLogger()
|
|
||||||
cr := &containerReference{input: &NewContainerInput{
|
cr := &containerReference{input: &NewContainerInput{
|
||||||
NetworkMode: "bridge",
|
NetworkMode: "bridge",
|
||||||
RunnerOptions: "--env RUNNER_SECRET",
|
RunnerOptions: "--env RUNNER_SECRET",
|
||||||
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
|
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
config, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
config := &container.Config{}
|
||||||
require.NoError(t, err)
|
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), config, &container.HostConfig{}))
|
||||||
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
|
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSanitizeOptionsHostConfig(t *testing.T) {
|
func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
|
|
||||||
// every field the sanitizer resets, so a reset dropped in a refactor fails here
|
// every field the sanitizer resets, so a reset dropped in a refactor fails here
|
||||||
hostConfig := &container.HostConfig{
|
hostConfig := &container.HostConfig{
|
||||||
@@ -705,10 +748,9 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
|||||||
assert.Equal(t, &container.HostConfig{}, hostConfig)
|
assert.Equal(t, &container.HostConfig{}, hostConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeOptions merges both option sources into a bare container, returning the result and its log.
|
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *logtest.Handler) {
|
||||||
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *test.Hook) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
logger, hook := test.NewNullLogger()
|
logger, hook := logtest.NewLogger()
|
||||||
cr := &containerReference{input: &NewContainerInput{
|
cr := &containerReference{input: &NewContainerInput{
|
||||||
RunnerOptions: runnerOptions,
|
RunnerOptions: runnerOptions,
|
||||||
WorkflowOptions: workflowOptions,
|
WorkflowOptions: workflowOptions,
|
||||||
@@ -717,12 +759,12 @@ func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privilege
|
|||||||
ValidVolumes: []string{"**"},
|
ValidVolumes: []string{"**"},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{
|
hostConfig := &container.HostConfig{
|
||||||
Privileged: privileged,
|
Privileged: privileged,
|
||||||
UsernsMode: container.UsernsMode("private"),
|
UsernsMode: container.UsernsMode("private"),
|
||||||
NetworkMode: container.NetworkMode("bridge"),
|
NetworkMode: container.NetworkMode("bridge"),
|
||||||
})
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, hostConfig))
|
||||||
return hostConfig, hook
|
return hostConfig, hook
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -760,8 +802,8 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
ctx := common.WithLogger(context.Background(), logger)
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
|
|
||||||
base := t.TempDir()
|
base := t.TempDir()
|
||||||
allowed := filepath.Join(base, "allowed")
|
allowed := filepath.Join(base, "allowed")
|
||||||
@@ -776,23 +818,26 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
escapingPath := allowed + string(filepath.Separator) + ".." + string(filepath.Separator) + "denied"
|
escapingPath := allowed + string(filepath.Separator) + ".." + string(filepath.Separator) + "denied"
|
||||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
hostConf := &container.HostConfig{
|
||||||
Binds: []string{escapingPath + ":/mnt"},
|
Binds: []string{escapingPath + ":/mnt"},
|
||||||
})
|
}
|
||||||
|
cr.sanitizeConfig(ctx, hostConf)
|
||||||
assert.Empty(t, hostConf.Binds)
|
assert.Empty(t, hostConf.Binds)
|
||||||
|
|
||||||
linkPath := filepath.Join(allowed, "link")
|
linkPath := filepath.Join(allowed, "link")
|
||||||
if err := os.Symlink(denied, linkPath); err != nil {
|
if err := os.Symlink(denied, linkPath); err != nil {
|
||||||
t.Skipf("cannot create symlink: %v", err)
|
t.Skipf("cannot create symlink: %v", err)
|
||||||
}
|
}
|
||||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
hostConf = &container.HostConfig{
|
||||||
Binds: []string{linkPath + ":/mnt"},
|
Binds: []string{linkPath + ":/mnt"},
|
||||||
})
|
}
|
||||||
|
cr.sanitizeConfig(ctx, hostConf)
|
||||||
assert.Empty(t, hostConf.Binds)
|
assert.Empty(t, hostConf.Binds)
|
||||||
|
|
||||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
hostConf = &container.HostConfig{
|
||||||
Binds: []string{filepath.Join(linkPath, "missing") + ":/mnt"},
|
Binds: []string{filepath.Join(linkPath, "missing") + ":/mnt"},
|
||||||
})
|
}
|
||||||
|
cr.sanitizeConfig(ctx, hostConf)
|
||||||
assert.Empty(t, hostConf.Binds)
|
assert.Empty(t, hostConf.Binds)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -801,14 +846,15 @@ func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
|
|||||||
require.Len(t, hostConfig.Mounts, 1)
|
require.Len(t, hostConfig.Mounts, 1)
|
||||||
|
|
||||||
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
||||||
logger, hook := test.NewNullLogger()
|
logger, hook := logtest.NewLogger()
|
||||||
_, sanitized := (&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{Mounts: []mount.Mount{
|
sanitized := &container.HostConfig{Mounts: []mount.Mount{
|
||||||
hostConfig.Mounts[0],
|
hostConfig.Mounts[0],
|
||||||
{Type: mount.TypeVolume, Target: "/anonymous"},
|
{Type: mount.TypeVolume, Target: "/anonymous"},
|
||||||
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
||||||
}})
|
}}
|
||||||
|
(&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), sanitized)
|
||||||
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
||||||
assert.Len(t, hook.AllEntries(), 2)
|
assert.Len(t, hook.AllRecords(), 2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -891,8 +937,8 @@ func TestContainerInfoFromInspect(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
ctx := common.WithLogger(context.Background(), logger)
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
cr := &containerReference{
|
cr := &containerReference{
|
||||||
input: &NewContainerInput{
|
input: &NewContainerInput{
|
||||||
NetworkMode: "bridge",
|
NetworkMode: "bridge",
|
||||||
@@ -902,11 +948,11 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
|
hostConf := &container.HostConfig{
|
||||||
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
|
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
|
||||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
|
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
|
||||||
})
|
}
|
||||||
require.NoError(t, err)
|
require.NoError(t, cr.mergeContainerConfigs(ctx, &container.Config{}, hostConf))
|
||||||
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
|
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
|
||||||
assert.Empty(t, hostConf.Mounts)
|
assert.Empty(t, hostConf.Mounts)
|
||||||
}
|
}
|
||||||
@@ -914,7 +960,7 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
|||||||
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
|
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
|
||||||
warnings := func(runnerOptions, workflowOptions string) int {
|
warnings := func(runnerOptions, workflowOptions string) int {
|
||||||
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
|
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
|
||||||
return len(hook.AllEntries())
|
return len(hook.AllRecords())
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
|
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
|
||||||
@@ -923,14 +969,42 @@ func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
|
|||||||
assert.Equal(t, 1, warnings("", "--privileged"))
|
assert.Equal(t, 1, warnings("", "--privileged"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||||
|
for _, entrypoint := range [][]string{nil, {"sleep", "3600"}} {
|
||||||
|
for _, optionEntrypoint := range []string{"/sbin/dumb-init", ""} {
|
||||||
|
logger, hook := logtest.NewLogger()
|
||||||
|
cr := &containerReference{
|
||||||
|
input: &NewContainerInput{
|
||||||
|
NetworkMode: "bridge",
|
||||||
|
Entrypoint: entrypoint,
|
||||||
|
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||||
|
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||||
|
},
|
||||||
|
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||||
|
if len(entrypoint) > 0 {
|
||||||
|
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||||
|
} else {
|
||||||
|
assert.Equal(t, []string{optionEntrypoint}, opts.Config.Entrypoint)
|
||||||
|
}
|
||||||
|
require.NotNil(t, opts.HostConfig.Init)
|
||||||
|
assert.True(t, *opts.HostConfig.Init)
|
||||||
|
return mobyclient.ContainerCreateResult{ID: "created"}, nil
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
require.NoError(t, cr.create(nil, nil)(common.WithLogger(t.Context(), logger)))
|
||||||
|
assert.Equal(t, "created", cr.id)
|
||||||
|
assert.Equal(t, len(entrypoint) > 0, len(hook.AllRecords()) == 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
||||||
logger, _ := test.NewNullLogger()
|
logger := logging.New(logtest.New())
|
||||||
cr := &containerReference{input: &NewContainerInput{
|
cr := &containerReference{input: &NewContainerInput{
|
||||||
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
|
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []string{"redis", "redis-primary"}, cr.input.NetworkAliases)
|
assert.Equal(t, []string{"redis", "redis-primary"}, cr.input.NetworkAliases)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/docker/cli/cli/connhelper/ssh"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sshArgs(daemonURL string) ([]string, error) {
|
||||||
|
spec, err := ssh.ParseURL(daemonURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
remote := []string{"docker", "system", "dial-stdio"}
|
||||||
|
if strings.Trim(spec.Path, "/") != "" {
|
||||||
|
remote = []string{"docker", "--host=unix://" + spec.Path, "system", "dial-stdio"}
|
||||||
|
}
|
||||||
|
return spec.Command([]string{"-o", "ConnectTimeout=30", "-T"}, remote...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func dialCommand(name string, args ...string) (net.Conn, error) {
|
||||||
|
conn := &commandConn{cmd: exec.Command(name, args...)}
|
||||||
|
conn.cmd.SysProcAttr = sshSysProcAttr() // a session of its own like docker's, so a ProxyCommand never reads the runner's terminal: https://github.com/docker/cli/issues/1707
|
||||||
|
conn.cmd.Stderr = &conn.stderr
|
||||||
|
conn.cmd.WaitDelay = time.Second // a ProxyCommand child keeping stderr open would otherwise block Wait after ssh exits
|
||||||
|
conn.wait = sync.OnceValue(func() error {
|
||||||
|
if err := conn.cmd.Wait(); err != nil && !errors.Is(err, exec.ErrWaitDelay) {
|
||||||
|
return fmt.Errorf("%s: %w: %s", strings.Join(conn.cmd.Args, " "), err, strings.TrimSpace(conn.stderr.String()))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
var err error
|
||||||
|
if conn.stdin, err = conn.cmd.StdinPipe(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if conn.stdout, err = conn.cmd.StdoutPipe(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return conn, conn.cmd.Start()
|
||||||
|
}
|
||||||
|
|
||||||
|
type commandConn struct {
|
||||||
|
cmd *exec.Cmd
|
||||||
|
stdin io.WriteCloser
|
||||||
|
stdout io.ReadCloser
|
||||||
|
stderr cappedBuffer
|
||||||
|
wait func() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *commandConn) waitWithin(timeout time.Duration) error {
|
||||||
|
timer := time.AfterFunc(timeout, func() { _ = c.cmd.Process.Kill() })
|
||||||
|
defer timer.Stop()
|
||||||
|
return c.wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *commandConn) Read(p []byte) (int, error) {
|
||||||
|
n, err := c.stdout.Read(p)
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
if waitErr := c.waitWithin(10 * time.Second); waitErr != nil {
|
||||||
|
return n, waitErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *commandConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
|
||||||
|
|
||||||
|
// CloseWrite lets the daemon see the end of a hijacked stream, like a TCP half-close.
|
||||||
|
func (c *commandConn) CloseWrite() error { return c.stdin.Close() }
|
||||||
|
|
||||||
|
func (c *commandConn) Close() error {
|
||||||
|
_ = c.stdin.Close()
|
||||||
|
if c.cmd.Process.Signal(syscall.SIGTERM) != nil { // SIGTERM lets ssh take its ProxyCommand down, Windows only kills
|
||||||
|
_ = c.cmd.Process.Kill()
|
||||||
|
}
|
||||||
|
_ = c.waitWithin(3 * time.Second)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *commandConn) LocalAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
|
||||||
|
|
||||||
|
func (c *commandConn) RemoteAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
|
||||||
|
|
||||||
|
func (c *commandConn) SetDeadline(_deadline time.Time) error { return nil }
|
||||||
|
|
||||||
|
func (c *commandConn) SetReadDeadline(_deadline time.Time) error { return nil }
|
||||||
|
|
||||||
|
func (c *commandConn) SetWriteDeadline(_deadline time.Time) error { return nil }
|
||||||
|
|
||||||
|
type cappedBuffer struct{ bytes.Buffer }
|
||||||
|
|
||||||
|
func (b *cappedBuffer) Write(p []byte) (int, error) {
|
||||||
|
if b.Len() > 4096 {
|
||||||
|
b.Reset()
|
||||||
|
}
|
||||||
|
return b.Buffer.Write(p)
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func sshSysProcAttr() *syscall.SysProcAttr {
|
||||||
|
return &syscall.SysProcAttr{Setsid: true, Pdeathsig: syscall.SIGKILL}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os/exec"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSSHArgs(t *testing.T) {
|
||||||
|
args, err := sshArgs("ssh://me@host:2222/run/user/1000/my docker.sock")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []string{"-l", "me", "-p", "2222", "-o", "ConnectTimeout=30", "-T", "--", "host", "docker '--host=unix:///run/user/1000/my docker.sock' system dial-stdio"}, args)
|
||||||
|
|
||||||
|
args, err = sshArgs("ssh://[::1]")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []string{"-o", "ConnectTimeout=30", "-T", "--", "::1", "docker system dial-stdio"}, args)
|
||||||
|
|
||||||
|
for _, invalid := range []string{"ssh://me:secret@host", "ssh://host?x=1", "ssh://host#x", "ssh://"} {
|
||||||
|
_, err := sshArgs(invalid)
|
||||||
|
assert.Error(t, err, invalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if _, err := exec.LookPath("sh"); err != nil {
|
||||||
|
t.Skip("needs a POSIX shell")
|
||||||
|
}
|
||||||
|
conn, err := dialCommand("sh", "-c", "cat")
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = conn.Write([]byte("ping"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, conn.(interface{ CloseWrite() error }).CloseWrite())
|
||||||
|
echoed, err := io.ReadAll(conn)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "ping", string(echoed))
|
||||||
|
require.NoError(t, conn.Close())
|
||||||
|
|
||||||
|
conn, err = dialCommand("sh", "-c", "exec 3<&0; cat <&3 >/dev/null & echo 'no route to host' >&2; exit 255")
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = io.ReadAll(conn)
|
||||||
|
require.ErrorContains(t, err, "no route to host")
|
||||||
|
require.NoError(t, conn.Close())
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func sshSysProcAttr() *syscall.SysProcAttr {
|
||||||
|
return &syscall.SysProcAttr{Setsid: true}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !WITHOUT_DOCKER
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func sshSysProcAttr() *syscall.SysProcAttr { return nil }
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
mobyclient "github.com/moby/moby/client"
|
mobyclient "github.com/moby/moby/client"
|
||||||
@@ -15,7 +14,7 @@ import (
|
|||||||
// so the daemon has to be pinged explicitly here to decide whether to skip.
|
// so the daemon has to be pinged explicitly here to decide whether to skip.
|
||||||
func requireDocker(t *testing.T) {
|
func requireDocker(t *testing.T) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("skipping: docker client unavailable: %v", err)
|
t.Skipf("skipping: docker client unavailable: %v", err)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package container
|
|||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -27,9 +28,6 @@ import (
|
|||||||
"gitea.com/gitea/runner/internal/pkg/process"
|
"gitea.com/gitea/runner/internal/pkg/process"
|
||||||
|
|
||||||
"github.com/creack/pty"
|
"github.com/creack/pty"
|
||||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
|
||||||
"github.com/go-git/go-billy/v5/osfs"
|
|
||||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
|
||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -67,15 +65,11 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
|
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.NewPipelineExecutor()
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Close() common.Executor {
|
func (e *HostEnvironment) Close() common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.NewPipelineExecutor()
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
|
func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||||
@@ -100,25 +94,16 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGi
|
|||||||
srcPrefix += string(filepath.Separator)
|
srcPrefix += string(filepath.Separator)
|
||||||
}
|
}
|
||||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||||
var ignorer gitignore.Matcher
|
|
||||||
if useGitIgnore {
|
|
||||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
|
||||||
if err != nil {
|
|
||||||
logger.Debugf("Error loading .gitignore: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ignorer = gitignore.NewMatcher(ps)
|
|
||||||
}
|
|
||||||
fc := &filecollector.FileCollector{
|
fc := &filecollector.FileCollector{
|
||||||
Ignorer: ignorer,
|
SrcPath: srcPath,
|
||||||
SrcPath: srcPath,
|
SrcPrefix: srcPrefix,
|
||||||
SrcPrefix: srcPrefix,
|
SkipGitDir: skipGitDir,
|
||||||
SkipGitDir: skipGitDir,
|
UseGitIgnore: useGitIgnore,
|
||||||
Handler: &filecollector.CopyCollector{
|
Handler: &filecollector.CopyCollector{
|
||||||
DstDir: destPath,
|
DstDir: destPath,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
return filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,7 +137,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
|||||||
SrcPrefix: srcPrefix,
|
SrcPrefix: srcPrefix,
|
||||||
Handler: tc,
|
Handler: tc,
|
||||||
}
|
}
|
||||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
err = filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -180,15 +165,11 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Pull(_ bool) common.Executor {
|
func (e *HostEnvironment) Pull(_ bool) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.NewPipelineExecutor()
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Start(_ bool) common.Executor {
|
func (e *HostEnvironment) Start(_ bool) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.NewPipelineExecutor()
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ptyWriter struct {
|
type ptyWriter struct {
|
||||||
@@ -205,7 +186,6 @@ func (w *ptyWriter) Write(buf []byte) (int, error) {
|
|||||||
}
|
}
|
||||||
if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
|
if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
|
||||||
_, _ = w.Out.Write([]byte("\n"))
|
_, _ = w.Out.Write([]byte("\n"))
|
||||||
return n, io.EOF
|
|
||||||
}
|
}
|
||||||
return n, io.EOF
|
return n, io.EOF
|
||||||
}
|
}
|
||||||
@@ -245,49 +225,38 @@ func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
|
|||||||
return ppty, tty, nil
|
return ppty, tty, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeKeepAlive(ppty io.Writer) {
|
func writeKeepAlive(ctx context.Context, ppty io.Writer) {
|
||||||
c := 1
|
for ctx.Err() == nil {
|
||||||
var err error
|
if count, err := ppty.Write([]byte{4}); count != 1 || err != nil {
|
||||||
for c == 1 && err == nil {
|
return
|
||||||
c, err = ppty.Write([]byte{4})
|
}
|
||||||
<-time.After(time.Second)
|
select {
|
||||||
}
|
case <-ctx.Done():
|
||||||
}
|
return
|
||||||
|
case <-time.After(time.Second):
|
||||||
func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) {
|
}
|
||||||
defer func() {
|
|
||||||
finishLog()
|
|
||||||
}()
|
|
||||||
if _, err := io.Copy(writer, ppty); err != nil {
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) UpdateFromImageEnv(_ *map[string]string) common.Executor {
|
func (e *HostEnvironment) UpdateFromImageEnv(_ *map[string]string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.NewPipelineExecutor()
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func getEnvListFromMap(env map[string]string) []string {
|
func getEnvListFromMap(env map[string]string) []string {
|
||||||
envList := make([]string, 0)
|
envList := make([]string, 0, len(env))
|
||||||
for k, v := range env {
|
for key, value := range env {
|
||||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
envList = append(envList, key+"="+value)
|
||||||
}
|
}
|
||||||
return envList
|
return envList
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error {
|
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, workdir string) error {
|
||||||
envList := getEnvListFromMap(env)
|
wd := e.Path
|
||||||
var wd string
|
|
||||||
if workdir != "" {
|
if workdir != "" {
|
||||||
if filepath.IsAbs(workdir) {
|
wd = workdir
|
||||||
wd = workdir
|
if !filepath.IsAbs(workdir) {
|
||||||
} else {
|
|
||||||
wd = filepath.Join(e.Path, workdir)
|
wd = filepath.Join(e.Path, workdir)
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
wd = e.Path
|
|
||||||
}
|
}
|
||||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||||
@@ -300,9 +269,8 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
|||||||
cmd := exec.CommandContext(ctx, f)
|
cmd := exec.CommandContext(ctx, f)
|
||||||
cmd.Path = f
|
cmd.Path = f
|
||||||
cmd.Args = command
|
cmd.Args = command
|
||||||
cmd.Stdin = nil
|
|
||||||
cmd.Stdout = e.StdOut
|
cmd.Stdout = e.StdOut
|
||||||
cmd.Env = envList
|
cmd.Env = getEnvListFromMap(env)
|
||||||
cmd.Stderr = e.StdOut
|
cmd.Stderr = e.StdOut
|
||||||
cmd.Dir = wd
|
cmd.Dir = wd
|
||||||
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
||||||
@@ -334,8 +302,11 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
|||||||
writer = &ptyWriter{Out: e.StdOut}
|
writer = &ptyWriter{Out: e.StdOut}
|
||||||
var finishLog context.CancelFunc
|
var finishLog context.CancelFunc
|
||||||
logctx, finishLog = context.WithCancel(context.Background())
|
logctx, finishLog = context.WithCancel(context.Background())
|
||||||
go copyPtyOutput(writer, ppty, finishLog)
|
go func() {
|
||||||
go writeKeepAlive(ppty)
|
defer finishLog()
|
||||||
|
_, _ = io.Copy(writer, ppty)
|
||||||
|
}()
|
||||||
|
go writeKeepAlive(logctx, ppty)
|
||||||
}
|
}
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -350,32 +321,32 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
|||||||
} else {
|
} else {
|
||||||
defer k.Close()
|
defer k.Close()
|
||||||
}
|
}
|
||||||
err = cmd.Wait()
|
err = treeKill.Wait(ctx, cmd)
|
||||||
if err != nil {
|
|
||||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
|
||||||
return ExitCodeError(exitErr.ExitCode())
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if tty != nil {
|
if tty != nil {
|
||||||
writer.AutoStop.Store(true)
|
writer.AutoStop.Store(true)
|
||||||
if _, err := tty.WriteString("\x04"); err != nil {
|
if _, err := tty.WriteString("\x04"); err != nil {
|
||||||
common.Logger(ctx).Debug("Failed to write EOT")
|
common.Logger(ctx).Debug("Failed to write EOT")
|
||||||
}
|
}
|
||||||
<-logctx.Done()
|
select {
|
||||||
|
case <-logctx.Done():
|
||||||
|
case <-ctx.Done(): // a cancelled step does not wait on output nobody consumes
|
||||||
|
}
|
||||||
ppty.Close()
|
ppty.Close()
|
||||||
ppty = nil
|
ppty = nil
|
||||||
}
|
}
|
||||||
|
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||||
|
return ExitCodeError(exitErr.ExitCode())
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor {
|
func (e *HostEnvironment) Exec(command []string, env map[string]string, _user, workdir string) common.Executor {
|
||||||
return e.ExecWithCmdLine(command, "", env, user, workdir)
|
return e.ExecWithCmdLine(command, "", env, workdir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor {
|
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, workdir string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil {
|
if err := e.exec(ctx, command, cmdline, env, workdir); err != nil {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return fmt.Errorf("this step has been cancelled: %w", err)
|
return fmt.Errorf("this step has been cancelled: %w", err)
|
||||||
@@ -388,7 +359,7 @@ func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(e, srcPath, env)
|
return parseEnvFile(e, srcPath, env, runtime.GOOS == "windows")
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeAll is a var so tests can substitute a blocking stub.
|
// removeAll is a var so tests can substitute a blocking stub.
|
||||||
@@ -617,8 +588,7 @@ func (*HostEnvironment) GetPathVariableName() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) DefaultPathVariable() string {
|
func (e *HostEnvironment) DefaultPathVariable() string {
|
||||||
v, _ := os.LookupEnv(e.GetPathVariableName())
|
return os.Getenv(e.GetPathVariableName())
|
||||||
return v
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
||||||
@@ -628,30 +598,22 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
|||||||
// Reference for Arch values for runner.arch
|
// Reference for Arch values for runner.arch
|
||||||
// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
|
// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
|
||||||
func goArchToActionArch(arch string) string {
|
func goArchToActionArch(arch string) string {
|
||||||
archMapper := map[string]string{
|
return cmp.Or(map[string]string{
|
||||||
"x86_64": "X64",
|
"x86_64": "X64",
|
||||||
"amd64": "X64",
|
"amd64": "X64",
|
||||||
"386": "X86",
|
"386": "X86",
|
||||||
"arm": "ARM",
|
"arm": "ARM",
|
||||||
"aarch64": "ARM64",
|
"aarch64": "ARM64",
|
||||||
"arm64": "ARM64",
|
"arm64": "ARM64",
|
||||||
}
|
}[arch], arch)
|
||||||
if arch, ok := archMapper[arch]; ok {
|
|
||||||
return arch
|
|
||||||
}
|
|
||||||
return arch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func goOsToActionOs(os string) string {
|
func goOsToActionOs(os string) string {
|
||||||
osMapper := map[string]string{
|
return cmp.Or(map[string]string{
|
||||||
"linux": "Linux",
|
"linux": "Linux",
|
||||||
"darwin": "macOS",
|
"darwin": "macOS",
|
||||||
"windows": "Windows",
|
"windows": "Windows",
|
||||||
}
|
}[os], os)
|
||||||
if os, ok := osMapper[os]; ok {
|
|
||||||
return os
|
|
||||||
}
|
|
||||||
return os
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any {
|
func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any {
|
||||||
|
|||||||
@@ -10,21 +10,20 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Type assert HostEnvironment implements ExecutionsEnvironment
|
|
||||||
var _ ExecutionsEnvironment = &HostEnvironment{}
|
var _ ExecutionsEnvironment = &HostEnvironment{}
|
||||||
|
|
||||||
func TestActionPlatformNames(t *testing.T) {
|
func TestActionPlatformNames(t *testing.T) {
|
||||||
@@ -33,38 +32,25 @@ func TestActionPlatformNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyDir(t *testing.T) {
|
func TestCopyDir(t *testing.T) {
|
||||||
dir := t.TempDir()
|
for _, test := range []struct {
|
||||||
ctx := context.Background()
|
name string
|
||||||
e := &HostEnvironment{
|
useGitIgnore, skipGitDir bool
|
||||||
Path: filepath.Join(dir, "path"),
|
}{
|
||||||
TmpDir: filepath.Join(dir, "tmp"),
|
{name: "git dir kept for a workspace copy"},
|
||||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
{name: "git dir skipped for an action copy", skipGitDir: true},
|
||||||
ActPath: filepath.Join(dir, "act_path"),
|
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
||||||
StdOut: os.Stdout,
|
} {
|
||||||
Workdir: path.Join("testdata", "scratch"),
|
t.Run(test.name, func(t *testing.T) {
|
||||||
}
|
|
||||||
_ = os.MkdirAll(e.Path, 0o700)
|
|
||||||
_ = os.MkdirAll(e.TmpDir, 0o700)
|
|
||||||
_ = os.MkdirAll(e.ToolCache, 0o700)
|
|
||||||
_ = os.MkdirAll(e.ActPath, 0o700)
|
|
||||||
err := e.CopyDir(e.Workdir, e.Path, true, false)(ctx)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyDirSkipGitDir(t *testing.T) {
|
|
||||||
for name, skipGitDir := range map[string]bool{"kept for a workspace copy": false, "skipped for an action copy": true} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
src := filepath.Join(t.TempDir(), "action")
|
src := filepath.Join(t.TempDir(), "action")
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||||
|
|
||||||
dest := filepath.Join(t.TempDir(), "dest")
|
dest := filepath.Join(t.TempDir(), "dest")
|
||||||
e := &HostEnvironment{StdOut: os.Stdout}
|
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
||||||
require.NoError(t, e.CopyDir(dest, src+string(filepath.Separator), false, skipGitDir)(context.Background()))
|
|
||||||
|
|
||||||
assert.FileExists(t, filepath.Join(dest, "action.yml"))
|
assert.FileExists(t, filepath.Join(dest, "action.yml"))
|
||||||
if skipGitDir {
|
if test.skipGitDir {
|
||||||
assert.NoDirExists(t, filepath.Join(dest, ".git"))
|
assert.NoDirExists(t, filepath.Join(dest, ".git"))
|
||||||
} else {
|
} else {
|
||||||
assert.FileExists(t, filepath.Join(dest, ".git", "objects", "pack"))
|
assert.FileExists(t, filepath.Join(dest, ".git", "objects", "pack"))
|
||||||
@@ -74,63 +60,68 @@ func TestCopyDirSkipGitDir(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGetContainerArchive(t *testing.T) {
|
func TestGetContainerArchive(t *testing.T) {
|
||||||
dir := t.TempDir()
|
ctx := t.Context()
|
||||||
ctx := context.Background()
|
e := &HostEnvironment{Path: t.TempDir()}
|
||||||
e := &HostEnvironment{
|
|
||||||
Path: filepath.Join(dir, "path"),
|
|
||||||
TmpDir: filepath.Join(dir, "tmp"),
|
|
||||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
|
||||||
ActPath: filepath.Join(dir, "act_path"),
|
|
||||||
StdOut: os.Stdout,
|
|
||||||
Workdir: path.Join("testdata", "scratch"),
|
|
||||||
}
|
|
||||||
_ = os.MkdirAll(e.Path, 0o700)
|
|
||||||
_ = os.MkdirAll(e.TmpDir, 0o700)
|
|
||||||
_ = os.MkdirAll(e.ToolCache, 0o700)
|
|
||||||
_ = os.MkdirAll(e.ActPath, 0o700)
|
|
||||||
expectedContent := []byte("sdde/7sh")
|
expectedContent := []byte("sdde/7sh")
|
||||||
err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600)
|
err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600)
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
archive, err := e.GetContainerArchive(ctx, e.Path)
|
archive, err := e.GetContainerArchive(ctx, e.Path)
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
defer archive.Close()
|
defer archive.Close()
|
||||||
reader := tar.NewReader(archive)
|
reader := tar.NewReader(archive)
|
||||||
h, err := reader.Next()
|
h, err := reader.Next()
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "action.yml", h.Name)
|
assert.Equal(t, "action.yml", h.Name)
|
||||||
content, err := io.ReadAll(reader)
|
content, err := io.ReadAll(reader)
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(t, expectedContent, content)
|
assert.Equal(t, expectedContent, content)
|
||||||
_, err = reader.Next()
|
_, err = reader.Next()
|
||||||
assert.ErrorIs(t, err, io.EOF)
|
assert.ErrorIs(t, err, io.EOF)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
dir := t.TempDir()
|
for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
|
||||||
ctx := context.Background()
|
t.Run(name, func(t *testing.T) {
|
||||||
e := &HostEnvironment{
|
t.Parallel()
|
||||||
Path: filepath.Join(dir, "path"),
|
synctest.Test(t, func(t *testing.T) {
|
||||||
TmpDir: filepath.Join(dir, "tmp"),
|
reader, writer := io.Pipe()
|
||||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
defer reader.Close()
|
||||||
ActPath: filepath.Join(dir, "act_path"),
|
e := &HostEnvironment{Path: t.TempDir(), StdOut: writer, AllocatePTY: allocatePTY}
|
||||||
StdOut: io.Discard,
|
t.Cleanup(func() { assert.NoError(t, e.Remove()(t.Context())) })
|
||||||
Workdir: filepath.Join(dir, "path"),
|
|
||||||
}
|
|
||||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
|
||||||
assert.NoError(t, os.MkdirAll(p, 0o700)) //nolint:testifylint // test setup
|
|
||||||
}
|
|
||||||
|
|
||||||
err := e.Exec([]string{"sh", "-c", "exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(ctx)
|
result := make(chan error, 1)
|
||||||
var exitErr ExitCodeError
|
go func() {
|
||||||
require.ErrorAs(t, err, &exitErr)
|
defer writer.Close()
|
||||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
result <- e.Exec([]string{"sh", "-c", "printf 'failed\\n'; exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(t.Context())
|
||||||
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
}()
|
||||||
|
first := make([]byte, 1)
|
||||||
|
_, err := io.ReadFull(reader, first)
|
||||||
|
require.NoError(t, err)
|
||||||
|
synctest.Wait()
|
||||||
|
select {
|
||||||
|
case err := <-result:
|
||||||
|
t.Fatalf("Exec returned before its output was consumed: %v", err)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
remaining, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = <-result
|
||||||
|
var exitErr ExitCodeError
|
||||||
|
require.ErrorAs(t, err, &exitErr)
|
||||||
|
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||||
|
assert.Equal(t, "Process completed with exit code 3.", err.Error())
|
||||||
|
assert.Equal(t, "failed\n", string(first)+string(remaining))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
@@ -143,25 +134,18 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
|||||||
{name: "on", allocPTY: true, expect: "TTY"},
|
{name: "on", allocPTY: true, expect: "TTY"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
t.Parallel()
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
e := &HostEnvironment{
|
e := &HostEnvironment{
|
||||||
Path: filepath.Join(dir, "path"),
|
Path: t.TempDir(),
|
||||||
TmpDir: filepath.Join(dir, "tmp"),
|
|
||||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
|
||||||
ActPath: filepath.Join(dir, "act_path"),
|
|
||||||
StdOut: buf,
|
StdOut: buf,
|
||||||
Workdir: filepath.Join(dir, "path"),
|
|
||||||
AllocatePTY: tc.allocPTY,
|
AllocatePTY: tc.allocPTY,
|
||||||
}
|
}
|
||||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
|
||||||
require.NoError(t, os.MkdirAll(p, 0o700))
|
|
||||||
}
|
|
||||||
|
|
||||||
err := e.Exec(
|
err := e.Exec(
|
||||||
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
|
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
|
||||||
map[string]string{"PATH": os.Getenv("PATH")}, "", "",
|
map[string]string{"PATH": os.Getenv("PATH")}, "", "",
|
||||||
)(context.Background())
|
)(t.Context())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
got := strings.TrimSpace(strings.ReplaceAll(buf.String(), "\r", ""))
|
got := strings.TrimSpace(strings.ReplaceAll(buf.String(), "\r", ""))
|
||||||
assert.Equal(t, tc.expect, got)
|
assert.Equal(t, tc.expect, got)
|
||||||
@@ -169,162 +153,91 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
|
func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
||||||
logger := logrus.New()
|
for name, owned := range map[string]bool{"preserves caller workdir": false, "cleans owned workdir": true} {
|
||||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
t.Run(name, func(t *testing.T) {
|
||||||
base := t.TempDir()
|
miscRoot, workdir := t.TempDir(), t.TempDir()
|
||||||
miscRoot := filepath.Join(base, "misc")
|
e := &HostEnvironment{
|
||||||
path := filepath.Join(miscRoot, "hostexecutor")
|
Path: filepath.Join(miscRoot, "hostexecutor"),
|
||||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
Workdir: workdir,
|
||||||
workdir := filepath.Join(base, "workspace", "owner", "repo")
|
CleanWorkdir: owned,
|
||||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
CleanUp: func() { _ = os.RemoveAll(miscRoot) },
|
||||||
|
}
|
||||||
e := &HostEnvironment{
|
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||||
Path: path,
|
require.NoError(t, e.Remove()(t.Context()))
|
||||||
Workdir: workdir,
|
_, err := os.Stat(workdir)
|
||||||
CleanUp: func() {
|
if owned {
|
||||||
_ = os.RemoveAll(miscRoot)
|
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||||
},
|
} else {
|
||||||
StdOut: os.Stdout,
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
require.NoError(t, e.Remove()(ctx))
|
|
||||||
_, err := os.Stat(workdir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHostEnvironmentRemoveCleansWorkdirWhenOwned(t *testing.T) {
|
|
||||||
logger := logrus.New()
|
|
||||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
|
||||||
base := t.TempDir()
|
|
||||||
miscRoot := filepath.Join(base, "misc")
|
|
||||||
path := filepath.Join(miscRoot, "hostexecutor")
|
|
||||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
|
||||||
workdir := filepath.Join(base, "workspace", "123", "owner", "repo")
|
|
||||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
|
||||||
|
|
||||||
e := &HostEnvironment{
|
|
||||||
Path: path,
|
|
||||||
Workdir: workdir,
|
|
||||||
CleanWorkdir: true,
|
|
||||||
CleanUp: func() {
|
|
||||||
_ = os.RemoveAll(miscRoot)
|
|
||||||
},
|
|
||||||
StdOut: os.Stdout,
|
|
||||||
}
|
|
||||||
require.NoError(t, e.Remove()(ctx))
|
|
||||||
_, err := os.Stat(workdir)
|
|
||||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||||
release := make(chan struct{})
|
synctest.Test(t, func(t *testing.T) {
|
||||||
stubDone := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
stubDone := make(chan struct{})
|
||||||
|
|
||||||
orig := removeAll
|
orig := removeAll
|
||||||
removeAll = func(string) error {
|
removeAll = func(string) error {
|
||||||
defer close(stubDone)
|
defer close(stubDone)
|
||||||
<-release
|
<-release
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// removeAllWithContext intentionally leaks the delete goroutine on timeout,
|
t.Cleanup(func() {
|
||||||
// and that goroutine still references removeAll. Unblock it and wait for it
|
close(release)
|
||||||
// to return before restoring the var, so the restore can't race the read.
|
<-stubDone
|
||||||
t.Cleanup(func() {
|
removeAll = orig
|
||||||
close(release)
|
})
|
||||||
<-stubDone
|
|
||||||
removeAll = orig
|
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := removeAllWithContext(ctx, t.TempDir())
|
||||||
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
})
|
})
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := removeAllWithContext(ctx, t.TempDir())
|
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp guards against a stalled
|
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||||
// CleanUp callback (e.g. an os.RemoveAll blocked by an AV/EDR filter driver or
|
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
||||||
// an unresponsive mount) wedging the runner slot forever at "Cleaning up
|
t.Run(phase, func(t *testing.T) {
|
||||||
// container". Remove must time out the callback and complete job teardown.
|
synctest.Test(t, func(t *testing.T) {
|
||||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp(t *testing.T) {
|
origTimeout := hostCleanupTimeout
|
||||||
// Keep the suite fast: shrink the per-phase teardown timeout for this test.
|
hostCleanupTimeout = 100 * time.Millisecond
|
||||||
orig := hostCleanupTimeout
|
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||||
hostCleanupTimeout = 100 * time.Millisecond
|
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||||
t.Cleanup(func() { hostCleanupTimeout = orig })
|
block := func() {
|
||||||
|
defer close(stubDone)
|
||||||
logger := logrus.New()
|
<-release
|
||||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
}
|
||||||
base := t.TempDir()
|
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||||
path := filepath.Join(base, "misc", "hostexecutor")
|
if phase == "CleanUp callback" {
|
||||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
e.CleanUp = block
|
||||||
|
} else {
|
||||||
release := make(chan struct{})
|
origRemoveAll := removeAll
|
||||||
t.Cleanup(func() { close(release) }) // unblock the leaked goroutine at test end
|
removeAll = func(string) error {
|
||||||
|
block()
|
||||||
e := &HostEnvironment{
|
return nil
|
||||||
Path: path,
|
}
|
||||||
CleanUp: func() {
|
t.Cleanup(func() { removeAll = origRemoveAll })
|
||||||
<-release // simulate a delete syscall stuck indefinitely
|
}
|
||||||
},
|
t.Cleanup(func() {
|
||||||
StdOut: os.Stdout,
|
close(release)
|
||||||
}
|
<-stubDone
|
||||||
|
})
|
||||||
done := make(chan error, 1)
|
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||||
go func() { done <- e.Remove()(ctx) }()
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||||
select {
|
select {
|
||||||
case err := <-done:
|
case err := <-done:
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
case <-time.After(10 * time.Second):
|
case <-time.After(10 * time.Second):
|
||||||
t.Fatal("Remove() hung on a stuck CleanUp callback")
|
t.Fatalf("Remove() hung on a stuck %s", phase)
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
|
})
|
||||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval guards against a
|
|
||||||
// stalled os.RemoveAll on the misc/workspace paths (same AV/EDR wedge as
|
|
||||||
// #1023) wedging job completion after the CleanUp callback has already timed
|
|
||||||
// out or finished.
|
|
||||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval(t *testing.T) {
|
|
||||||
origTimeout := hostCleanupTimeout
|
|
||||||
hostCleanupTimeout = 100 * time.Millisecond
|
|
||||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
stubDone := make(chan struct{})
|
|
||||||
|
|
||||||
origRemoveAll := removeAll
|
|
||||||
removeAll = func(string) error {
|
|
||||||
defer close(stubDone)
|
|
||||||
<-release
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// The stuck delete goroutine outlives the timed-out Remove and still reads
|
|
||||||
// removeAll; unblock it and wait before restoring to avoid a restore/read race.
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(release)
|
|
||||||
<-stubDone
|
|
||||||
removeAll = origRemoveAll
|
|
||||||
})
|
|
||||||
|
|
||||||
logger := logrus.New()
|
|
||||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
|
||||||
base := t.TempDir()
|
|
||||||
path := filepath.Join(base, "misc", "hostexecutor")
|
|
||||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
|
||||||
|
|
||||||
e := &HostEnvironment{
|
|
||||||
Path: path,
|
|
||||||
StdOut: os.Stdout,
|
|
||||||
}
|
|
||||||
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- e.Remove()(ctx) }()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("Remove() hung on a stuck path removal")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,17 +245,11 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
|||||||
t.Run("single dir", func(t *testing.T) {
|
t.Run("single dir", func(t *testing.T) {
|
||||||
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
||||||
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
||||||
// Self-PID guard is essential — without it the script could taskkill
|
|
||||||
// the PowerShell process running it.
|
|
||||||
assert.Contains(t, s, "$selfPid = $PID")
|
assert.Contains(t, s, "$selfPid = $PID")
|
||||||
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
||||||
// Must match both ExecutablePath (binaries from the workspace) and
|
|
||||||
// CommandLine (system binaries invoked with workspace paths in args),
|
|
||||||
// both bounded by dir+separator so a name-prefix sibling is spared.
|
|
||||||
assert.Contains(t, s, `$prefix = $p + '\'`)
|
assert.Contains(t, s, `$prefix = $p + '\'`)
|
||||||
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
||||||
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
||||||
// Each matched PID must be tree-killed, not just stopped.
|
|
||||||
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -355,25 +262,16 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
|||||||
assert.Contains(t, s, `'C:\work\path'`)
|
assert.Contains(t, s, `'C:\work\path'`)
|
||||||
assert.Contains(t, s, `'C:\work\workdir'`)
|
assert.Contains(t, s, `'C:\work\workdir'`)
|
||||||
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
||||||
// Commas between entries — no trailing comma, no leading comma.
|
|
||||||
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("path with single quote is escaped", func(t *testing.T) {
|
t.Run("path with single quote is escaped", func(t *testing.T) {
|
||||||
// In PowerShell single-quoted strings the only special char is the
|
|
||||||
// quote itself, escaped by doubling. A workspace path that ever
|
|
||||||
// contained `'` would inject a command into the script otherwise.
|
|
||||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
||||||
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
||||||
// And it must NOT appear unescaped — otherwise the quote would
|
|
||||||
// terminate the literal early.
|
|
||||||
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
||||||
// A path containing [ ] ? * must be embedded verbatim and matched with
|
|
||||||
// ordinal String methods, not -like, otherwise the metacharacters would
|
|
||||||
// be interpreted as wildcards and the leftover process could escape.
|
|
||||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
||||||
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
||||||
assert.NotContains(t, s, "-like")
|
assert.NotContains(t, s, "-like")
|
||||||
@@ -383,7 +281,6 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
||||||
s := buildWindowsWorkspaceKillScript(nil)
|
s := buildWindowsWorkspaceKillScript(nil)
|
||||||
// Empty array literal — script runs, matches nothing, is a no-op.
|
|
||||||
assert.Contains(t, s, "$paths = @()")
|
assert.Contains(t, s, "$paths = @()")
|
||||||
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
||||||
})
|
})
|
||||||
|
|||||||
+143
-20
@@ -7,12 +7,15 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
"maps"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -23,6 +26,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
"github.com/kballard/go-shellquote"
|
"github.com/kballard/go-shellquote"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
@@ -36,6 +40,7 @@ type KubernetesOptions struct {
|
|||||||
MaxLifetime time.Duration
|
MaxLifetime time.Duration
|
||||||
ForcePull bool
|
ForcePull bool
|
||||||
RunnerUUID string
|
RunnerUUID string
|
||||||
|
Transfer *KubernetesTransfer
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubernetesPod runs a job container and its service containers as one pod.
|
// KubernetesPod runs a job container and its service containers as one pod.
|
||||||
@@ -46,6 +51,10 @@ type KubernetesPod struct {
|
|||||||
job *kubernetesContainer
|
job *kubernetesContainer
|
||||||
services []*kubernetesContainer
|
services []*kubernetesContainer
|
||||||
arch, path string
|
arch, path string
|
||||||
|
ips []net.IP
|
||||||
|
transfer *transferServer
|
||||||
|
transferImage string
|
||||||
|
transferHelper string // empty when copies go through the API server
|
||||||
}
|
}
|
||||||
|
|
||||||
type kubernetesContainer struct {
|
type kubernetesContainer struct {
|
||||||
@@ -55,6 +64,7 @@ type kubernetesContainer struct {
|
|||||||
name string
|
name string
|
||||||
input *NewContainerInput
|
input *NewContainerInput
|
||||||
probe map[string]any
|
probe map[string]any
|
||||||
|
gzip bool // the container can gunzip, so copies into it go compressed
|
||||||
}
|
}
|
||||||
|
|
||||||
type containerStatus struct {
|
type containerStatus struct {
|
||||||
@@ -74,6 +84,7 @@ type podStatus struct {
|
|||||||
Spec struct{ Containers []struct{} }
|
Spec struct{ Containers []struct{} }
|
||||||
Status struct {
|
Status struct {
|
||||||
Phase, Reason, Message string
|
Phase, Reason, Message string
|
||||||
|
PodIPs []struct{ IP string }
|
||||||
Conditions []struct{ Type, Status, Message string }
|
Conditions []struct{ Type, Status, Message string }
|
||||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||||
}
|
}
|
||||||
@@ -93,17 +104,29 @@ func (p *KubernetesPod) JobContainer(input *NewContainerInput) ExecutionsEnviron
|
|||||||
return p.job
|
return p.job
|
||||||
}
|
}
|
||||||
|
|
||||||
var invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
var (
|
||||||
|
invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
||||||
|
dnsSubdomain = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
|
||||||
|
)
|
||||||
|
|
||||||
// KubernetesServiceName returns the container name and hostname of service id, made a DNS label as both must be.
|
// kubernetesServiceName returns the container name of service id, made a DNS label as it must be.
|
||||||
func KubernetesServiceName(id string) string {
|
func kubernetesServiceName(id string) string {
|
||||||
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
|
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
|
||||||
return strings.TrimRight(name[:min(63, len(name))], "-")
|
return strings.TrimRight(name[:min(63, len(name))], "-")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServiceContainer returns a container of the pod the job reaches at hostname KubernetesServiceName(id).
|
// KubernetesServiceHostnames returns the hostnames of service id, its container name and, where hostAliases accept it as a DNS subdomain, id lowercased.
|
||||||
|
func KubernetesServiceHostnames(id string) []string {
|
||||||
|
name, host := kubernetesServiceName(id), strings.ToLower(id)
|
||||||
|
if host != name && len(host) <= 253 && dnsSubdomain.MatchString(host) { // https://github.com/kubernetes/apimachinery/blob/v0.35.1/pkg/util/validation/validation.go#L198-L207
|
||||||
|
return []string{name, host}
|
||||||
|
}
|
||||||
|
return []string{name}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServiceContainer returns a container of the pod the job reaches at KubernetesServiceHostnames(id).
|
||||||
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
|
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
|
||||||
service := &kubernetesContainer{pod: p, id: id, name: KubernetesServiceName(id), input: input}
|
service := &kubernetesContainer{pod: p, id: id, name: kubernetesServiceName(id), input: input}
|
||||||
p.services = append(p.services, service)
|
p.services = append(p.services, service)
|
||||||
return service
|
return service
|
||||||
}
|
}
|
||||||
@@ -117,21 +140,31 @@ func (p *KubernetesPod) create(ctx context.Context) error {
|
|||||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||||
}
|
}
|
||||||
metadata := map[string]any{"name": name, "labels": labels}
|
metadata := map[string]any{"name": name, "labels": labels}
|
||||||
|
var runnerImage string
|
||||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||||
var runner struct{ Metadata struct{ Name, UID string } }
|
var runner struct {
|
||||||
|
Metadata struct{ Name, UID string }
|
||||||
|
Spec struct{ Containers []struct{ Image string } }
|
||||||
|
}
|
||||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||||
|
if len(runner.Spec.Containers) == 1 {
|
||||||
|
runnerImage = runner.Spec.Containers[0].Image
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := p.setupTransfer(runnerImage); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
secretData := map[string]string{}
|
secretData := map[string]string{}
|
||||||
manifest, err := p.manifest(name, secretData, metadata)
|
manifest, err := p.manifest(name, secretData, metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, service := range p.services {
|
for _, service := range p.services {
|
||||||
if !strings.EqualFold(service.id, service.name) {
|
if !slices.ContainsFunc(KubernetesServiceHostnames(service.id), func(hostname string) bool { return strings.EqualFold(hostname, service.id) }) { // glibc and Go match /etc/hosts case-insensitively
|
||||||
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
|
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -165,7 +198,8 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
|||||||
serviceTemplate = templateContainers[index]
|
serviceTemplate = templateContainers[index]
|
||||||
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
|
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
|
||||||
}
|
}
|
||||||
var containers, hostnames, volumes []any
|
var containers, volumes []any
|
||||||
|
var hostnames []string
|
||||||
names := map[string]bool{}
|
names := map[string]bool{}
|
||||||
for _, container := range templateContainers {
|
for _, container := range templateContainers {
|
||||||
names[manifestField(container, "name")] = true
|
names[manifestField(container, "name")] = true
|
||||||
@@ -184,7 +218,7 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
|
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
|
||||||
hostnames = append(hostnames, c.name)
|
hostnames = append(hostnames, KubernetesServiceHostnames(c.id)...)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"workspace", "act"} {
|
for _, name := range []string{"workspace", "act"} {
|
||||||
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
|
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
|
||||||
@@ -192,6 +226,18 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||||
|
if p.transferHelper != "" {
|
||||||
|
transfer := map[string]any{
|
||||||
|
"name": "gitea-runner-transfer", "image": p.transferImage, "terminationMessagePolicy": "FallbackToLogsOnError",
|
||||||
|
"command": []string{"gitea-runner", KubernetesTransferCommand, "install", p.transferHelper},
|
||||||
|
"volumeMounts": []any{map[string]any{"name": "act", "mountPath": p.job.GetActPath()}},
|
||||||
|
"securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}},
|
||||||
|
}
|
||||||
|
if p.options.ForcePull {
|
||||||
|
transfer["imagePullPolicy"] = "Always"
|
||||||
|
}
|
||||||
|
spec["initContainers"] = []any{transfer}
|
||||||
|
}
|
||||||
if len(hostnames) > 0 {
|
if len(hostnames) > 0 {
|
||||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||||
}
|
}
|
||||||
@@ -328,7 +374,13 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if pod.Status.Phase == "Failed" {
|
if pod.Status.Phase == "Failed" {
|
||||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
reason := cmp.Or(pod.Status.Message, pod.Status.Reason)
|
||||||
|
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||||
|
if terminated := status.State.Terminated; reason == "" && terminated != nil && terminated.ExitCode != 0 {
|
||||||
|
reason = fmt.Sprintf("container %s exited with %d: %s", status.Name, terminated.ExitCode, strings.TrimSpace(terminated.Message))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pod %s failed: %s", p.name, reason)
|
||||||
}
|
}
|
||||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||||
@@ -341,6 +393,9 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if started {
|
if started {
|
||||||
|
for _, ip := range pod.Status.PodIPs {
|
||||||
|
p.ips = append(p.ips, net.ParseIP(ip.IP))
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
for _, condition := range pod.Status.Conditions {
|
for _, condition := range pod.Status.Conditions {
|
||||||
@@ -374,16 +429,44 @@ func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
|||||||
if err := c.pod.waitStarted(ctx); err != nil {
|
if err := c.pod.waitStarted(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var stdout bytes.Buffer
|
output, err := c.pod.probe(ctx, c.name)
|
||||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard); err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
machine, path, _ := strings.Cut(stdout.String(), "\n")
|
probed := strings.SplitN(output, "\n", 3)
|
||||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
for len(probed) < 3 {
|
||||||
|
probed = append(probed, "")
|
||||||
|
}
|
||||||
|
c.pod.arch, c.gzip, c.pod.path = goArchToActionArch(probed[0]), probed[1] == "gzip", probed[2]
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// probe retries exec while it fails short of the container, as a new node runs pods before its kubelet has a serving certificate.
|
||||||
|
func (p *KubernetesPod) probe(ctx context.Context, container string) (string, error) {
|
||||||
|
var failure string
|
||||||
|
deadline := time.Now().Add(time.Minute)
|
||||||
|
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s\n%s' "$(uname -m)" "$(command -v gzip >/dev/null && echo gzip)" "$PATH"`}, nil, &stdout, io.Discard)
|
||||||
|
var exitErr ExitCodeError
|
||||||
|
var apiErr *apiError
|
||||||
|
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
|
||||||
|
errors.As(err, &apiErr) && apiErr.code < http.StatusInternalServerError {
|
||||||
|
return stdout.String(), err
|
||||||
|
}
|
||||||
|
if err.Error() != failure {
|
||||||
|
failure = err.Error()
|
||||||
|
common.Logger(ctx).Infof("Waiting for exec into pod %s: %v", p.name, err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (*kubernetesContainer) Pull(bool) common.Executor {
|
func (*kubernetesContainer) Pull(bool) common.Executor {
|
||||||
return func(context.Context) error { return nil }
|
return func(context.Context) error { return nil }
|
||||||
}
|
}
|
||||||
@@ -414,26 +497,66 @@ func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, w
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
// compressCopyMin is the size from which Copy compresses and uses the transfer helper: below it that saves next to nothing on the wire and costs a process in the pod.
|
||||||
|
const compressCopyMin = 64 << 10
|
||||||
|
|
||||||
|
// extract streams the tar write produces into destPath, gzipped with compress, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||||
|
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, compress bool, write func(io.Writer) error) error {
|
||||||
|
if compress && c.pod.transferHelper != "" {
|
||||||
|
request, done := c.pod.transfer.offer(c.pod.ips, write)
|
||||||
|
stdin, _ := json.Marshal(request)
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{c.pod.transferHelper, KubernetesTransferCommand, "fetch", destPath}, bytes.NewReader(stdin), io.Discard, &stderr)
|
||||||
|
taken, writeErr := done()
|
||||||
|
if err == nil || ctx.Err() != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if taken || !c.pod.options.Transfer.Fallback {
|
||||||
|
return fmt.Errorf("copy to %s over the pod network: %w: %s", destPath, errors.Join(writeErr, err), strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
c.pod.transferHelper = "" // a pod that cannot reach the runner once will not later
|
||||||
|
logging.Warnf("Copying into pod %s over the pod network failed, copying through the API server instead: %v: %s", c.pod.name, err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
compress = compress && c.gzip
|
||||||
reader, writer := io.Pipe()
|
reader, writer := io.Pipe()
|
||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
go func() { writer.CloseWithError(write(writer)) }()
|
go func() {
|
||||||
|
if compress {
|
||||||
|
writer.CloseWithError(writeGzip(writer, write))
|
||||||
|
} else {
|
||||||
|
writer.CloseWithError(write(writer))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
script := `mkdir -p "$1" && tar -xf - -C "$1"`
|
||||||
|
if compress {
|
||||||
|
script = `mkdir -p "$1" && gzip -dc | tar -xf - -C "$1"` // gzip -dc rather than tar -z, which GNU tar runs gzip for anyway
|
||||||
|
}
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", script, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeGzip gzips what write produces into w, at a level fast enough to not slow copies down and nearly as small as the default.
|
||||||
|
func writeGzip(w io.Writer, write func(io.Writer) error) error {
|
||||||
|
compressed, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||||
|
return errors.Join(write(compressed), compressed.Close())
|
||||||
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
size := 0
|
||||||
|
for _, file := range files {
|
||||||
|
size += len(file.Body)
|
||||||
|
}
|
||||||
|
return c.extract(ctx, destPath, size >= compressCopyMin, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -462,7 +585,7 @@ func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath s
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||||
return parseEnvFile(c, srcPath, env)
|
return parseEnvFile(c, srcPath, env, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||||
|
|||||||
@@ -43,8 +43,9 @@ type podManifest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fakeCluster struct {
|
type fakeCluster struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
requests map[string][]string
|
requests map[string][]string
|
||||||
|
execFailures []http.HandlerFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeCluster) record(request, content string) {
|
func (f *fakeCluster) record(request, content string) {
|
||||||
@@ -53,7 +54,42 @@ func (f *fakeCluster) record(request, content string) {
|
|||||||
f.requests[request] = append(f.requests[request], content)
|
f.requests[request] = append(f.requests[request], content)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
func (f *fakeCluster) failExec(failures ...http.HandlerFunc) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.execFailures = failures
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCluster) nextExecFailure() http.HandlerFunc {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if len(f.execFailures) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
failure := f.execFailures[0]
|
||||||
|
f.execFailures = f.execFailures[1:]
|
||||||
|
return failure
|
||||||
|
}
|
||||||
|
|
||||||
|
// execFailure answers like the API server, code 0 for a failure reported after the websocket upgrade.
|
||||||
|
func execFailure(t *testing.T, code int, status string) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if code != 0 {
|
||||||
|
w.WriteHeader(code)
|
||||||
|
_, _ = io.WriteString(w, status)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() { _ = conn.CloseNow() }()
|
||||||
|
_ = conn.Write(r.Context(), websocket.MessageBinary, append([]byte{3}, status...))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fakeKubernetes(t *testing.T, podStatus string, env ...string) (string, *fakeCluster) {
|
||||||
cluster := &fakeCluster{requests: map[string][]string{}}
|
cluster := &fakeCluster{requests: map[string][]string{}}
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||||
@@ -70,7 +106,11 @@ func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
|||||||
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
|
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
|
||||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
case strings.HasSuffix(r.URL.Path, "/exec"):
|
||||||
cluster.record("exec", r.URL.RawQuery)
|
cluster.record("exec", r.URL.RawQuery)
|
||||||
serveLocalExec(t, w, r)
|
if failure := cluster.nextExecFailure(); failure != nil {
|
||||||
|
failure(w, r)
|
||||||
|
} else {
|
||||||
|
serveLocalExec(t, w, r, env)
|
||||||
|
}
|
||||||
case strings.HasSuffix(r.URL.Path, "/log"):
|
case strings.HasSuffix(r.URL.Path, "/log"):
|
||||||
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
||||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
|
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
|
||||||
@@ -92,7 +132,7 @@ users: [{name: test, user: {tokenFile: token}}]
|
|||||||
return kubeconfig, cluster
|
return kubeconfig, cluster
|
||||||
}
|
}
|
||||||
|
|
||||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request, env []string) {
|
||||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
@@ -114,7 +154,7 @@ func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
|||||||
stdin.Write(frame[1:])
|
stdin.Write(frame[1:])
|
||||||
}
|
}
|
||||||
cmd := exec.Command(command[0], command[1:]...)
|
cmd := exec.Command(command[0], command[1:]...)
|
||||||
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
|
cmd.Env = append(append(os.Environ(), "COPYFILE_DISABLE=1"), env...)
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
||||||
status := `{"status":"Success"}`
|
status := `{"status":"Success"}`
|
||||||
@@ -129,16 +169,31 @@ func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKubernetesServiceHostnames(t *testing.T) {
|
||||||
|
for id, hostnames := range map[string][]string{
|
||||||
|
"db": {"db"},
|
||||||
|
"My_DB": {"my-db"},
|
||||||
|
"devstoreaccount1.azurite.local": {"devstoreaccount1-azurite-local", "devstoreaccount1.azurite.local"},
|
||||||
|
"DB.Local": {"db-local", "db.local"},
|
||||||
|
strings.Repeat("a", 64): {strings.Repeat("a", 63), strings.Repeat("a", 64)},
|
||||||
|
strings.Repeat("a", 254): {strings.Repeat("a", 63)},
|
||||||
|
} {
|
||||||
|
assert.Equal(t, hostnames, KubernetesServiceHostnames(id), id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("uses POSIX shell")
|
t.Skip("uses POSIX shell")
|
||||||
}
|
}
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
|
||||||
{"name":"job","ready":true,"state":{"running":{}}},
|
|
||||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
|
|
||||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
path := workdir + ":" + os.Getenv("PATH")
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}},
|
||||||
|
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`, "PATH="+path)
|
||||||
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
||||||
{"spec": map[string]any{
|
{"spec": map[string]any{
|
||||||
"automountServiceAccountToken": true,
|
"automountServiceAccountToken": true,
|
||||||
@@ -199,8 +254,13 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
||||||
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
|
cluster.failExec(
|
||||||
|
execFailure(t, http.StatusInternalServerError, `{"message":"error dialing backend: remote error: tls: internal error"}`),
|
||||||
|
execFailure(t, 0, `{"status":"Failure","reason":"InternalError","message":"Internal error occurred: error sending request: remote error: tls: internal error"}`),
|
||||||
|
)
|
||||||
require.NoError(t, job.Start(false)(ctx))
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
|
assert.Len(t, cluster.requests["exec"], 3)
|
||||||
|
assert.True(t, pod.job.gzip)
|
||||||
|
|
||||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||||
@@ -217,7 +277,9 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.NotContains(t, query, "hunter2")
|
assert.NotContains(t, query, "hunter2")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||||
|
assert.NotContains(t, lastExec(), "gzip")
|
||||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
reader := tar.NewReader(archive)
|
reader := tar.NewReader(archive)
|
||||||
@@ -230,6 +292,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
require.NoError(t, archive.Close())
|
require.NoError(t, archive.Close())
|
||||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
large := strings.Repeat("a", compressCopyMin)
|
||||||
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/large", Mode: 0o644, Body: large})(ctx))
|
||||||
|
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||||
|
content, err = os.ReadFile(workdir + "/act/workflow/large")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, large, string(content))
|
||||||
|
|
||||||
actionDir := filepath.Join(t.TempDir(), "action")
|
actionDir := filepath.Join(t.TempDir(), "action")
|
||||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||||
@@ -238,11 +306,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "main()", string(script))
|
assert.Equal(t, "main()", string(script))
|
||||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
assert.Contains(t, lastExec(), "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||||
|
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||||
|
|
||||||
pathEnv := map[string]string{}
|
pathEnv := map[string]string{}
|
||||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
assert.Equal(t, path, pathEnv["PATH"])
|
||||||
|
|
||||||
info, err := service.Inspect(ctx)
|
info, err := service.Inspect(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -262,6 +331,38 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
|||||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKubernetesCopiesUncompressedWithoutGzip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("uses POSIX shell")
|
||||||
|
}
|
||||||
|
ctx := t.Context()
|
||||||
|
workdir, bin := t.TempDir(), t.TempDir()
|
||||||
|
for _, tool := range []string{"uname", "tar", "mkdir"} { // all the job container needs but gzip, as on a minimal image
|
||||||
|
found, err := exec.LookPath(tool)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Symlink(found, filepath.Join(bin, tool)))
|
||||||
|
}
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}}]}}`, "PATH="+bin)
|
||||||
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job := pod.JobContainer(&NewContainerInput{Image: "node", WorkingDir: workdir})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||||
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
|
assert.False(t, pod.job.gzip)
|
||||||
|
assert.Equal(t, bin, pod.path)
|
||||||
|
|
||||||
|
actionDir := filepath.Join(t.TempDir(), "action")
|
||||||
|
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||||
|
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||||
|
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "main()", string(script))
|
||||||
|
assert.NotContains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "gzip")
|
||||||
|
}
|
||||||
|
|
||||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||||
@@ -282,11 +383,33 @@ func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
|||||||
}
|
}
|
||||||
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
|
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
|
||||||
}
|
}
|
||||||
|
running, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[{"name":"job","state":{"running":{}}}]}}`)
|
||||||
|
for _, test := range []struct {
|
||||||
|
failure http.HandlerFunc
|
||||||
|
err string
|
||||||
|
}{
|
||||||
|
{execFailure(t, http.StatusForbidden, `{"message":"pods \"job\" is forbidden"}`), "is forbidden"},
|
||||||
|
{execFailure(t, 0, `{"status":"Failure","details":{"causes":[{"reason":"ExitCode","message":"126"}]}}`), ExitCodeError(126).Error()},
|
||||||
|
} {
|
||||||
|
cluster.failExec(test.failure)
|
||||||
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: running})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job := pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||||
|
require.ErrorContains(t, job.Start(false)(t.Context()), test.err)
|
||||||
|
}
|
||||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||||
|
failed, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Failed","initContainerStatuses":[
|
||||||
|
{"name":"gitea-runner-transfer","state":{"terminated":{"exitCode":1,"message":"exec format error\n"}}}]}}`)
|
||||||
|
pod, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: failed})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job = pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||||
|
require.ErrorContains(t, job.Start(false)(t.Context()), "container gitea-runner-transfer exited with 1: exec format error")
|
||||||
|
|
||||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"context"
|
||||||
|
"crypto/ecdsa"
|
||||||
|
"crypto/elliptic"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/big"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KubernetesTransfer has job pods fetch large copies from the runner over the pod network instead of through the API server.
|
||||||
|
type KubernetesTransfer struct {
|
||||||
|
Host string // empty uses the outbound IP
|
||||||
|
Port uint16 // 0 picks a free port
|
||||||
|
Image string // has gitea-runner on its PATH, for an init container to copy into each job pod
|
||||||
|
Fallback bool // copy through the API server when a job pod cannot reach the runner, else fail
|
||||||
|
}
|
||||||
|
|
||||||
|
// KubernetesTransferCommand is the hidden runner subcommand job pods run the transfer helper with.
|
||||||
|
const KubernetesTransferCommand = "kubernetes-transfer"
|
||||||
|
|
||||||
|
// transferRequest goes to the helper on the exec stream's stdin, keeping the token out of the command line and the audit log.
|
||||||
|
type transferRequest struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Pin string `json:"pin"` // hex SHA-256 of the runner's certificate
|
||||||
|
}
|
||||||
|
|
||||||
|
type transferServer struct {
|
||||||
|
base string
|
||||||
|
pin string
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]transferOffer
|
||||||
|
}
|
||||||
|
|
||||||
|
type transferOffer struct {
|
||||||
|
clients []net.IP
|
||||||
|
write func(io.Writer) error
|
||||||
|
result chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
transferServersMu sync.Mutex
|
||||||
|
transferServers = map[string]*transferServer{}
|
||||||
|
)
|
||||||
|
|
||||||
|
// startTransferServer returns the runner's server for these settings, started on first use.
|
||||||
|
func startTransferServer(host string, port uint16) (*transferServer, error) {
|
||||||
|
transferServersMu.Lock()
|
||||||
|
defer transferServersMu.Unlock()
|
||||||
|
configured := net.JoinHostPort(host, strconv.Itoa(int(port)))
|
||||||
|
if server, ok := transferServers[configured]; ok {
|
||||||
|
return server, nil
|
||||||
|
}
|
||||||
|
if host == "" {
|
||||||
|
ip := common.GetOutboundIP()
|
||||||
|
if ip == nil {
|
||||||
|
return nil, errors.New("no outbound IP for job pods to reach the runner at, set kubernetes.transfer.host")
|
||||||
|
}
|
||||||
|
host = ip.String()
|
||||||
|
}
|
||||||
|
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
template := &x509.Certificate{SerialNumber: big.NewInt(1)} // trusted by its pin alone, so no CA, names or validity
|
||||||
|
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port))))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listen for job pod transfers: %w", err)
|
||||||
|
}
|
||||||
|
pin := sha256.Sum256(certificate)
|
||||||
|
server := &transferServer{base: "https://" + listener.Addr().String() + "/", pin: hex.EncodeToString(pin[:]), pending: map[string]transferOffer{}}
|
||||||
|
httpServer := &http.Server{
|
||||||
|
Handler: server,
|
||||||
|
TLSConfig: &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{certificate}, PrivateKey: privateKey}}, MinVersion: tls.VersionTLS13},
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
IdleTimeout: 30 * time.Second,
|
||||||
|
}
|
||||||
|
go func() { _ = httpServer.ServeTLS(listener, "", "") }()
|
||||||
|
transferServers[configured] = server
|
||||||
|
return server, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// offer serves write once to one of clients until done, which reports whether a client took it and what write failed with.
|
||||||
|
func (s *transferServer) offer(clients []net.IP, write func(io.Writer) error) (request transferRequest, done func() (taken bool, err error)) {
|
||||||
|
token := rand.Text()
|
||||||
|
result := make(chan error, 1)
|
||||||
|
s.mu.Lock()
|
||||||
|
s.pending[token] = transferOffer{clients: clients, write: write, result: result}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return transferRequest{URL: s.base + token, Pin: s.pin}, func() (bool, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
_, pending := s.pending[token]
|
||||||
|
delete(s.pending, token)
|
||||||
|
s.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case err := <-result:
|
||||||
|
return true, err
|
||||||
|
default:
|
||||||
|
return !pending, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *transferServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
token := strings.TrimPrefix(r.URL.Path, "/")
|
||||||
|
s.mu.Lock()
|
||||||
|
offer, ok := s.pending[token]
|
||||||
|
ok = ok && r.Method == http.MethodGet && slices.ContainsFunc(offer.clients, net.ParseIP(host).Equal)
|
||||||
|
if ok {
|
||||||
|
delete(s.pending, token)
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Encoding", "gzip")
|
||||||
|
err := writeGzip(w, offer.write)
|
||||||
|
offer.result <- err
|
||||||
|
if err != nil {
|
||||||
|
panic(http.ErrAbortHandler) // a cut-off body fails the fetch, a short tar might not
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FetchKubernetesTransfer reads a transferRequest from request and extracts the tar it points to into dest.
|
||||||
|
func FetchKubernetesTransfer(ctx context.Context, request io.Reader, dest string) error {
|
||||||
|
var transfer transferRequest
|
||||||
|
if err := json.NewDecoder(request).Decode(&transfer); err != nil {
|
||||||
|
return fmt.Errorf("read transfer request: %w", err)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: &http.Transport{
|
||||||
|
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||||
|
TLSHandshakeTimeout: 10 * time.Second,
|
||||||
|
ResponseHeaderTimeout: time.Minute,
|
||||||
|
TLSClientConfig: &tls.Config{
|
||||||
|
MinVersion: tls.VersionTLS13,
|
||||||
|
InsecureSkipVerify: true, // VerifyConnection checks the pin instead
|
||||||
|
VerifyConnection: func(state tls.ConnectionState) error {
|
||||||
|
if sum := sha256.Sum256(state.PeerCertificates[0].Raw); hex.EncodeToString(sum[:]) != transfer.Pin {
|
||||||
|
return errors.New("runner certificate does not match the pin")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, transfer.URL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
response, err := client.Do(httpRequest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
return fmt.Errorf("fetch: %s", response.Status)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dest, 0o777); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", dest)
|
||||||
|
cmd.Stdin, cmd.Stdout, cmd.Stderr = response.Body, os.Stderr, os.Stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(io.Discard, response.Body) // tar stops at the first zero blocks, a cut-off body after them still fails
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// InstallKubernetesTransferHelper copies the running binary to target.
|
||||||
|
func InstallKubernetesTransferHelper(target string) error {
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source, err := os.Open(executable)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = os.Remove(target) // a file left in a reused act volume would keep its mode
|
||||||
|
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, source)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupTransfer turns on transfers for the pod, with the runner pod's image unless one is set.
|
||||||
|
func (p *KubernetesPod) setupTransfer(runnerImage string) error {
|
||||||
|
settings := p.options.Transfer
|
||||||
|
if settings == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
image := cmp.Or(settings.Image, runnerImage)
|
||||||
|
var err error
|
||||||
|
var server *transferServer
|
||||||
|
if image == "" {
|
||||||
|
err = errors.New("set kubernetes.transfer.image, the runner is no single-container pod in kubernetes.namespace to take it from")
|
||||||
|
} else {
|
||||||
|
server, err = startTransferServer(settings.Host, settings.Port)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if !settings.Fallback {
|
||||||
|
return fmt.Errorf("kubernetes.transfer: %w", err)
|
||||||
|
}
|
||||||
|
logging.Warnf("Job pod copies go through the API server: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.transfer, p.transferImage = server, image
|
||||||
|
p.transferHelper = path.Join(p.job.GetActPath(), "bin", "gitea-runner")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/tls"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
const transferHelperEnv = "GITEA_RUNNER_TEST_TRANSFER_HELPER"
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
if os.Getenv(transferHelperEnv) == "1" && len(os.Args) == 4 && os.Args[1] == KubernetesTransferCommand && os.Args[2] == "fetch" {
|
||||||
|
if err := FetchKubernetesTransfer(context.Background(), os.Stdin, os.Args[3]); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetch(t *testing.T, request transferRequest) error {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(request)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return FetchKubernetesTransfer(t.Context(), bytes.NewReader(body), t.TempDir())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTransferServer(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("extracts with tar")
|
||||||
|
}
|
||||||
|
server, err := startTransferServer("127.0.0.1", 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
again, err := startTransferServer("127.0.0.1", 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Same(t, server, again)
|
||||||
|
loopback := []net.IP{net.ParseIP("127.0.0.1")}
|
||||||
|
nothing := func(io.Writer) error { return nil }
|
||||||
|
|
||||||
|
t.Run("serves an offer gzipped once to an IP of its job pod", func(t *testing.T) {
|
||||||
|
request, done := server.offer([]net.IP{net.ParseIP("fd00::1"), loopback[0]}, nothing)
|
||||||
|
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableCompression: true}}
|
||||||
|
response, err := client.Get(request.URL)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, response.Body.Close())
|
||||||
|
assert.Equal(t, "gzip", response.Header.Get("Content-Encoding"))
|
||||||
|
require.ErrorContains(t, fetch(t, request), "404")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.True(t, taken)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("another pod can neither take nor block an offer", func(t *testing.T) {
|
||||||
|
request, done := server.offer([]net.IP{net.ParseIP("10.1.2.3")}, nothing)
|
||||||
|
require.ErrorContains(t, fetch(t, request), "404")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.False(t, taken)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a failed write fails the fetch even past the end blocks tar stops at, and done reports it", func(t *testing.T) {
|
||||||
|
request, done := server.offer(loopback, func(w io.Writer) error {
|
||||||
|
_, _ = w.Write(make([]byte, 1024))
|
||||||
|
_, _ = io.CopyN(w, rand.Reader, 1<<20)
|
||||||
|
return errors.New("source gone")
|
||||||
|
})
|
||||||
|
require.Error(t, fetch(t, request))
|
||||||
|
taken, err := done()
|
||||||
|
assert.True(t, taken)
|
||||||
|
require.EqualError(t, err, "source gone")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the helper refuses a runner whose certificate does not match its pin", func(t *testing.T) {
|
||||||
|
request, done := server.offer(loopback, nothing)
|
||||||
|
request.Pin = strings.Repeat("0", 64)
|
||||||
|
require.ErrorContains(t, fetch(t, request), "does not match the pin")
|
||||||
|
taken, _ := done()
|
||||||
|
assert.False(t, taken)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKubernetesPodTransferImage(t *testing.T) {
|
||||||
|
newPod := func(image string, fallback bool) *KubernetesPod {
|
||||||
|
pod := &KubernetesPod{options: KubernetesOptions{Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: image, Fallback: fallback}}}
|
||||||
|
pod.JobContainer(&NewContainerInput{})
|
||||||
|
return pod
|
||||||
|
}
|
||||||
|
|
||||||
|
pod := newPod("", false)
|
||||||
|
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||||
|
assert.Equal(t, "gitea/runner:latest", pod.transferImage)
|
||||||
|
|
||||||
|
pod = newPod("registry.example/runner:1", false)
|
||||||
|
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||||
|
assert.Equal(t, "registry.example/runner:1", pod.transferImage)
|
||||||
|
|
||||||
|
pod = newPod("", true)
|
||||||
|
require.NoError(t, pod.setupTransfer(""))
|
||||||
|
assert.Empty(t, pod.transferHelper)
|
||||||
|
|
||||||
|
pod = newPod("", false)
|
||||||
|
require.ErrorContains(t, pod.setupTransfer(""), "kubernetes.transfer.image")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKubernetesPodCopiesLargeFilesOverPodNetwork(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("uses POSIX shell")
|
||||||
|
}
|
||||||
|
ctx := t.Context()
|
||||||
|
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","podIPs":[{"ip":"127.0.0.1"}],"containerStatuses":[
|
||||||
|
{"name":"job","ready":true,"state":{"running":{}}}]}}`, transferHelperEnv+"=1")
|
||||||
|
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, ForcePull: true, Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: "gitea/runner:latest", Fallback: true}})
|
||||||
|
require.NoError(t, err)
|
||||||
|
job := pod.JobContainer(&NewContainerInput{Name: "job", Image: "node", WorkingDir: workdir, Stdout: io.Discard, Stderr: io.Discard})
|
||||||
|
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||||
|
|
||||||
|
var manifest struct {
|
||||||
|
Spec struct {
|
||||||
|
InitContainers []struct {
|
||||||
|
Image, ImagePullPolicy string
|
||||||
|
Command []string
|
||||||
|
VolumeMounts []struct{ Name, MountPath string }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||||
|
require.Len(t, manifest.Spec.InitContainers, 1)
|
||||||
|
initContainer := manifest.Spec.InitContainers[0]
|
||||||
|
assert.Equal(t, "gitea/runner:latest", initContainer.Image)
|
||||||
|
assert.Equal(t, "Always", initContainer.ImagePullPolicy)
|
||||||
|
assert.Equal(t, []string{"gitea-runner", KubernetesTransferCommand, "install", "/var/run/act/bin/gitea-runner"}, initContainer.Command)
|
||||||
|
assert.Equal(t, []struct{ Name, MountPath string }{{"act", "/var/run/act"}}, initContainer.VolumeMounts)
|
||||||
|
require.NoError(t, job.Start(false)(ctx))
|
||||||
|
|
||||||
|
helper := filepath.Join(t.TempDir(), "bin", "gitea-runner")
|
||||||
|
require.NoError(t, InstallKubernetesTransferHelper(helper))
|
||||||
|
pod.transferHelper = helper
|
||||||
|
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||||
|
large := &FileEntry{Name: "large", Mode: 0o644, Body: strings.Repeat("a", compressCopyMin)}
|
||||||
|
copyLarge := func(dir string) error { return job.Copy(filepath.Join(workdir, dir)+"/", large)(ctx) }
|
||||||
|
require.NoError(t, copyLarge("transferred"))
|
||||||
|
content, err := os.ReadFile(filepath.Join(workdir, "transferred", "large"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, large.Body, string(content))
|
||||||
|
assert.Contains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||||
|
assert.NotContains(t, lastExec(), "127.0.0.1")
|
||||||
|
|
||||||
|
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "event.json", Mode: 0o644, Body: `{}`})(ctx))
|
||||||
|
assert.NotContains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||||
|
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workdir, "file"), nil, 0o644))
|
||||||
|
require.ErrorContains(t, copyLarge("file"), "over the pod network")
|
||||||
|
assert.Equal(t, helper, pod.transferHelper)
|
||||||
|
|
||||||
|
unreachable := filepath.Join(workdir, "unreachable")
|
||||||
|
require.NoError(t, os.WriteFile(unreachable, []byte("#!/bin/sh\necho 'connection refused' >&2\nexit 1\n"), 0o755))
|
||||||
|
pod.transferHelper = unreachable
|
||||||
|
pod.options.Transfer.Fallback = false
|
||||||
|
require.ErrorContains(t, copyLarge("direct"), "connection refused")
|
||||||
|
assert.NoFileExists(t, filepath.Join(workdir, "direct", "large"))
|
||||||
|
|
||||||
|
pod.options.Transfer.Fallback = true
|
||||||
|
require.NoError(t, copyLarge("auto"))
|
||||||
|
assert.FileExists(t, filepath.Join(workdir, "auto", "large"))
|
||||||
|
assert.Empty(t, pod.transferHelper)
|
||||||
|
}
|
||||||
@@ -11,28 +11,28 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
type LinuxContainerEnvironmentExtensions struct{}
|
type LinuxContainerEnvironmentExtensions struct{}
|
||||||
|
|
||||||
|
var windowsPathRegex = regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
||||||
|
|
||||||
// Resolves the equivalent host path inside the container
|
// Resolves the equivalent host path inside the container
|
||||||
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
||||||
// For use in docker volumes and binds
|
// For use in docker volumes and binds
|
||||||
func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
|
func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
|
||||||
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
|
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
|
||||||
log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
|
logging.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
abspath, err := filepath.Abs(path)
|
abspath, err := filepath.Abs(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
logging.Error(err)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test if the path is a windows path
|
|
||||||
windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
|
||||||
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
|
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
|
||||||
|
|
||||||
// Return as-is if no match
|
// Return as-is if no match
|
||||||
@@ -43,11 +43,7 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
|
|||||||
// Convert to WSL2-compatible path if it is a windows path
|
// Convert to WSL2-compatible path if it is a windows path
|
||||||
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
|
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
|
||||||
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
|
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
|
||||||
driveLetter := strings.ToLower(windowsPathComponents[1])
|
return "/mnt/" + strings.ToLower(windowsPathComponents[1]) + "/" + strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
||||||
translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
|
||||||
// Should make something like /mnt/c/Users/person/My Folder/MyActProject
|
|
||||||
result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
|
|
||||||
return result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
||||||
@@ -69,10 +65,41 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
|
|||||||
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
|
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
|
||||||
const DefaultToolCache = "/opt/hostedtoolcache"
|
const DefaultToolCache = "/opt/hostedtoolcache"
|
||||||
|
|
||||||
|
type runnerArchKey struct{}
|
||||||
|
|
||||||
|
func WithRunnerArch(ctx context.Context) context.Context { // shares one architecture lookup per run
|
||||||
|
if ctx.Value(runnerArchKey{}) != nil {
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, runnerArchKey{}, &runnerArchCache{lock: make(chan struct{}, 1)})
|
||||||
|
}
|
||||||
|
|
||||||
|
type runnerArchCache struct {
|
||||||
|
lock chan struct{} // lets cancelled jobs stop waiting for the cache
|
||||||
|
arch string
|
||||||
|
}
|
||||||
|
|
||||||
|
func cachedRunnerArch(ctx context.Context) string {
|
||||||
|
cache, ok := ctx.Value(runnerArchKey{}).(*runnerArchCache)
|
||||||
|
if !ok {
|
||||||
|
return RunnerArch(ctx)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case cache.lock <- struct{}{}:
|
||||||
|
defer func() { <-cache.lock }()
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if cache.arch == "" {
|
||||||
|
cache.arch = RunnerArch(ctx)
|
||||||
|
}
|
||||||
|
return cache.arch
|
||||||
|
}
|
||||||
|
|
||||||
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
|
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
"os": "Linux",
|
"os": "Linux",
|
||||||
"arch": RunnerArch(ctx),
|
"arch": cachedRunnerArch(ctx),
|
||||||
"temp": "/tmp",
|
"temp": "/tmp",
|
||||||
"tool_cache": DefaultToolCache,
|
"tool_cache": DefaultToolCache,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,14 +5,13 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestContainerPath(t *testing.T) {
|
func TestContainerPath(t *testing.T) {
|
||||||
@@ -22,42 +21,27 @@ func TestContainerPath(t *testing.T) {
|
|||||||
workDir string
|
workDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
linuxcontainerext := &LinuxContainerEnvironmentExtensions{}
|
cwd, err := os.Getwd()
|
||||||
|
require.NoError(t, err)
|
||||||
|
tests := []containerPathJob{
|
||||||
|
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
||||||
|
{"/home/act", `/home/act/`, ""},
|
||||||
|
{cwd, ".", ""},
|
||||||
|
}
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
cwd, err := os.Getwd()
|
|
||||||
if err != nil {
|
|
||||||
log.Error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rootDrive := os.Getenv("SystemDrive")
|
rootDrive := os.Getenv("SystemDrive")
|
||||||
rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
|
tests = []containerPathJob{
|
||||||
for _, v := range []containerPathJob{
|
|
||||||
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
|
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
|
||||||
{"/mnt/f/work/dir", `F:\work\dir`, ""},
|
{"/mnt/f/work/dir", `F:\work\dir`, ""},
|
||||||
{"/mnt/c/windows/to/unix", "windows\\to\\unix", rootDrive + "\\"},
|
{"/mnt/c/windows/to/unix", "windows\\to\\unix", rootDrive + "\\"},
|
||||||
{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", rootDrive + "\\"},
|
{"/mnt/" + strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + "/act", "act", rootDrive + "\\"},
|
||||||
} {
|
|
||||||
if v.workDir != "" {
|
|
||||||
t.Chdir(v.workDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
t.Chdir(cwd)
|
for _, test := range tests {
|
||||||
} else {
|
if test.workDir != "" {
|
||||||
cwd, err := os.Getwd()
|
t.Chdir(test.workDir)
|
||||||
if err != nil {
|
|
||||||
log.Error(err)
|
|
||||||
}
|
|
||||||
for _, v := range []containerPathJob{
|
|
||||||
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
|
||||||
{"/home/act", `/home/act/`, ""},
|
|
||||||
{cwd, ".", ""},
|
|
||||||
} {
|
|
||||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
|
||||||
}
|
}
|
||||||
|
assert.Equal(t, test.destinationPath, (&LinuxContainerEnvironmentExtensions{}).ToContainerPath(test.sourcePath))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,5 +50,4 @@ type typeAssertMockContainer struct {
|
|||||||
LinuxContainerEnvironmentExtensions
|
LinuxContainerEnvironmentExtensions
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment
|
|
||||||
var _ ExecutionsEnvironment = &typeAssertMockContainer{}
|
var _ ExecutionsEnvironment = &typeAssertMockContainer{}
|
||||||
@@ -7,6 +7,7 @@ package container
|
|||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -18,7 +19,8 @@ import (
|
|||||||
"golang.org/x/text/transform"
|
"golang.org/x/text/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor {
|
// parseEnvFile reads a GITHUB_ENV style file, splitting lines like GitHub's runner on Windows or elsewhere.
|
||||||
|
func parseEnvFile(e Container, srcPath string, env *map[string]string, windows bool) common.Executor {
|
||||||
localEnv := *env
|
localEnv := *env
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
envTar, err := e.GetContainerArchive(ctx, srcPath)
|
||||||
@@ -38,8 +40,22 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
s := bufio.NewScanner(decoded)
|
s := bufio.NewScanner(decoded)
|
||||||
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
|
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
|
||||||
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||||
|
scanned := 0
|
||||||
|
s.Split(func(data []byte, atEOF bool) (int, []byte, error) {
|
||||||
|
if index := bytes.IndexByte(data[scanned:], '\n'); index >= 0 {
|
||||||
|
advance := scanned + index + 1
|
||||||
|
scanned = 0
|
||||||
|
return advance, data[:advance], nil
|
||||||
|
}
|
||||||
|
scanned = len(data)
|
||||||
|
if atEOF && len(data) > 0 {
|
||||||
|
scanned = 0
|
||||||
|
return len(data), data, nil
|
||||||
|
}
|
||||||
|
return 0, nil, nil
|
||||||
|
})
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
line := s.Text()
|
line := trimEnvLineEnding(s.Text(), windows)
|
||||||
// GitHub's runner ignores blank lines
|
// GitHub's runner ignores blank lines
|
||||||
if strings.TrimSpace(line) == "" {
|
if strings.TrimSpace(line) == "" {
|
||||||
continue
|
continue
|
||||||
@@ -52,10 +68,13 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
case multiLineEnv != -1:
|
case multiLineEnv != -1:
|
||||||
var multiLineEnvContent []string
|
var multiLineEnvContent []string
|
||||||
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
multiLineEnvDelimiter := line[multiLineEnv+2:]
|
||||||
|
if multiLineEnv == 0 || multiLineEnvDelimiter == "" {
|
||||||
|
return fmt.Errorf("invalid format '%v', name and delimiter must not be empty", line)
|
||||||
|
}
|
||||||
delimiterFound := false
|
delimiterFound := false
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
content := s.Text()
|
content := s.Text()
|
||||||
if content == multiLineEnvDelimiter {
|
if trimEnvLineEnding(content, windows) == multiLineEnvDelimiter {
|
||||||
delimiterFound = true
|
delimiterFound = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -67,7 +86,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
if !delimiterFound {
|
if !delimiterFound {
|
||||||
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
|
||||||
}
|
}
|
||||||
localEnv[line[:multiLineEnv]] = strings.Join(multiLineEnvContent, "\n")
|
localEnv[line[:multiLineEnv]] = trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
|
||||||
}
|
}
|
||||||
@@ -75,7 +94,13 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
|||||||
if err := s.Err(); err != nil {
|
if err := s.Err(); err != nil {
|
||||||
return fmt.Errorf("reading env file: %w", err)
|
return fmt.Errorf("reading env file: %w", err)
|
||||||
}
|
}
|
||||||
env = &localEnv
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func trimEnvLineEnding(line string, windows bool) string {
|
||||||
|
if windows && strings.HasSuffix(line, "\r\n") {
|
||||||
|
return line[:len(line)-2]
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(line, "\n")
|
||||||
|
}
|
||||||
@@ -5,7 +5,6 @@ package container
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -13,7 +12,6 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"golang.org/x/text/encoding"
|
|
||||||
"golang.org/x/text/encoding/unicode"
|
"golang.org/x/text/encoding/unicode"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,117 +21,78 @@ func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
|||||||
return e, filepath.Join(e.Path, "envfile")
|
return e, filepath.Join(e.Path, "envfile")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileSingleLine(t *testing.T) {
|
func TestParseEnvFile(t *testing.T) {
|
||||||
e, envPath := newTestHostEnv(t)
|
t.Parallel()
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\nBAZ=qux\n"), 0o600))
|
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||||
|
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||||
env := map[string]string{}
|
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "bar", env["FOO"])
|
return string(content)
|
||||||
assert.Equal(t, "qux", env["BAZ"])
|
}
|
||||||
}
|
for _, test := range []struct {
|
||||||
|
name, content string
|
||||||
func TestParseEnvFileMultiLine(t *testing.T) {
|
want map[string]string
|
||||||
e, envPath := newTestHostEnv(t)
|
}{
|
||||||
content := "FOO<<EOF\nline1\nline2\nEOF\n"
|
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
{"single line CRLF and final bare CR", "FOO=bar\r\nBAZ=qux\r", map[string]string{"FOO": "bar\r", "BAZ": "qux\r"}},
|
||||||
|
{"single line without final newline", "FOO=bar", map[string]string{"FOO": "bar"}},
|
||||||
env := map[string]string{}
|
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
|
||||||
assert.Equal(t, "line1\nline2", env["FOO"])
|
{"multiline without final newline", "FOO<<EOF\nline1\r\nline2\r\nEOF", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||||
}
|
{"empty multiline without final newline", "FOO<<EOF\nEOF", map[string]string{"FOO": ""}},
|
||||||
|
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
|
||||||
func TestParseEnvFileLargeValueWithinLimit(t *testing.T) {
|
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
|
||||||
e, envPath := newTestHostEnv(t)
|
{"multiline CRLF", "FOO<<EOF\r\nline1\r\nline2\r\nEOF\r\n", map[string]string{"FOO": "line1\r\nline2\r"}},
|
||||||
big := strings.Repeat("x", 2*1024*1024)
|
{"multiline mixed line endings", "FOO<<EOF\nline1\r\nline2\n\r\nEOF\n", map[string]string{"FOO": "line1\r\nline2\n\r"}},
|
||||||
content := "FOO<<EOF\n" + big + "\nEOF\n"
|
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
|
||||||
|
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||||
env := map[string]string{}
|
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
} {
|
||||||
assert.Equal(t, big, env["FOO"])
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
e, envPath := newTestHostEnv(t)
|
||||||
|
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||||
|
env := map[string]string{}
|
||||||
|
require.NoError(t, parseEnvFile(e, envPath, &env, false)(t.Context()))
|
||||||
|
assert.Equal(t, test.want, env)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("Windows line endings", func(t *testing.T) {
|
||||||
|
hostEnv, envPath := newTestHostEnv(t)
|
||||||
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||||
|
env := map[string]string{}
|
||||||
|
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
||||||
|
assert.Equal(t, map[string]string{"FOO": "bar", "MULTI": "line1\r\nline2"}, env)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
e, envPath := newTestHostEnv(t)
|
e, envPath := newTestHostEnv(t)
|
||||||
tooBig := strings.Repeat("x", 17*1024*1024) // over the 16 MiB cap
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
||||||
content := "FOO<<EOF\n" + tooBig + "\nEOF\n"
|
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
err := parseEnvFile(e, envPath, &env)(context.Background())
|
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||||
assert.Contains(t, err.Error(), "reading env file")
|
assert.Contains(t, err.Error(), "reading env file")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Regression test: a blank line used to fail the job at "Complete Job", after
|
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||||
// every step had already been recorded as successful.
|
for name, content := range map[string]string{
|
||||||
func TestParseEnvFileBlankLines(t *testing.T) {
|
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
||||||
e, envPath := newTestHostEnv(t)
|
"empty name": "<<EOF\nline1\nEOF\n",
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("\nFOO=bar\n\n \nBAZ=qux\n\n"), 0o600))
|
"empty delimiter": "FOO<<\nline1\n\n",
|
||||||
|
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
||||||
env := map[string]string{}
|
} {
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
t.Run(name, func(t *testing.T) {
|
||||||
assert.Equal(t, "bar", env["FOO"])
|
|
||||||
assert.Equal(t, "qux", env["BAZ"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// blank lines inside a heredoc value are content, not separators
|
|
||||||
func TestParseEnvFileMultiLineKeepsBlankLines(t *testing.T) {
|
|
||||||
e, envPath := newTestHostEnv(t)
|
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\n\nline2\nEOF\n"), 0o600))
|
|
||||||
|
|
||||||
env := map[string]string{}
|
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
|
||||||
assert.Equal(t, "line1\n\nline2", env["FOO"])
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n\nline2\nEOF\n"), 0o600))
|
|
||||||
env = map[string]string{}
|
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
|
||||||
assert.Equal(t, "\nline2", env["FOO"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseEnvFileUTF8BOM(t *testing.T) {
|
|
||||||
e, envPath := newTestHostEnv(t)
|
|
||||||
content := append([]byte{0xEF, 0xBB, 0xBF}, []byte("FOO=bar\n")...)
|
|
||||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
|
||||||
|
|
||||||
env := map[string]string{}
|
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
|
||||||
assert.Equal(t, "bar", env["FOO"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Windows host mode: PowerShell 5.1 redirection writes UTF-16, which used to be
|
|
||||||
// unrecognisable as KEY=VALUE, so the writes were silently ignored.
|
|
||||||
func TestParseEnvFileUTF16(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
encoder *encoding.Encoder
|
|
||||||
}{
|
|
||||||
{"little endian", unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder()},
|
|
||||||
{"big endian", unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder()},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
e, envPath := newTestHostEnv(t)
|
e, envPath := newTestHostEnv(t)
|
||||||
content, err := tt.encoder.Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||||
assert.Equal(t, "bar", env["FOO"])
|
require.Error(t, err)
|
||||||
assert.Equal(t, "line1", env["MULTI"])
|
assert.Contains(t, err.Error(), "delimiter")
|
||||||
|
assert.Empty(t, env)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileMissingDelimiter(t *testing.T) {
|
|
||||||
e, envPath := newTestHostEnv(t)
|
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
|
|
||||||
|
|
||||||
env := map[string]string{}
|
|
||||||
err := parseEnvFile(e, envPath, &env)(context.Background())
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "delimiter")
|
|
||||||
}
|
|
||||||
+6
-20
@@ -13,10 +13,6 @@ import (
|
|||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/filecollector"
|
"gitea.com/gitea/runner/act/filecollector"
|
||||||
|
|
||||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
|
||||||
"github.com/go-git/go-billy/v5/osfs"
|
|
||||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
||||||
@@ -46,23 +42,13 @@ func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useG
|
|||||||
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
||||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||||
|
|
||||||
var ignorer gitignore.Matcher
|
|
||||||
if useGitIgnore {
|
|
||||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
|
||||||
if err != nil {
|
|
||||||
logger.Debugf("Error loading .gitignore: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ignorer = gitignore.NewMatcher(ps)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
|
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
|
||||||
Ignorer: ignorer,
|
SrcPath: srcPath,
|
||||||
SrcPath: srcPath,
|
SrcPrefix: srcPrefix,
|
||||||
SrcPrefix: srcPrefix,
|
SkipGitDir: skipGitDir,
|
||||||
SkipGitDir: skipGitDir,
|
UseGitIgnore: useGitIgnore,
|
||||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
||||||
}).CollectFiles(ctx, []string{})); err != nil {
|
}).CollectFiles(ctx)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return tw.Close()
|
return tw.Close()
|
||||||
|
|||||||
Vendored
-5
@@ -1,5 +0,0 @@
|
|||||||
FROM scratch
|
|
||||||
ENV PATH="/this/path/does/not/exists/anywhere:/this/either"
|
|
||||||
ENV SOME_RANDOM_VAR=""
|
|
||||||
ENV ANOTHER_ONE="BUT_I_HAVE_VALUE"
|
|
||||||
ENV CONFLICT_VAR="I_EXIST_ONLY_HERE"
|
|
||||||
-1
@@ -1 +0,0 @@
|
|||||||
testfile
|
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/moby/moby/api/types/mount"
|
||||||
|
)
|
||||||
|
|
||||||
|
type VolumeSpec struct {
|
||||||
|
Type mount.Type
|
||||||
|
Source string
|
||||||
|
Target string
|
||||||
|
ReadOnly bool
|
||||||
|
NoCopy bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseVolume parses spec like `docker run -v` does, without knowing the target platform.
|
||||||
|
func ParseVolume(spec string) (VolumeSpec, error) {
|
||||||
|
switch len(spec) {
|
||||||
|
case 0:
|
||||||
|
return VolumeSpec{}, errors.New("invalid empty volume spec")
|
||||||
|
case 1, 2:
|
||||||
|
return VolumeSpec{Type: mount.TypeVolume, Target: spec}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
spec = string([]rune(spec)) // each invalid UTF-8 byte becomes U+FFFD, as in the JSON the daemon receives
|
||||||
|
var parts []string
|
||||||
|
for rest := spec; ; {
|
||||||
|
start := 0
|
||||||
|
if hasDriveLetter(rest) {
|
||||||
|
start = strings.IndexByte(rest, ':') + 1
|
||||||
|
}
|
||||||
|
end := strings.IndexByte(rest[start:], ':')
|
||||||
|
if end < 0 {
|
||||||
|
parts = append(parts, rest)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
parts = append(parts, rest[:start+end])
|
||||||
|
rest = rest[start+end+1:]
|
||||||
|
}
|
||||||
|
if len(parts) > 3 {
|
||||||
|
return VolumeSpec{}, fmt.Errorf("invalid spec: %s: too many colons", spec)
|
||||||
|
}
|
||||||
|
if slices.Contains(parts, "") {
|
||||||
|
return VolumeSpec{}, fmt.Errorf("invalid spec: %s: empty section between colons", spec)
|
||||||
|
}
|
||||||
|
|
||||||
|
volume := VolumeSpec{Type: mount.TypeVolume, Target: parts[0]}
|
||||||
|
if len(parts) > 1 {
|
||||||
|
volume.Source, volume.Target = parts[0], parts[1]
|
||||||
|
if isHostPath(volume.Source) {
|
||||||
|
volume.Type = mount.TypeBind
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(parts) == 3 {
|
||||||
|
for option := range strings.SplitSeq(parts[2], ",") {
|
||||||
|
switch option {
|
||||||
|
case "ro":
|
||||||
|
volume.ReadOnly = true
|
||||||
|
case "rw":
|
||||||
|
volume.ReadOnly = false
|
||||||
|
case "nocopy":
|
||||||
|
volume.NoCopy = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return volume, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasDriveLetter(s string) bool {
|
||||||
|
letter, size := utf8.DecodeRuneInString(s)
|
||||||
|
return size < len(s) && s[size] == ':' && unicode.IsLetter(letter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isHostPath(source string) bool {
|
||||||
|
return strings.ContainsAny(source[:1], "./~") || strings.HasPrefix(source, `\\`) || hasDriveLetter(source)
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/moby/moby/api/types/mount"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseVolumeReplacesInvalidUTF8LikeTheDaemonSees(t *testing.T) {
|
||||||
|
parsed, err := ParseVolume("/data/\xff\xfe:/mnt:ro")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, VolumeSpec{Type: mount.TypeBind, Source: "/data/��", Target: "/mnt", ReadOnly: true}, parsed)
|
||||||
|
}
|
||||||
@@ -16,10 +16,10 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
git "github.com/go-git/go-git/v5"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
"gitea.com/gitea/runner/act/common/git"
|
||||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
|
||||||
"github.com/go-git/go-git/v5/plumbing/format/index"
|
"golang.org/x/text/unicode/norm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Handler interface {
|
type Handler interface {
|
||||||
@@ -43,7 +43,6 @@ func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
|
|||||||
// update the name to correctly reflect the desired destination when untaring
|
// update the name to correctly reflect the desired destination when untaring
|
||||||
header.Name = path.Join(tc.DstDir, fpath)
|
header.Name = path.Join(tc.DstDir, fpath)
|
||||||
header.Mode = int64(fi.Mode())
|
header.Mode = int64(fi.Mode())
|
||||||
header.ModTime = fi.ModTime()
|
|
||||||
header.Uid = tc.UID
|
header.Uid = tc.UID
|
||||||
header.Gid = tc.GID
|
header.Gid = tc.GID
|
||||||
|
|
||||||
@@ -58,10 +57,8 @@ func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// copy file data into tar writer
|
// copy file data into tar writer
|
||||||
if _, err := io.Copy(tc.TarWriter, f); err != nil {
|
_, err = io.Copy(tc.TarWriter, f)
|
||||||
return err
|
return err
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type CopyCollector struct {
|
type CopyCollector struct {
|
||||||
@@ -87,80 +84,67 @@ func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer df.Close()
|
defer df.Close()
|
||||||
if _, err := io.Copy(df, f); err != nil {
|
_, err = io.Copy(df, f)
|
||||||
return err
|
return err
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type FileCollector struct {
|
type FileCollector struct {
|
||||||
Ignorer gitignore.Matcher
|
SrcPath string
|
||||||
SrcPath string
|
SrcPrefix string
|
||||||
SrcPrefix string
|
SkipGitDir bool
|
||||||
SkipGitDir bool
|
UseGitIgnore bool
|
||||||
Handler Handler
|
Handler Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
func openGitIndex(path string) (*index.Index, error) {
|
// foldPath compares git's spelling of a path with the filesystem's, which may differ in case or Unicode normalization.
|
||||||
repo, err := git.PlainOpen(path)
|
func foldPath(name string) string {
|
||||||
if err != nil {
|
return strings.ToLower(norm.NFC.String(name))
|
||||||
return nil, err
|
}
|
||||||
|
|
||||||
|
func (fc *FileCollector) gitFiles(ctx context.Context) map[string]bool {
|
||||||
|
if !fc.UseGitIgnore {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
return repo.Storer.Index()
|
files, err := git.ListFiles(ctx, fc.SrcPath)
|
||||||
|
if err != nil {
|
||||||
|
common.Logger(ctx).Debugf("Copying %s without applying .gitignore: %v", fc.SrcPath, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
keep := map[string]bool{".": true}
|
||||||
|
for _, file := range files {
|
||||||
|
for file = foldPath(file); !keep[file]; file = path.Dir(file) {
|
||||||
|
keep[file] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keep
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc {
|
func (fc *FileCollector) CollectFiles(ctx context.Context) filepath.WalkFunc {
|
||||||
i, _ := openGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...)))
|
keep := fc.gitFiles(ctx)
|
||||||
return func(file string, fi os.FileInfo, err error) error {
|
return func(file string, fi os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if ctx != nil && ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return errors.New("copy cancelled")
|
return errors.New("copy cancelled")
|
||||||
}
|
}
|
||||||
|
if fc.SkipGitDir && fi.Name() == ".git" {
|
||||||
sansPrefix := strings.TrimPrefix(file, fc.SrcPrefix)
|
|
||||||
split := strings.Split(sansPrefix, string(filepath.Separator))
|
|
||||||
// The root folders should be skipped, submodules only have the last path component set to "." by filepath.Walk
|
|
||||||
if fi.IsDir() && len(split) > 0 && split[len(split)-1] == "." {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if fc.SkipGitDir && split[len(split)-1] == ".git" {
|
|
||||||
if fi.IsDir() {
|
if fi.IsDir() {
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
var entry *index.Entry
|
if keep != nil {
|
||||||
if i != nil {
|
rel, _ := filepath.Rel(fc.SrcPath, file)
|
||||||
entry, err = i.Entry(strings.Join(split[len(submodulePath):], "/"))
|
if rel = filepath.ToSlash(rel); !keep[foldPath(rel)] && !strings.Contains("/"+rel+"/", "/.git/") {
|
||||||
} else {
|
if fi.IsDir() {
|
||||||
err = index.ErrEntryNotFound
|
|
||||||
}
|
|
||||||
if err != nil && fc.Ignorer != nil && fc.Ignorer.Match(split, fi.IsDir()) {
|
|
||||||
if fi.IsDir() {
|
|
||||||
if i != nil {
|
|
||||||
ms, err := i.Glob(strings.Join(append(split[len(submodulePath):], "**"), "/"))
|
|
||||||
if err != nil || len(ms) == 0 {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err == nil && entry.Mode == filemode.Submodule {
|
path := filepath.ToSlash(strings.TrimPrefix(file, fc.SrcPrefix))
|
||||||
err = filepath.Walk(file, fc.CollectFiles(ctx, split))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
path := filepath.ToSlash(sansPrefix)
|
|
||||||
|
|
||||||
// return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update)
|
|
||||||
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||||
linkName, err := os.Readlink(file)
|
linkName, err := os.Readlink(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -171,17 +155,12 @@ func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []strin
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// open file
|
|
||||||
f, err := os.Open(file)
|
f, err := os.Open(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
defer context.AfterFunc(ctx, func() { _ = f.Close() })()
|
||||||
if ctx != nil {
|
|
||||||
stop := context.AfterFunc(ctx, func() { _ = f.Close() })
|
|
||||||
defer stop()
|
|
||||||
}
|
|
||||||
|
|
||||||
return fc.Handler.WriteFile(path, fi, "", f)
|
return fc.Handler.WriteFile(path, fi, "", f)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,106 +9,78 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
git "github.com/go-git/go-git/v5"
|
|
||||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestIgnoredTrackedfile(t *testing.T) {
|
func gitCmd(t *testing.T, args ...string) {
|
||||||
|
t.Helper()
|
||||||
|
out, err := exec.Command("git", args...).CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
||||||
repo, err := git.PlainInit(repoDir, false)
|
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
|
||||||
require.NoError(t, err)
|
gitCmd(t, "init", "-q", dir)
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte(".*\n"), 0o644))
|
}
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
|
for name, body := range map[string]string{
|
||||||
worktree, err := repo.Worktree()
|
".gitignore": ".*\nign/\n",
|
||||||
require.NoError(t, err)
|
".env": "",
|
||||||
_, err = worktree.Add(".gitignore")
|
"Renamed": "",
|
||||||
require.NoError(t, err)
|
"ign/tracked": "",
|
||||||
|
"ign/untracked": "",
|
||||||
|
"sub/.gitignore": "dropped\n",
|
||||||
|
"sub/dropped": "",
|
||||||
|
"sub/kept": "",
|
||||||
|
"nested/kept": "",
|
||||||
|
} {
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
||||||
|
}
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
||||||
|
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
||||||
|
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
||||||
|
for _, gitlink := range []string{"sub", "uninitialized"} {
|
||||||
|
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
|
||||||
|
}
|
||||||
|
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
||||||
|
expected["test.env"] = ".env"
|
||||||
|
}
|
||||||
|
|
||||||
var archive bytes.Buffer
|
var archive bytes.Buffer
|
||||||
tw := tar.NewWriter(&archive)
|
tw := tar.NewWriter(&archive)
|
||||||
patterns, err := gitignore.ReadPatterns(worktree.Filesystem, nil)
|
fc := &FileCollector{SrcPath: repoDir, SrcPrefix: repoDir + string(filepath.Separator), UseGitIgnore: true, Handler: &TarCollector{TarWriter: tw}}
|
||||||
require.NoError(t, err)
|
require.NoError(t, filepath.Walk(repoDir, fc.CollectFiles(t.Context())))
|
||||||
ignorer := gitignore.NewMatcher(patterns)
|
|
||||||
fc := &FileCollector{
|
|
||||||
Ignorer: ignorer,
|
|
||||||
SrcPath: repoDir,
|
|
||||||
SrcPrefix: repoDir + string(filepath.Separator),
|
|
||||||
Handler: &TarCollector{
|
|
||||||
TarWriter: tw,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
|
|
||||||
assert.NoError(t, err, "successfully collect files")
|
|
||||||
require.NoError(t, tw.Close())
|
require.NoError(t, tw.Close())
|
||||||
|
|
||||||
|
files := map[string]string{}
|
||||||
tr := tar.NewReader(&archive)
|
tr := tar.NewReader(&archive)
|
||||||
h, err := tr.Next()
|
for header, err := tr.Next(); err != io.EOF; header, err = tr.Next() {
|
||||||
assert.NoError(t, err, "tar must not be empty") //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(t, ".gitignore", h.Name)
|
files[header.Name] = header.Linkname
|
||||||
_, err = tr.Next()
|
}
|
||||||
assert.ErrorIs(t, err, io.EOF, "tar must only contain one element")
|
assert.Contains(t, files, ".git/HEAD")
|
||||||
|
maps.DeleteFunc(files, func(name, _ string) bool { return strings.Contains("/"+name, "/.git/") })
|
||||||
|
assert.Equal(t, expected, files)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSymlinks(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("creating symlinks requires elevated privileges on Windows")
|
|
||||||
}
|
|
||||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
|
||||||
repo, err := git.PlainInit(repoDir, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
|
|
||||||
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
|
||||||
worktree, err := repo.Worktree()
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = worktree.Add("test.env")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var archive bytes.Buffer
|
|
||||||
tw := tar.NewWriter(&archive)
|
|
||||||
fc := &FileCollector{
|
|
||||||
SrcPath: repoDir,
|
|
||||||
SrcPrefix: repoDir + string(filepath.Separator),
|
|
||||||
Handler: &TarCollector{
|
|
||||||
TarWriter: tw,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
|
|
||||||
assert.NoError(t, err, "successfully collect files")
|
|
||||||
require.NoError(t, tw.Close())
|
|
||||||
tr := tar.NewReader(&archive)
|
|
||||||
h, err := tr.Next()
|
|
||||||
files := map[string]tar.Header{}
|
|
||||||
for err == nil {
|
|
||||||
files[h.Name] = *h
|
|
||||||
h, err = tr.Next()
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, ".env", files[".env"].Name)
|
|
||||||
assert.Equal(t, "test.env", files["test.env"].Name)
|
|
||||||
assert.Equal(t, ".env", files["test.env"].Linkname)
|
|
||||||
assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regression for https://gitea.com/gitea/runner/issues/876 and /941:
|
|
||||||
// re-copying an action directory must overwrite a pre-existing read-only
|
|
||||||
// file (e.g. a git pack .idx at mode 0444) instead of failing with EACCES
|
|
||||||
// on macOS or "Access is denied" on Windows.
|
|
||||||
func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
|
func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
|
||||||
dst := t.TempDir()
|
dst := t.TempDir()
|
||||||
target := filepath.Join(dst, "sub", "pack.idx")
|
target := filepath.Join(dst, "sub", "pack.idx")
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
|
||||||
require.NoError(t, os.WriteFile(target, []byte("old"), 0o444))
|
require.NoError(t, os.WriteFile(target, []byte("old"), 0o444))
|
||||||
|
fi, err := os.Stat(target)
|
||||||
src := filepath.Join(t.TempDir(), "pack.idx")
|
|
||||||
require.NoError(t, os.WriteFile(src, []byte("new"), 0o444))
|
|
||||||
fi, err := os.Stat(src)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
cc := &CopyCollector{DstDir: dst}
|
cc := &CopyCollector{DstDir: dst}
|
||||||
@@ -119,8 +91,6 @@ func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
|
|||||||
assert.Equal(t, "new", string(got))
|
assert.Equal(t, "new", string(got))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Without the destination removal, os.Symlink fails with EEXIST when the
|
|
||||||
// path already holds a regular file from an earlier copy of the action.
|
|
||||||
func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("creating symlinks requires elevated privileges on Windows")
|
t.Skip("creating symlinks requires elevated privileges on Windows")
|
||||||
@@ -129,11 +99,8 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
|||||||
target := filepath.Join(dst, "link")
|
target := filepath.Join(dst, "link")
|
||||||
require.NoError(t, os.WriteFile(target, []byte("stale"), 0o644))
|
require.NoError(t, os.WriteFile(target, []byte("stale"), 0o644))
|
||||||
|
|
||||||
fi, err := os.Lstat(target)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
cc := &CopyCollector{DstDir: dst}
|
cc := &CopyCollector{DstDir: dst}
|
||||||
require.NoError(t, cc.WriteFile("link", fi, "target", nil))
|
require.NoError(t, cc.WriteFile("link", nil, "target", nil))
|
||||||
|
|
||||||
resolved, err := os.Readlink(target)
|
resolved, err := os.Readlink(target)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -143,7 +110,7 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
|||||||
func TestFileCollectorCancellationAndWalkError(t *testing.T) {
|
func TestFileCollectorCancellationAndWalkError(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
cancel()
|
cancel()
|
||||||
walk := (&FileCollector{}).CollectFiles(ctx, nil)
|
walk := (&FileCollector{}).CollectFiles(ctx)
|
||||||
|
|
||||||
err := walk("file", nil, nil)
|
err := walk("file", nil, nil)
|
||||||
require.EqualError(t, err, "copy cancelled")
|
require.EqualError(t, err, "copy cancelled")
|
||||||
|
|||||||
@@ -28,12 +28,12 @@ var (
|
|||||||
|
|
||||||
type pinnedKey struct{}
|
type pinnedKey struct{}
|
||||||
|
|
||||||
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
|
// WithPinnedCheckout keeps a run's sha, ref and repository fixed when HEAD moves, in pins unless ctx has some.
|
||||||
func WithPinnedCheckout(ctx context.Context) context.Context {
|
func WithPinnedCheckout(ctx context.Context, pins *sync.Map) context.Context {
|
||||||
if ctx.Value(pinnedKey{}) != nil {
|
if ctx.Value(pinnedKey{}) != nil {
|
||||||
return ctx
|
return ctx
|
||||||
}
|
}
|
||||||
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
|
return context.WithValue(ctx, pinnedKey{}, pins)
|
||||||
}
|
}
|
||||||
|
|
||||||
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
||||||
@@ -57,12 +57,29 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
case "pull_request_target":
|
case "pull_request_target":
|
||||||
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||||
case "pull_request", "pull_request_review", "pull_request_review_comment":
|
case "pull_request", "pull_request_review", "pull_request_review_comment":
|
||||||
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", ghc.Event["number"])
|
number := ghc.Event["number"]
|
||||||
|
if number == nil {
|
||||||
|
number = model.NestedMapLookup(ghc.Event, "pull_request", "number")
|
||||||
|
}
|
||||||
|
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", number)
|
||||||
|
if ghc.EventName == "pull_request" && ghc.Event["action"] == "closed" && model.NestedMapLookup(ghc.Event, "pull_request", "merged") == true {
|
||||||
|
ghc.Ref = "refs/heads/" + ghc.BaseRef
|
||||||
|
}
|
||||||
case "deployment", "deployment_status":
|
case "deployment", "deployment_status":
|
||||||
ghc.Ref = model.AsString(model.NestedMapLookup(ghc.Event, "deployment", "ref"))
|
ghc.Ref = model.AsString(model.NestedMapLookup(ghc.Event, "deployment", "ref"))
|
||||||
case "release":
|
case "release":
|
||||||
ghc.Ref = "refs/tags/" + model.AsString(model.NestedMapLookup(ghc.Event, "release", "tag_name"))
|
ghc.Ref = "refs/tags/" + model.AsString(model.NestedMapLookup(ghc.Event, "release", "tag_name"))
|
||||||
case "push", "create", "workflow_dispatch":
|
case "create":
|
||||||
|
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||||
|
if ghc.Ref != "" && !strings.HasPrefix(ghc.Ref, "refs/") {
|
||||||
|
switch model.AsString(ghc.Event["ref_type"]) {
|
||||||
|
case "branch":
|
||||||
|
ghc.Ref = "refs/heads/" + ghc.Ref
|
||||||
|
case "tag":
|
||||||
|
ghc.Ref = "refs/tags/" + ghc.Ref
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "push", "workflow_dispatch":
|
||||||
ghc.Ref = model.AsString(ghc.Event["ref"])
|
ghc.Ref = model.AsString(ghc.Event["ref"])
|
||||||
default:
|
default:
|
||||||
defaultBranch := model.AsString(model.NestedMapLookup(ghc.Event, "repository", "default_branch"))
|
defaultBranch := model.AsString(model.NestedMapLookup(ghc.Event, "repository", "default_branch"))
|
||||||
@@ -74,7 +91,7 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
if ghc.Ref == "" {
|
if ghc.Ref == "" {
|
||||||
ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) })
|
ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warningf("unable to get git ref: %v", err)
|
logger.Warnf("unable to get git ref: %v", err)
|
||||||
} else {
|
} else {
|
||||||
logger.Debugf("using github ref: %s", ref)
|
logger.Debugf("using github ref: %s", ref)
|
||||||
ghc.Ref = ref
|
ghc.Ref = ref
|
||||||
@@ -117,11 +134,10 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
|
|
||||||
if ghc.Sha == "" {
|
if ghc.Sha == "" {
|
||||||
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
|
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
|
||||||
_, revision, err := findGitRevision(ctx, repoPath)
|
return findGitRevision(ctx, repoPath)
|
||||||
return revision, err
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warningf("unable to get git revision: %v", err)
|
logger.Warnf("unable to get git revision: %v", err)
|
||||||
} else {
|
} else {
|
||||||
ghc.Sha = sha
|
ghc.Sha = sha
|
||||||
}
|
}
|
||||||
@@ -130,14 +146,14 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
|||||||
|
|
||||||
// SetRepositoryAndOwner resolves the repository of the context from the git
|
// SetRepositoryAndOwner resolves the repository of the context from the git
|
||||||
// remote in repoPath when it is not set yet, and derives its owner.
|
// remote in repoPath when it is not set yet, and derives its owner.
|
||||||
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, githubInstance, repoPath string) {
|
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||||
if ghc.Repository == "" {
|
if ghc.Repository == "" {
|
||||||
repo, err := findGithubRepo(ctx, repoPath, githubInstance)
|
repo, err := pinned(ctx, "repo:"+repoPath, func() (string, error) { return findGithubRepo(ctx, repoPath) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v, repoPath: %v): %v", githubInstance, repoPath, err)
|
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ghc.Repository = repo
|
ghc.Repository = repo
|
||||||
}
|
}
|
||||||
ghc.RepositoryOwner = strings.Split(ghc.Repository, "/")[0]
|
ghc.RepositoryOwner, _, _ = strings.Cut(ghc.Repository, "/")
|
||||||
}
|
}
|
||||||
@@ -7,29 +7,21 @@ package ghcontext
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.dev/actionslib/pkg/model"
|
"gitea.dev/actionslib/pkg/model"
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSetRef(t *testing.T) {
|
func TestSetRef(t *testing.T) {
|
||||||
log.SetLevel(log.DebugLevel)
|
|
||||||
|
|
||||||
oldFindGitRef := findGitRef
|
oldFindGitRef := findGitRef
|
||||||
oldFindGitRevision := findGitRevision
|
|
||||||
defer func() { findGitRef = oldFindGitRef }()
|
defer func() { findGitRef = oldFindGitRef }()
|
||||||
defer func() { findGitRevision = oldFindGitRevision }()
|
|
||||||
|
|
||||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||||
return "refs/heads/master", nil
|
return "refs/heads/master", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
|
||||||
return "", "1234fakesha", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
tables := []struct {
|
tables := []struct {
|
||||||
eventName string
|
eventName string
|
||||||
event map[string]any
|
event map[string]any
|
||||||
@@ -50,6 +42,30 @@ func TestSetRef(t *testing.T) {
|
|||||||
ref: "refs/pull/1234/merge",
|
ref: "refs/pull/1234/merge",
|
||||||
refName: "1234/merge",
|
refName: "1234/merge",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request_review",
|
||||||
|
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request",
|
||||||
|
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": true}},
|
||||||
|
ref: "refs/heads/master",
|
||||||
|
refName: "master",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request",
|
||||||
|
event: map[string]any{"number": 1234., "action": "closed", "pull_request": map[string]any{"merged": false}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "pull_request_review_comment",
|
||||||
|
event: map[string]any{"pull_request": map[string]any{"number": 1234.}},
|
||||||
|
ref: "refs/pull/1234/merge",
|
||||||
|
refName: "1234/merge",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
eventName: "deployment",
|
eventName: "deployment",
|
||||||
event: map[string]any{
|
event: map[string]any{
|
||||||
@@ -78,6 +94,24 @@ func TestSetRef(t *testing.T) {
|
|||||||
ref: "refs/heads/somebranch",
|
ref: "refs/heads/somebranch",
|
||||||
refName: "somebranch",
|
refName: "somebranch",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "feature/new", "ref_type": "branch"},
|
||||||
|
ref: "refs/heads/feature/new",
|
||||||
|
refName: "feature/new",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "v1", "ref_type": "tag"},
|
||||||
|
ref: "refs/tags/v1",
|
||||||
|
refName: "v1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventName: "create",
|
||||||
|
event: map[string]any{"ref": "refs/tags/v1", "ref_type": "tag"},
|
||||||
|
ref: "refs/tags/v1",
|
||||||
|
refName: "v1",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
eventName: "unknown",
|
eventName: "unknown",
|
||||||
event: map[string]any{
|
event: map[string]any{
|
||||||
@@ -104,7 +138,7 @@ func TestSetRef(t *testing.T) {
|
|||||||
Event: table.event,
|
Event: table.event,
|
||||||
}
|
}
|
||||||
|
|
||||||
SetRef(context.Background(), ghc, "/some/dir")
|
SetRef(t.Context(), ghc, "/some/dir")
|
||||||
ghc.SetRefTypeAndName()
|
ghc.SetRefTypeAndName()
|
||||||
|
|
||||||
assert.Equal(t, table.ref, ghc.Ref)
|
assert.Equal(t, table.ref, ghc.Ref)
|
||||||
@@ -113,7 +147,7 @@ func TestSetRef(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.Run("no-default-branch", func(t *testing.T) {
|
t.Run("no-default-branch", func(t *testing.T) {
|
||||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||||
return "", errors.New("no default branch")
|
return "", errors.New("no default branch")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,26 +156,22 @@ func TestSetRef(t *testing.T) {
|
|||||||
Event: map[string]any{},
|
Event: map[string]any{},
|
||||||
}
|
}
|
||||||
|
|
||||||
SetRef(context.Background(), ghc, "/some/dir")
|
SetRef(t.Context(), ghc, "/some/dir")
|
||||||
|
|
||||||
assert.Equal(t, "refs/heads/master", ghc.Ref)
|
assert.Equal(t, "refs/heads/master", ghc.Ref)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSetSha(t *testing.T) {
|
func TestSetSha(t *testing.T) {
|
||||||
log.SetLevel(log.DebugLevel)
|
|
||||||
|
|
||||||
oldFindGitRef := findGitRef
|
oldFindGitRef := findGitRef
|
||||||
oldFindGitRevision := findGitRevision
|
oldFindGitRevision := findGitRevision
|
||||||
|
oldFindGithubRepo := findGithubRepo
|
||||||
defer func() { findGitRef = oldFindGitRef }()
|
defer func() { findGitRef = oldFindGitRef }()
|
||||||
defer func() { findGitRevision = oldFindGitRevision }()
|
defer func() { findGitRevision = oldFindGitRevision }()
|
||||||
|
defer func() { findGithubRepo = oldFindGithubRepo }()
|
||||||
|
|
||||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
findGitRevision = func(_ctx context.Context, _file string) (string, error) {
|
||||||
return "refs/heads/master", nil
|
return "1234fakesha", nil
|
||||||
}
|
|
||||||
|
|
||||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
|
||||||
return "", "1234fakesha", nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tables := []struct {
|
tables := []struct {
|
||||||
@@ -176,11 +206,6 @@ func TestSetSha(t *testing.T) {
|
|||||||
},
|
},
|
||||||
sha: "deployment-sha",
|
sha: "deployment-sha",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
eventName: "release",
|
|
||||||
event: map[string]any{},
|
|
||||||
sha: "1234fakesha",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
eventName: "push",
|
eventName: "push",
|
||||||
event: map[string]any{
|
event: map[string]any{
|
||||||
@@ -194,37 +219,34 @@ func TestSetSha(t *testing.T) {
|
|||||||
event: map[string]any{},
|
event: map[string]any{},
|
||||||
sha: "1234fakesha",
|
sha: "1234fakesha",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
eventName: "no-event",
|
|
||||||
event: map[string]any{},
|
|
||||||
sha: "1234fakesha",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
t.Run(table.eventName, func(t *testing.T) {
|
t.Run(table.eventName, func(t *testing.T) {
|
||||||
ghc := &model.GithubContext{
|
ghc := &model.GithubContext{
|
||||||
EventName: table.eventName,
|
EventName: table.eventName,
|
||||||
BaseRef: "master",
|
|
||||||
Event: table.event,
|
Event: table.event,
|
||||||
}
|
}
|
||||||
|
|
||||||
SetSha(context.Background(), ghc, "/some/dir")
|
SetSha(t.Context(), ghc, "/some/dir")
|
||||||
|
|
||||||
assert.Equal(t, table.sha, ghc.Sha)
|
assert.Equal(t, table.sha, ghc.Sha)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
|
t.Run("pinned checkout keeps the first sha, ref and repository", func(t *testing.T) {
|
||||||
ctx := WithPinnedCheckout(context.Background())
|
ctx := WithPinnedCheckout(t.Context(), &sync.Map{})
|
||||||
for _, head := range []string{"first", "moved"} {
|
for _, head := range []string{"first", "moved"} {
|
||||||
findGitRevision = func(context.Context, string) (string, string, error) { return "", head, nil }
|
findGitRevision = func(context.Context, string) (string, error) { return head, nil }
|
||||||
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
||||||
|
findGithubRepo = func(context.Context, string) (string, error) { return "owner/" + head, nil }
|
||||||
ghc := &model.GithubContext{Event: map[string]any{}}
|
ghc := &model.GithubContext{Event: map[string]any{}}
|
||||||
SetSha(ctx, ghc, "/some/dir")
|
SetSha(ctx, ghc, "/some/dir")
|
||||||
SetRef(ctx, ghc, "/some/dir")
|
SetRef(ctx, ghc, "/some/dir")
|
||||||
|
SetRepositoryAndOwner(ctx, ghc, "/some/dir")
|
||||||
assert.Equal(t, "first", ghc.Sha)
|
assert.Equal(t, "first", ghc.Sha)
|
||||||
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
||||||
|
assert.Equal(t, "owner/first", ghc.Repository)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -13,35 +13,23 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLookPath2SearchesPathAndEmptyElement(t *testing.T) {
|
func TestLookPath2SearchesPathAndEmptyElementOrUsesDirectPath(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
exe := filepath.Join(dir, "tool")
|
exe := filepath.Join(dir, "tool")
|
||||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := LookPath2("tool", map[string]string{"PATH": string(filepath.ListSeparator) + dir})
|
for file, searchPath := range map[string]string{"tool": string(filepath.ListSeparator) + dir, exe: ""} {
|
||||||
if err != nil {
|
t.Run(file, func(t *testing.T) {
|
||||||
t.Fatal(err)
|
got, err := LookPath2(file, map[string]string{"PATH": searchPath})
|
||||||
}
|
if err != nil {
|
||||||
if got != exe {
|
t.Fatal(err)
|
||||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
}
|
||||||
}
|
if got != exe {
|
||||||
}
|
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||||
|
}
|
||||||
func TestLookPath2DirectPathDoesNotSearchPath(t *testing.T) {
|
})
|
||||||
dir := t.TempDir()
|
|
||||||
exe := filepath.Join(dir, "tool")
|
|
||||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := LookPath2(exe, map[string]string{"PATH": ""})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got != exe {
|
|
||||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loaded 100 of 260 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user