mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-15 20:33:42 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 | ||
|
|
f2fbd67962 | ||
|
|
d6f5b68a0d | ||
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
|||||||
|
|
||||||
/bin
|
/bin
|
||||||
/dist
|
/dist
|
||||||
|
/.tmp
|
||||||
|
/.git/hooks
|
||||||
|
/.git/logs
|
||||||
|
/.git/rr-cache
|
||||||
/.env
|
/.env
|
||||||
/.runner
|
/.runner
|
||||||
|
/.runner.lock
|
||||||
/config.yaml
|
/config.yaml
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: go-setup
|
||||||
|
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||||
|
id: setup-go
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: false
|
||||||
|
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||||
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||||
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: cache-seeder
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- go.mod
|
||||||
|
- go.sum
|
||||||
|
- .golangci.yml
|
||||||
|
- Makefile
|
||||||
|
- .gitea/actions/go-setup/action.yml
|
||||||
|
- .gitea/workflows/cache-seeder.yml
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
seed:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- run: make deps-tools lint build
|
||||||
|
- name: compile the test binaries without running them
|
||||||
|
run: |
|
||||||
|
make test GOTEST_FLAGS='-race -exec true'
|
||||||
|
go test -tags e2e -exec true ./e2e/...
|
||||||
@@ -21,7 +21,7 @@ jobs:
|
|||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 26
|
||||||
- run: make lint-pr-title
|
- run: make lint-pr-title
|
||||||
env:
|
env:
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
@@ -34,8 +34,15 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
- name: Import GPG key
|
||||||
|
id: import_gpg
|
||||||
|
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||||
|
with:
|
||||||
|
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||||
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release --nightly
|
args: release --nightly
|
||||||
@@ -47,6 +54,7 @@ jobs:
|
|||||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||||
GORELEASER_FORCE_TOKEN: "gitea"
|
GORELEASER_FORCE_TOKEN: "gitea"
|
||||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||||
|
|
||||||
release-image:
|
release-image:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -79,7 +87,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
passphrase: ${{ secrets.PASSPHRASE }}
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release
|
args: release
|
||||||
@@ -82,7 +82,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
+33
-11
@@ -17,25 +17,24 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||||
- name: pre-pull test images
|
- name: pre-pull test images
|
||||||
env:
|
env:
|
||||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:d6aa754f16b3197301076f047b5def2f02ea1dbbc2ca920407d46d7ec7f87b20 # renovate: datasource=docker
|
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||||
run: |
|
run: |
|
||||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||||
|
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||||
for attempt in 1 2 3; do
|
for attempt in 1 2 3; do
|
||||||
docker pull "$image" && break
|
docker pull "$image" && break
|
||||||
[ "$attempt" = 3 ] || sleep 5
|
[ "$attempt" = 3 ] || sleep 5
|
||||||
done
|
done
|
||||||
|
fi
|
||||||
docker tag "$image" "${image%@*}"
|
docker tag "$image" "${image%@*}"
|
||||||
done
|
done
|
||||||
- name: lint
|
- name: lint
|
||||||
@@ -58,6 +57,32 @@ jobs:
|
|||||||
make coverage-report
|
make coverage-report
|
||||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
|
windows:
|
||||||
|
name: test on windows
|
||||||
|
runs-on: ubuntu-latest-kvm
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- name: prepare anonymous docker config
|
||||||
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
|
- name: windows vm cache key
|
||||||
|
id: windows-vm
|
||||||
|
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
- name: test on windows
|
||||||
|
id: test-windows
|
||||||
|
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||||
|
env:
|
||||||
|
WINDOWS_TEST_TIMEOUT: 45
|
||||||
|
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||||
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
|
||||||
e2e:
|
e2e:
|
||||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -68,10 +93,7 @@ jobs:
|
|||||||
- gitea/gitea:main-nightly
|
- gitea/gitea:main-nightly
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
- name: e2e compatibility
|
- name: e2e compatibility
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
/gitea-runner
|
/gitea-runner
|
||||||
|
/gitea-runner.exe
|
||||||
.env
|
.env
|
||||||
!/act/runner/testdata/secrets/.env
|
!/act/runner/testdata/secrets/.env
|
||||||
.runner
|
.runner
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ linters:
|
|||||||
deny:
|
deny:
|
||||||
- pkg: golang.org/x/exp
|
- pkg: golang.org/x/exp
|
||||||
desc: it's experimental and unreliable
|
desc: it's experimental and unreliable
|
||||||
|
- pkg: unsafe
|
||||||
|
desc: use typed APIs instead
|
||||||
logging:
|
logging:
|
||||||
files:
|
files:
|
||||||
- $all
|
- $all
|
||||||
|
|||||||
+5
-2
@@ -34,8 +34,6 @@ builds:
|
|||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm
|
goarch: arm
|
||||||
- goos: windows
|
|
||||||
goarch: arm64
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
@@ -82,6 +80,7 @@ builds:
|
|||||||
publishers:
|
publishers:
|
||||||
- name: cloudflare-r2
|
- name: cloudflare-r2
|
||||||
checksum: true
|
checksum: true
|
||||||
|
signature: true
|
||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
- glob: ./**.sha256
|
- glob: ./**.sha256
|
||||||
@@ -102,6 +101,10 @@ checksum:
|
|||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
|
|
||||||
|
signs:
|
||||||
|
- artifacts: checksum
|
||||||
|
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||||
|
|
||||||
snapshot:
|
snapshot:
|
||||||
version_template: "{{ .Branch }}-devel"
|
version_template: "{{ .Branch }}-devel"
|
||||||
|
|
||||||
|
|||||||
+20
-2
@@ -57,7 +57,25 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
|||||||
|
|
||||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||||
parallel.
|
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||||
|
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||||
|
|
||||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||||
|
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||||
|
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||||
|
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||||
|
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||||
|
Docker and Kubernetes.
|
||||||
|
|
||||||
|
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||||
|
|
||||||
|
## Windows tests
|
||||||
|
|
||||||
|
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
|
||||||
|
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
|
||||||
|
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||||
|
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||||
|
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||||
|
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||||
|
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||||
+5
-3
@@ -12,12 +12,14 @@ ENV GOPROXY=${GOPROXY:-}
|
|||||||
COPY . /opt/src/runner
|
COPY . /opt/src/runner
|
||||||
WORKDIR /opt/src/runner
|
WORKDIR /opt/src/runner
|
||||||
|
|
||||||
RUN make clean && make build
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
make clean && make build
|
||||||
|
|
||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.1-dind AS dind
|
FROM docker:29.9.2-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -39,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
DIST := dist
|
DIST := dist
|
||||||
EXECUTABLE := gitea-runner
|
|
||||||
GO ?= go
|
GO ?= go
|
||||||
|
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||||
|
|
||||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||||
|
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||||
|
|
||||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||||
|
|
||||||
@@ -15,8 +16,8 @@ DOCKER_IMAGE ?= gitea/runner
|
|||||||
DOCKER_TAG ?= nightly
|
DOCKER_TAG ?= nightly
|
||||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||||
|
|
||||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||||
|
|
||||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||||
|
|
||||||
@@ -150,11 +151,15 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
|||||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||||
|
|
||||||
|
.PHONY: test-windows
|
||||||
|
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||||
|
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||||
|
|
||||||
E2E_CONCURRENCY ?= 8
|
E2E_CONCURRENCY ?= 8
|
||||||
|
|
||||||
.PHONY: test-e2e
|
.PHONY: test-e2e
|
||||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||||
|
|
||||||
.PHONY: install
|
.PHONY: install
|
||||||
install: $(GOFILES) ## install the runner binary via `go install`
|
install: $(GOFILES) ## install the runner binary via `go install`
|
||||||
|
|||||||
@@ -6,7 +6,5 @@
|
|||||||
//
|
//
|
||||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||||
//
|
//
|
||||||
// TODO: Authorization
|
|
||||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
|
||||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||||
package artifactcache
|
package artifactcache
|
||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
@@ -58,6 +59,10 @@ type credKey struct{}
|
|||||||
type JobCredential struct {
|
type JobCredential struct {
|
||||||
Repo string `json:"repo"`
|
Repo string `json:"repo"`
|
||||||
|
|
||||||
|
Ref string `json:"ref"`
|
||||||
|
BaseRef string `json:"base_ref"`
|
||||||
|
DefaultRef string `json:"default_ref"`
|
||||||
|
|
||||||
// Results is the instance whose artifact service this server forwards for the job, and
|
// Results is the instance whose artifact service this server forwards for the job, and
|
||||||
// InsecureTLS how the runner reaches it; see results.go.
|
// InsecureTLS how the runner reaches it; see results.go.
|
||||||
Results string `json:"-"`
|
Results string `json:"-"`
|
||||||
@@ -67,6 +72,10 @@ type JobCredential struct {
|
|||||||
PublicURL string `json:"public_url"`
|
PublicURL string `json:"public_url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cred JobCredential) owns(cache *Cache) bool {
|
||||||
|
return cache.Repo == cred.Repo && cache.Ref == cred.Ref
|
||||||
|
}
|
||||||
|
|
||||||
// credEntry holds a registered job's credential along with an active
|
// credEntry holds a registered job's credential along with an active
|
||||||
// registration count. RegisterJob is reference-counted so that if two tasks
|
// registration count. RegisterJob is reference-counted so that if two tasks
|
||||||
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
||||||
@@ -96,6 +105,7 @@ type Handler struct {
|
|||||||
syncing sync.WaitGroup
|
syncing sync.WaitGroup
|
||||||
|
|
||||||
outboundIP string
|
outboundIP string
|
||||||
|
instanceTransport http.RoundTripper
|
||||||
|
|
||||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||||
// runner can use these endpoints to pre-register per-job
|
// runner can use these endpoints to pre-register per-job
|
||||||
@@ -115,7 +125,7 @@ type Handler struct {
|
|||||||
|
|
||||||
policy Policy
|
policy Policy
|
||||||
|
|
||||||
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
|
// freeDisk is a field so tests can drive makeRoom without a full volume.
|
||||||
freeDisk func(string) (uint64, error)
|
freeDisk func(string) (uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,6 +146,7 @@ type Options struct {
|
|||||||
|
|
||||||
Policy Policy
|
Policy Policy
|
||||||
Logger *logging.Logger
|
Logger *logging.Logger
|
||||||
|
InstanceTransport http.RoundTripper // reaches the instance artifact requests are forwarded to, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
||||||
@@ -147,6 +158,7 @@ func StartHandler(opts Options) (*Handler, error) {
|
|||||||
policy: opts.Policy.withDefaults(),
|
policy: opts.Policy.withDefaults(),
|
||||||
freeDisk: disk.FreeBytes,
|
freeDisk: disk.FreeBytes,
|
||||||
dbLock: make(chan struct{}, 1),
|
dbLock: make(chan struct{}, 1),
|
||||||
|
instanceTransport: opts.InstanceTransport,
|
||||||
}
|
}
|
||||||
|
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
@@ -652,7 +664,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, keys, version)
|
cache, err := h.lookupCache(db, cred, keys, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -668,11 +680,10 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
|
// lookupCache returns the entry to restore or (nil, nil), dropping matches whose blob was pruned.
|
||||||
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
|
func (h *Handler) lookupCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||||
// entry is dropped on the way out.
|
for {
|
||||||
func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
cache, err := findCache(db, cred, keys, version)
|
||||||
cache, err := findCache(db, repo, keys, version)
|
|
||||||
if err != nil || cache == nil {
|
if err != nil || cache == nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -680,14 +691,16 @@ func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version s
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !ok {
|
if ok {
|
||||||
h.deleteCache(db, cache)
|
|
||||||
return nil, nil //nolint:nilnil // absence is not an error here
|
|
||||||
}
|
|
||||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||||
// this call and the GET that follows it.
|
// this call and the GET that follows it.
|
||||||
h.touch(db, cache)
|
h.touch(db, cache)
|
||||||
return cache, nil
|
return cache, nil
|
||||||
|
}
|
||||||
|
if !h.deleteCache(db, cache) {
|
||||||
|
return nil, nil //nolint:nilnil // absence is not an error here
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// POST /_apis/artifactcache/caches
|
// POST /_apis/artifactcache/caches
|
||||||
@@ -699,7 +712,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size}
|
cache := &Cache{Repo: cred.Repo, Ref: cred.Ref, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||||
if cache.Size == 0 {
|
if cache.Size == 0 {
|
||||||
cache.Size = -1
|
cache.Size = -1
|
||||||
}
|
}
|
||||||
@@ -710,9 +723,10 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache.CreatedAt = now
|
cache.CreatedAt = now.Unix()
|
||||||
cache.UsedAt = now
|
cache.CreatedAtNanos = now.Nanosecond()
|
||||||
|
cache.UsedAt = now.Unix()
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -747,7 +761,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cache.Repo != cred.Repo {
|
if !cred.owns(cache) {
|
||||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -762,6 +776,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, 400, err)
|
h.responseJSON(w, r, 400, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.makeRoom(r.ContentLength)
|
||||||
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -778,6 +793,17 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, 400, err)
|
h.responseJSON(w, r, 400, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var request struct {
|
||||||
|
Size *int64 `json:"size"`
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||||
|
if err == nil && len(body) > 0 {
|
||||||
|
err = json.Unmarshal(body, &request)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
db, err := h.openDB()
|
db, err := h.openDB()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -795,7 +821,7 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cache.Repo != cred.Repo {
|
if !cred.owns(cache) {
|
||||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -805,6 +831,14 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.Size != nil {
|
||||||
|
if *request.Size < 0 || cache.Size >= 0 && cache.Size != *request.Size {
|
||||||
|
h.responseJSON(w, r, http.StatusBadRequest, errors.New("invalid cache size"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cache.Size = *request.Size
|
||||||
|
}
|
||||||
|
|
||||||
db.Close()
|
db.Close()
|
||||||
|
|
||||||
if err := h.commitCache(cache); err != nil {
|
if err := h.commitCache(cache); err != nil {
|
||||||
@@ -845,11 +879,8 @@ func (h *Handler) commitCache(cache *Cache) error {
|
|||||||
if h.s3 != nil {
|
if h.s3 != nil {
|
||||||
return nil // lifecycle rules on the bucket bound its size
|
return nil // lifecycle rules on the bucket bound its size
|
||||||
}
|
}
|
||||||
// A commit is the only thing that grows the store, so the only thing that can push the
|
|
||||||
// volume under the floor.
|
|
||||||
h.evictRepo(db, cache.Repo)
|
h.evictRepo(db, cache.Repo)
|
||||||
h.evictTotal(db)
|
h.evictTotal(db)
|
||||||
h.evictForFreeSpace(db)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1050,27 +1081,37 @@ func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time
|
|||||||
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if not found, return (nil, nil) instead of an error.
|
// findCache tries every key in one ref before the next ref, and returns (nil, nil) on a miss.
|
||||||
func findCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
func findCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||||
for _, prefix := range keys {
|
refs := []string{cred.Ref}
|
||||||
// if a key in the list matches exactly, don't return partial matches
|
for _, ref := range []string{cred.BaseRef, cred.DefaultRef} {
|
||||||
exact, err := findExactCache(db, repo, prefix, version, true)
|
if ref != "" && !slices.Contains(refs, ref) {
|
||||||
if err != nil {
|
refs = append(refs, ref)
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
if exact != nil {
|
|
||||||
return exact, nil
|
|
||||||
}
|
}
|
||||||
caches, err := db.find(func(cache *Cache) bool {
|
caches, err := db.find(func(cache *Cache) bool {
|
||||||
return cache.Repo == repo && strings.HasPrefix(cache.Key, prefix) && cache.Version == version && cache.Complete
|
return cache.Repo == cred.Repo && cache.Version == version && cache.Complete && slices.Contains(refs, cache.Ref) &&
|
||||||
|
slices.ContainsFunc(keys, func(prefix string) bool { return strings.HasPrefix(cache.Key, prefix) })
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("find cache: %w", err)
|
return nil, fmt.Errorf("find cache: %w", err)
|
||||||
}
|
}
|
||||||
if cache := latest(caches, func(cache *Cache) int64 { return cache.CreatedAt }); cache != nil {
|
for _, ref := range refs {
|
||||||
|
for _, prefix := range keys {
|
||||||
|
var exact, partial []*Cache
|
||||||
|
for _, cache := range caches {
|
||||||
|
if cache.Ref == ref && cache.Key == prefix {
|
||||||
|
exact = append(exact, cache)
|
||||||
|
} else if cache.Ref == ref && strings.HasPrefix(cache.Key, prefix) {
|
||||||
|
partial = append(partial, cache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// if a key in the list matches exactly, don't return partial matches
|
||||||
|
if cache := cmp.Or(latest(exact, (*Cache).createdAtNano), latest(partial, (*Cache).createdAtNano)); cache != nil {
|
||||||
return cache, nil
|
return cache, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1078,7 +1119,9 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
|||||||
if len(caches) == 0 {
|
if len(caches) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return slices.MaxFunc(caches, func(left, right *Cache) int { return cmp.Compare(stamp(left), stamp(right)) })
|
return slices.MaxFunc(caches, func(left, right *Cache) int {
|
||||||
|
return cmp.Or(cmp.Compare(stamp(left), stamp(right)), cmp.Compare(left.ID, right.ID))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
|
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
|
||||||
@@ -1088,16 +1131,16 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
|||||||
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
||||||
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
||||||
// touches UsedAt on every part.
|
// touches UsedAt on every part.
|
||||||
func findExactCache(db *cacheDB, repo, key, version string, complete bool) (*Cache, error) {
|
func findExactCache(db *cacheDB, cred JobCredential, key, version string, complete bool) (*Cache, error) {
|
||||||
caches, err := db.find(func(cache *Cache) bool {
|
caches, err := db.find(func(cache *Cache) bool {
|
||||||
return cache.Key == key && cache.Repo == repo && cache.Version == version && cache.Complete == complete
|
return cred.owns(cache) && cache.Key == key && cache.Version == version && cache.Complete == complete
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("find cache: %w", err)
|
return nil, fmt.Errorf("find cache: %w", err)
|
||||||
}
|
}
|
||||||
stamp := func(cache *Cache) int64 { return cache.UsedAt }
|
stamp := func(cache *Cache) int64 { return cache.UsedAt }
|
||||||
if complete {
|
if complete {
|
||||||
stamp = func(cache *Cache) int64 { return cache.CreatedAt }
|
stamp = (*Cache).createdAtNano
|
||||||
}
|
}
|
||||||
return latest(caches, stamp), nil
|
return latest(caches, stamp), nil
|
||||||
}
|
}
|
||||||
@@ -1131,9 +1174,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
|||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := db.get(id)
|
cache, err := db.get(id)
|
||||||
if errors.Is(err, errCacheNotFound) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1218,23 +1258,29 @@ func (h *Handler) gcCache() {
|
|||||||
h.evictExpired(db)
|
h.evictExpired(db)
|
||||||
h.evictSuperseded(db)
|
h.evictSuperseded(db)
|
||||||
h.evictOversized(db)
|
h.evictOversized(db)
|
||||||
h.evictForFreeSpace(db)
|
_ = db.Close()
|
||||||
|
h.makeRoom(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
|
// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
|
||||||
// accounted for.
|
func (h *Handler) makeRoom(need int64) {
|
||||||
func (h *Handler) evictForFreeSpace(db *cacheDB) {
|
|
||||||
free, err := h.freeDisk(h.dir)
|
free, err := h.freeDisk(h.dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if free >= uint64(h.policy.MinFreeDisk) {
|
shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
|
||||||
|
if h.s3 != nil || shortfall <= 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
db, err := h.openDB()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
caches := h.completedByUse(db)
|
caches := h.completedByUse(db)
|
||||||
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
|
total := totalSize(caches)
|
||||||
if total <= shortfall {
|
if total <= shortfall {
|
||||||
// Say so, or shedding everything and still being short reads as the backstop working.
|
// Say so, or shedding everything and still being short reads as the backstop working.
|
||||||
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
||||||
@@ -1257,17 +1303,18 @@ func (h *Handler) evictExpired(db *cacheDB) {
|
|||||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictSuperseded removes entries a newer one with the same key and version replaced. The
|
// evictSuperseded removes entries a newer one with the same repo, ref, key and version replaced.
|
||||||
// grouping includes Repo so two repos sharing a (key, version) do not evict each other.
|
|
||||||
func (h *Handler) evictSuperseded(db *cacheDB) {
|
func (h *Handler) evictSuperseded(db *cacheDB) {
|
||||||
type entry struct{ repo, key, version string }
|
type entry struct{ repo, ref, key, version string }
|
||||||
groups := make(map[entry][]*Cache)
|
groups := make(map[entry][]*Cache)
|
||||||
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
||||||
group := entry{cache.Repo, cache.Key, cache.Version}
|
group := entry{cache.Repo, cache.Ref, cache.Key, cache.Version}
|
||||||
groups[group] = append(groups[group], cache)
|
groups[group] = append(groups[group], cache)
|
||||||
}
|
}
|
||||||
for _, caches := range groups {
|
for _, caches := range groups {
|
||||||
slices.SortFunc(caches, func(left, right *Cache) int { return cmp.Compare(left.CreatedAt, right.CreatedAt) })
|
slices.SortFunc(caches, func(left, right *Cache) int {
|
||||||
|
return cmp.Or(cmp.Compare(left.createdAtNano(), right.createdAtNano()), cmp.Compare(left.ID, right.ID))
|
||||||
|
})
|
||||||
for _, cache := range caches[:len(caches)-1] {
|
for _, cache := range caches[:len(caches)-1] {
|
||||||
if inUse(cache) {
|
if inUse(cache) {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -15,8 +15,10 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/iotest"
|
||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -47,10 +49,12 @@ func signArtifactURL(h *Handler, id int64) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ExternalURL(t *testing.T) {
|
func TestHandler_ExternalURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
|
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_Upstream(t *testing.T) {
|
func TestHandler_Upstream(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var upstream *httptest.Server
|
var upstream *httptest.Server
|
||||||
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
assert.Equal(t, http.MethodPut, r.Method)
|
assert.Equal(t, http.MethodPut, r.Method)
|
||||||
@@ -100,6 +104,7 @@ func TestHandler_Upstream(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler(t *testing.T) {
|
func TestHandler(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
const version = "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
const version = "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
@@ -302,6 +307,37 @@ func TestHandler(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("chunked commit validates size omitted at reservation", func(t *testing.T) {
|
||||||
|
for _, size := range []int64{5, 10} {
|
||||||
|
t.Run(strconv.FormatInt(size, 10), func(t *testing.T) {
|
||||||
|
id := reserveCache(t, testClient, base, strings.ToLower(t.Name()), version, 0)
|
||||||
|
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), strings.NewReader("cache"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
req.Header.Set("Content-Range", "bytes 0-4/*")
|
||||||
|
resp, err := testClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
resp, err = testClient.Post(fmt.Sprintf("%s/caches/%d", base, id), "application/json", io.MultiReader(strings.NewReader(fmt.Sprintf(`{"size":%d}`, size))))
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
if size == 5 {
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
} else {
|
||||||
|
require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
cache, err := db.get(id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, size == 5, cache.Complete)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("get with bad id", func(t *testing.T) {
|
t.Run("get with bad id", func(t *testing.T) {
|
||||||
resp, err := testClient.Get(base + "/artifacts/invalid_id")
|
resp, err := testClient.Get(base + "/artifacts/invalid_id")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -438,6 +474,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_gcCache(t *testing.T) {
|
func TestHandler_gcCache(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir, Policy: Policy{Retention: testRetention}})
|
handler, err := StartHandler(Options{Dir: dir, Policy: Policy{Retention: testRetention}})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -540,6 +577,7 @@ func TestHandler_gcCache(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_evictPolicy(t *testing.T) {
|
func TestHandler_evictPolicy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
stale := func(d time.Duration) int64 { return now.Add(-d).Unix() }
|
stale := func(d time.Duration) int64 { return now.Add(-d).Unix() }
|
||||||
mib := func(n int64) int64 { return n * miB }
|
mib := func(n int64) int64 { return n * miB }
|
||||||
@@ -636,7 +674,8 @@ func TestHandler_evictPolicy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_evictForFreeSpace(t *testing.T) {
|
func TestHandler_makeRoom(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
free := func(n int64) func(string) (uint64, error) {
|
free := func(n int64) func(string) (uint64, error) {
|
||||||
return func(string) (uint64, error) { return uint64(n), nil }
|
return func(string) (uint64, error) { return uint64(n), nil }
|
||||||
}
|
}
|
||||||
@@ -644,14 +683,16 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
freeDisk func(string) (uint64, error)
|
freeDisk func(string) (uint64, error)
|
||||||
|
upload int64
|
||||||
kept []string
|
kept []string
|
||||||
}{
|
}{
|
||||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
|
{"ample free space evicts nothing", free(defaultMinFreeDisk), 0, []string{"oldest", "middle", "newest"}},
|
||||||
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
|
{"room for an upload sheds the oldest entry", free(defaultMinFreeDisk), 4 * miB, []string{"middle", "newest"}},
|
||||||
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
|
{"a shortfall the cache cannot cover sheds all of it", free(0), 0, nil},
|
||||||
{
|
{
|
||||||
"an unreadable volume is treated as unavailable, not as full",
|
"an unreadable volume is treated as unavailable, not as full",
|
||||||
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
||||||
|
0,
|
||||||
[]string{"oldest", "middle", "newest"},
|
[]string{"oldest", "middle", "newest"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
@@ -664,11 +705,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
handler := newTestHandler(t, Policy{}, entries...)
|
handler := newTestHandler(t, Policy{}, entries...)
|
||||||
handler.freeDisk = tc.freeDisk
|
handler.freeDisk = tc.freeDisk
|
||||||
|
handler.makeRoom(tc.upload)
|
||||||
db, err := handler.openDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
handler.evictForFreeSpace(db)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
||||||
})
|
})
|
||||||
@@ -676,6 +713,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
cache := &Cache{Key: "stuck", Complete: true, UsedAt: time.Now().Add(-(testRetention + time.Hour)).Unix()}
|
cache := &Cache{Key: "stuck", Complete: true, UsedAt: time.Now().Add(-(testRetention + time.Hour)).Unix()}
|
||||||
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
|
handler := newTestHandler(t, Policy{Retention: testRetention}, cache)
|
||||||
|
|
||||||
@@ -690,6 +728,7 @@ func TestHandler_SweepKeepsEntryWhenBlobSurvives(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
wanted := &Cache{Repo: testRepo, Key: "wanted", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-3 * time.Hour).Unix()}
|
wanted := &Cache{Repo: testRepo, Key: "wanted", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-3 * time.Hour).Unix()}
|
||||||
other := &Cache{Repo: testRepo, Key: "other", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-2 * time.Hour).Unix()}
|
other := &Cache{Repo: testRepo, Key: "other", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-2 * time.Hour).Unix()}
|
||||||
newest := &Cache{Repo: testRepo, Key: "newest", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
newest := &Cache{Repo: testRepo, Key: "newest", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
@@ -709,7 +748,67 @@ func TestHandler_FindProtectsFromEviction(t *testing.T) {
|
|||||||
assert.Equal(t, []string{wanted.Key}, keptKeys(t, handler, []*Cache{wanted, other}), "the entry just promised to a job survives, the next least recently used goes instead")
|
assert.Equal(t, []string{wanted.Key}, keptKeys(t, handler, []*Cache{wanted, other}), "the entry just promised to a job survives, the next least recently used goes instead")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandler_PrefixUsesNewestReservationWithinSecond(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
created := time.Now().Add(-time.Hour).Unix()
|
||||||
|
older := &Cache{Repo: testRepo, Key: "deps-old", Version: "v", Complete: true, CreatedAt: created}
|
||||||
|
newer := &Cache{Repo: testRepo, Key: "deps-new", Version: "v", Complete: true, CreatedAt: created}
|
||||||
|
handler := newTestHandler(t, Policy{}, older, newer)
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
cache, err := findCache(db, JobCredential{Repo: testRepo}, []string{"deps-"}, "v")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, newer, cache)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_RestoreRefs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
const main, release, pull = "refs/heads/main", "refs/heads/release", "refs/pull/1/head"
|
||||||
|
now := time.Now().Unix()
|
||||||
|
entry := func(ref, key string, age int64) *Cache {
|
||||||
|
return &Cache{Repo: testRepo, Ref: ref, Key: key, Version: "v", Complete: true, CreatedAt: now - 3600 + age, UsedAt: now}
|
||||||
|
}
|
||||||
|
exactOnMain := entry(main, "deps-exact", -1)
|
||||||
|
onRelease := entry(release, "deps-release", 0)
|
||||||
|
onMain := entry(main, "deps-main", 1)
|
||||||
|
onPull := entry(pull, "deps-pull", 2)
|
||||||
|
unscoped := entry("", "deps-unscoped", 3)
|
||||||
|
handler := newTestHandler(t, Policy{}, exactOnMain, onRelease, onMain, onPull, unscoped, entry("refs/pull/2/head", "deps-lost", 4))
|
||||||
|
for _, cache := range []*Cache{exactOnMain, onRelease, onMain, onPull, unscoped} {
|
||||||
|
writeBlob(t, handler, cache.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := handler.openDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
cred JobCredential
|
||||||
|
keys []string
|
||||||
|
want *Cache
|
||||||
|
}{
|
||||||
|
{"the default branch never restores what a pull request wrote", JobCredential{Ref: main, DefaultRef: main}, []string{"deps-"}, onMain},
|
||||||
|
{"a pull request restores its own entry first", JobCredential{Ref: pull, BaseRef: release, DefaultRef: main}, []string{"deps-"}, onPull},
|
||||||
|
{"a pull request falls back past a lost blob to its base branch before the default branch", JobCredential{Ref: "refs/pull/2/head", BaseRef: release, DefaultRef: main}, []string{"deps-"}, onRelease},
|
||||||
|
{"a branch does not restore a sibling branch", JobCredential{Ref: "refs/heads/other", DefaultRef: main}, []string{"deps-"}, onMain},
|
||||||
|
{"a prefix in the own ref wins over an exact key in the default branch", JobCredential{Ref: release, DefaultRef: main}, []string{"deps-exact", "deps-"}, onRelease},
|
||||||
|
{"a job with a ref never restores entries saved without one", JobCredential{Ref: "refs/heads/other"}, []string{"deps-"}, nil},
|
||||||
|
{"a job without a ref only restores entries saved without one", JobCredential{}, []string{"deps-"}, unscoped},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
tc.cred.Repo = testRepo
|
||||||
|
cache, err := handler.lookupCache(db, tc.cred, tc.keys, "v")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.want, cache)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandler_evictOnCommit(t *testing.T) {
|
func TestHandler_evictOnCommit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
handler := newTestHandler(t, Policy{RepoSizeLimit: 4 * miB}, full)
|
handler := newTestHandler(t, Policy{RepoSizeLimit: 4 * miB}, full)
|
||||||
|
|
||||||
@@ -719,6 +818,7 @@ func TestHandler_evictOnCommit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_gcCacheInterval(t *testing.T) {
|
func TestHandler_gcCacheInterval(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
cache := &Cache{Key: "temp", UsedAt: time.Now().Add(-time.Hour).Unix()}
|
cache := &Cache{Key: "temp", UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||||
handler := newTestHandler(t, Policy{SweepInterval: 30 * time.Minute}, cache)
|
handler := newTestHandler(t, Policy{SweepInterval: 30 * time.Minute}, cache)
|
||||||
|
|
||||||
@@ -728,6 +828,7 @@ func TestHandler_gcCacheInterval(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
fresh := &Cache{Repo: testRepo, Key: "fresh", Version: "v", Complete: true, UsedAt: time.Now().Unix()}
|
fresh := &Cache{Repo: testRepo, Key: "fresh", Version: "v", Complete: true, UsedAt: time.Now().Unix()}
|
||||||
pending := &Cache{Repo: testRepo, Key: "pending", Version: "v", UsedAt: time.Now().Add(-time.Second).Unix()}
|
pending := &Cache{Repo: testRepo, Key: "pending", Version: "v", UsedAt: time.Now().Add(-time.Second).Unix()}
|
||||||
handler := newTestHandler(t, Policy{}, fresh, pending)
|
handler := newTestHandler(t, Policy{}, fresh, pending)
|
||||||
@@ -786,6 +887,13 @@ func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
|
|||||||
return kept
|
return kept
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStorageDropsFailedWrites(t *testing.T) {
|
||||||
|
storage, err := NewStorage(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Error(t, storage.Write(1, 0, io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(io.ErrUnexpectedEOF))))
|
||||||
|
assert.NoFileExists(t, storage.tempName(1, 0))
|
||||||
|
}
|
||||||
|
|
||||||
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
||||||
@@ -794,6 +902,7 @@ func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_RejectsBearer(t *testing.T) {
|
func TestHandler_RejectsBearer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -830,6 +939,7 @@ func TestHandler_RejectsBearer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_UnregisterRevokes(t *testing.T) {
|
func TestHandler_UnregisterRevokes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
token string
|
token string
|
||||||
@@ -871,12 +981,14 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_CrossRepoIsolation(t *testing.T) {
|
func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer handler.Close()
|
defer handler.Close()
|
||||||
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
||||||
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
||||||
|
handler.RegisterJob("token-a-pull", JobCredential{Repo: "owner/repoA", Ref: "refs/pull/1/head"})
|
||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
key := "shared-key"
|
key := "shared-key"
|
||||||
@@ -911,16 +1023,24 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
|||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
for _, client := range []*http.Client{clientB, {Transport: &bearerTransport{token: "token-a-pull"}}} {
|
||||||
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
req.Header.Set("Content-Range", "bytes 0-5/*")
|
req.Header.Set("Content-Range", "bytes 0-5/*")
|
||||||
resp, err = clientB.Do(req)
|
resp, err = client.Do(req)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||||
|
|
||||||
|
resp, err = client.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp.Body.Close()
|
||||||
|
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ArtifactSignature(t *testing.T) {
|
func TestHandler_ArtifactSignature(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -952,6 +1072,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
|
|
||||||
first, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"})
|
first, err := StartHandler(Options{Dir: dir, OutboundIP: "127.0.0.1"})
|
||||||
@@ -968,6 +1089,7 @@ func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
insert := func(by *Handler, cache *Cache) {
|
insert := func(by *Handler, cache *Cache) {
|
||||||
db, err := by.openDB()
|
db, err := by.openDB()
|
||||||
@@ -985,6 +1107,7 @@ func TestHandler_SeesWritesOfOtherProcesses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1018,51 +1141,24 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
|||||||
assert.Equal(t, content, body)
|
assert.Equal(t, content, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_GC_PerRepoDedup(t *testing.T) {
|
func TestHandler_GC_DedupPerRepoAndRef(t *testing.T) {
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
t.Parallel()
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer handler.Close()
|
|
||||||
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"})
|
|
||||||
handler.RegisterJob("tok-b", JobCredential{Repo: "owner/repoB"})
|
|
||||||
|
|
||||||
key := "shared-dedup-key"
|
|
||||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
||||||
|
|
||||||
db, err := handler.openDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
||||||
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
|
entries := []*Cache{
|
||||||
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
|
{Repo: "owner/repoA", Key: "shared", Version: "v", Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1},
|
||||||
require.NoError(t, handler.insertCache(db, a))
|
{Repo: "owner/repoB", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||||
require.NoError(t, handler.insertCache(db, b))
|
{Repo: "owner/repoA", Ref: "refs/pull/1/head", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||||
require.NoError(t, handler.storage.Write(a.ID, 0, strings.NewReader("a")))
|
}
|
||||||
_, err = handler.storage.Commit(a.ID, 1)
|
handler := newTestHandler(t, Policy{}, entries...)
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, handler.storage.Write(b.ID, 0, strings.NewReader("b")))
|
|
||||||
_, err = handler.storage.Commit(b.ID, 1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
handler.gcAt = time.Time{}
|
handler.gcAt = time.Time{}
|
||||||
handler.gcCache()
|
handler.gcCache()
|
||||||
|
|
||||||
db, err = handler.openDB()
|
assert.Len(t, keptKeys(t, handler, entries), len(entries))
|
||||||
require.NoError(t, err)
|
|
||||||
defer db.Close()
|
|
||||||
after, err := db.find(func(cache *Cache) bool { return cache.Key == key && cache.Version == version })
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := make(map[string]bool)
|
|
||||||
for _, c := range after {
|
|
||||||
repos[c.Repo] = true
|
|
||||||
}
|
|
||||||
assert.True(t, repos["owner/repoA"], "repoA's cache must survive dedup against repoB")
|
|
||||||
assert.True(t, repos["owner/repoB"], "repoB's cache must survive dedup against repoA")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
handler, err := StartHandler(Options{Dir: dir})
|
handler, err := StartHandler(Options{Dir: dir})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1077,6 +1173,7 @@ func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||||
const secret = "internal-secret"
|
const secret = "internal-secret"
|
||||||
handler, err := StartHandler(Options{Dir: dir, InternalSecret: secret})
|
handler, err := StartHandler(Options{Dir: dir, InternalSecret: secret})
|
||||||
@@ -1122,10 +1219,10 @@ func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","ref":"refs/pull/1/head","base_ref":"refs/heads/release","default_ref":"refs/heads/main","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||||
cred, ok := handler.lookupCredential("via-internal-api")
|
cred, ok := handler.lookupCredential("via-internal-api")
|
||||||
require.True(t, ok)
|
require.True(t, ok)
|
||||||
assert.Equal(t, JobCredential{Repo: testRepo, PublicURL: "http://runner.example"}, cred)
|
assert.Equal(t, JobCredential{Repo: testRepo, Ref: "refs/pull/1/head", BaseRef: "refs/heads/release", DefaultRef: "refs/heads/main", PublicURL: "http://runner.example"}, cred)
|
||||||
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
||||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||||
// existing and silently never saved.
|
// existing and silently never saved.
|
||||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if existing != nil {
|
} else if existing != nil {
|
||||||
@@ -84,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||||
owner := hashedToken(bearerToken(r))
|
owner := hashedToken(bearerToken(r))
|
||||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||||
@@ -100,15 +100,17 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache := &Cache{
|
cache := &Cache{
|
||||||
Repo: cred.Repo,
|
Repo: cred.Repo,
|
||||||
|
Ref: cred.Ref,
|
||||||
Key: req.Key,
|
Key: req.Key,
|
||||||
Version: req.Version,
|
Version: req.Version,
|
||||||
Owner: owner,
|
Owner: owner,
|
||||||
Size: -1, // the size is only known at finalize time
|
Size: -1, // the size is only known at finalize time
|
||||||
CreatedAt: now,
|
CreatedAt: now.Unix(),
|
||||||
UsedAt: now,
|
CreatedAtNanos: now.Nanosecond(),
|
||||||
|
UsedAt: now.Unix(),
|
||||||
}
|
}
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -136,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||||
cache = nil // not the reservation this job made, so not this job's to commit
|
cache = nil // not the reservation this job made, so not this job's to commit
|
||||||
}
|
}
|
||||||
if err == nil && cache == nil {
|
if err == nil && cache == nil {
|
||||||
// A retry whose first response was lost finds the entry already committed.
|
// A retry whose first response was lost finds the entry already committed.
|
||||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -185,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
@@ -216,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.makeRoom(r.ContentLength)
|
||||||
|
|
||||||
query := r.URL.Query()
|
query := r.URL.Query()
|
||||||
switch strings.ToLower(query.Get("comp")) {
|
switch strings.ToLower(query.Get("comp")) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -78,11 +79,14 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||||
|
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||||
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||||
|
|
||||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
@@ -103,6 +107,7 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||||
@@ -150,6 +155,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||||
require.Equal(t, true, saved["ok"])
|
require.Equal(t, true, saved["ok"])
|
||||||
@@ -228,9 +234,28 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||||
|
clients := map[string]*http.Client{"x": testClient}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||||
|
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||||
|
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
require.Equal(t, true, created["ok"])
|
||||||
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||||
|
}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||||
|
}
|
||||||
|
for content, client := range clients {
|
||||||
|
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -267,6 +292,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
Key string `json:"key" `
|
Key string `json:"key" `
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
@@ -13,6 +15,7 @@ type Request struct {
|
|||||||
type Cache struct {
|
type Cache struct {
|
||||||
ID uint64 `json:"id"`
|
ID uint64 `json:"id"`
|
||||||
Repo string `json:"repo"`
|
Repo string `json:"repo"`
|
||||||
|
Ref string `json:"ref,omitzero"`
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
Size int64 `json:"cacheSize"`
|
Size int64 `json:"cacheSize"`
|
||||||
@@ -20,4 +23,9 @@ type Cache struct {
|
|||||||
Complete bool `json:"complete"`
|
Complete bool `json:"complete"`
|
||||||
UsedAt int64 `json:"usedAt"`
|
UsedAt int64 `json:"usedAt"`
|
||||||
CreatedAt int64 `json:"createdAt"`
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cache *Cache) createdAtNano() int64 {
|
||||||
|
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||||
}
|
}
|
||||||
@@ -67,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if cred.InsecureTLS {
|
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||||
proxy.Transport = insecureTransport
|
proxy.Transport = insecureTransport
|
||||||
}
|
}
|
||||||
proxy.ServeHTTP(w, r)
|
proxy.ServeHTTP(w, r)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFrontResultsService(t *testing.T) {
|
func TestFrontResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var gotHost, gotPath, gotProto string
|
var gotHost, gotPath, gotProto string
|
||||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||||
@@ -132,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||||
|
defer gitea.Close()
|
||||||
|
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer handler.Close()
|
||||||
|
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -19,6 +20,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var objects sync.Map
|
var objects sync.Map
|
||||||
var uploads atomic.Int32
|
var uploads atomic.Int32
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
@@ -51,14 +53,33 @@ func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
|||||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
assert.EqualValues(t, 1, uploads.Load())
|
assert.EqualValues(t, 1, uploads.Load())
|
||||||
|
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
cache, err := saver.s3.fetch(t.Context(), id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||||
require.Equal(t, true, found["ok"])
|
require.Equal(t, true, found["ok"])
|
||||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
|
for _, cache := range []*Cache{
|
||||||
|
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||||
|
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||||
|
} {
|
||||||
|
require.NoError(t, saver.s3.publish(cache))
|
||||||
|
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||||
|
}
|
||||||
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
|
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||||
|
require.Equal(t, true, found["ok"])
|
||||||
|
assert.Equal(t, "deps-new", found["matched_key"])
|
||||||
|
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
objects.Clear()
|
objects.Clear()
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||||
@@ -71,6 +92,7 @@ func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
for _, failingPart := range []string{"", "2"} {
|
for _, failingPart := range []string{"", "2"} {
|
||||||
var parts, aborts atomic.Int32
|
var parts, aborts atomic.Int32
|
||||||
var completion struct {
|
var completion struct {
|
||||||
@@ -110,6 +132,7 @@ func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -51,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(file, reader)
|
_, err = io.Copy(file, reader)
|
||||||
|
if err = errors.Join(err, file.Close()); err != nil {
|
||||||
|
_ = os.Remove(name)
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,50 +104,46 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written, err := assemble(name, tempNames)
|
written, err := assemble(name, tempNames)
|
||||||
if err != nil {
|
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||||
return 0, err
|
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||||
}
|
}
|
||||||
// If size is less than 0, it means the size is unknown.
|
if err != nil {
|
||||||
// We can't check the size of the file, just skip the check.
|
|
||||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
|
||||||
if size >= 0 && written != size {
|
|
||||||
_ = os.Remove(name)
|
_ = os.Remove(name)
|
||||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
return 0, err
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
|
||||||
func assemble(name string, tempNames []string) (int64, error) {
|
func assemble(name string, tempNames []string) (int64, error) {
|
||||||
if len(tempNames) == 1 {
|
if len(tempNames) == 0 {
|
||||||
info, err := os.Stat(tempNames[0])
|
return 0, errors.New("nothing was uploaded")
|
||||||
if err != nil {
|
}
|
||||||
|
if err := os.Rename(tempNames[0], name); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
return info.Size(), os.Rename(tempNames[0], name)
|
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Create(name)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
for _, v := range tempNames[1:] {
|
||||||
var written int64
|
part, err := os.Open(v)
|
||||||
for _, v := range tempNames {
|
|
||||||
f, err := os.Open(v)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
n, err := io.Copy(file, f)
|
_, err = io.Copy(file, part)
|
||||||
_ = f.Close()
|
_ = part.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written += n
|
_ = os.Remove(v)
|
||||||
}
|
}
|
||||||
return written, nil
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return info.Size(), file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||||
|
|||||||
+21
-7
@@ -13,6 +13,7 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -84,16 +85,25 @@ func uploads(router *http.ServeMux, baseDir string) {
|
|||||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
var start int64
|
||||||
appendUpload := req.Header.Get("Content-Range")
|
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if start == 0 {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
}
|
}
|
||||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
if req.Body == nil {
|
if req.Body == nil {
|
||||||
panic(errors.New("no body given"))
|
panic(errors.New("no body given"))
|
||||||
}
|
}
|
||||||
@@ -120,7 +130,7 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
runID := req.PathValue("runId")
|
runID := req.PathValue("runId")
|
||||||
|
|
||||||
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,6 +155,9 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
|
|
||||||
var files []ContainerItem
|
var files []ContainerItem
|
||||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !entry.IsDir() {
|
if !entry.IsDir() {
|
||||||
rel, err := filepath.Rel(safePath, path)
|
rel, err := filepath.Rel(safePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -157,12 +170,12 @@ func downloads(router *http.ServeMux, baseDir string) {
|
|||||||
files = append(files, ContainerItem{
|
files = append(files, ContainerItem{
|
||||||
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||||
ItemType: "file",
|
ItemType: "file",
|
||||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, filepath.ToSlash(rel)),
|
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: net.JoinHostPort(addr, port),
|
Addr: net.JoinHostPort(addr, port),
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
router := http.NewServeMux()
|
router := http.NewServeMux()
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
server := httptest.NewServer(router)
|
server := httptest.NewServer(router)
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -48,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
return resp.StatusCode, data
|
return resp.StatusCode, data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("no-artifacts", func(t *testing.T) {
|
||||||
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
var list NamedFileContainerResourceURLResponse
|
||||||
|
require.NoError(t, json.Unmarshal(data, &list))
|
||||||
|
require.Zero(t, list.Count)
|
||||||
|
require.Empty(t, list.Value)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("upload-and-download", func(t *testing.T) {
|
t.Run("upload-and-download", func(t *testing.T) {
|
||||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||||
|
|
||||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
@@ -81,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(data, &items))
|
require.NoError(t, json.Unmarshal(data, &items))
|
||||||
require.Len(t, items.Value, 1)
|
require.Len(t, items.Value, 1)
|
||||||
require.Equal(t, "file", items.Value[0].ItemType)
|
require.Equal(t, "file", items.Value[0].ItemType)
|
||||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
require.Equal(t, item, items.Value[0].Path)
|
||||||
|
|
||||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(stored))
|
require.Equal(t, content, string(stored))
|
||||||
})
|
})
|
||||||
@@ -95,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
t.Run("content-range", func(t *testing.T) {
|
t.Run("content-range", func(t *testing.T) {
|
||||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "first-second", string(stored))
|
require.Equal(t, "first-second", string(stored))
|
||||||
|
|
||||||
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||||
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, stored)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||||
@@ -156,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("v4", func(t *testing.T) {
|
||||||
|
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||||
|
call := func(method, body string) (int, map[string]any) {
|
||||||
|
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||||
|
var out map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||||
|
return status, out
|
||||||
|
}
|
||||||
|
put := func(rawURL, body string) int {
|
||||||
|
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
download := func() (string, http.Header) {
|
||||||
|
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, signed)
|
||||||
|
resp, err := client.Get(signed["signedUrl"].(string))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(body), resp.Header
|
||||||
|
}
|
||||||
|
|
||||||
|
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
upload := created["signedUploadUrl"].(string)
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||||
|
for attempt := range 2 {
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusBadRequest, status)
|
||||||
|
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status, finalized)
|
||||||
|
id := finalized["artifactId"].(string)
|
||||||
|
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
require.Equal(t, id, refinalized["artifactId"])
|
||||||
|
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusConflict, status)
|
||||||
|
require.Equal(t, "already_exists", conflict["code"])
|
||||||
|
|
||||||
|
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
artifacts := listed["artifacts"].([]any)
|
||||||
|
require.Len(t, artifacts, 1)
|
||||||
|
artifact := artifacts[0].(map[string]any)
|
||||||
|
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||||
|
delete(artifact, "createdAt")
|
||||||
|
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||||
|
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||||
|
require.Empty(t, other["artifacts"])
|
||||||
|
|
||||||
|
body, header := download()
|
||||||
|
require.Equal(t, "hello world", body)
|
||||||
|
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusNotFound, status)
|
||||||
|
require.Equal(t, "not_found", missing["code"])
|
||||||
|
|
||||||
|
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||||
|
require.Equal(t, http.StatusCreated, status)
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
body, header = download()
|
||||||
|
require.Equal(t, "gz", body)
|
||||||
|
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
rerun := http.NewServeMux()
|
||||||
|
registerV4(rerun, artifactPath)
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||||
|
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSafeResolve(t *testing.T) {
|
func TestSafeResolve(t *testing.T) {
|
||||||
@@ -176,7 +281,7 @@ func TestSafeResolve(t *testing.T) {
|
|||||||
|
|
||||||
for name, tc := range tests {
|
for name, tc := range tests {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||||
|
dir string
|
||||||
|
mu sync.Mutex
|
||||||
|
lastID int64
|
||||||
|
created map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
zipMimeType = "application/zip"
|
||||||
|
maxTwirpBytes = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerV4(router *http.ServeMux, baseDir string) {
|
||||||
|
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||||
|
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||||
|
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
type twirpError struct {
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *twirpError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
func notFound(name string) error {
|
||||||
|
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(format string, args ...any) error {
|
||||||
|
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req Req
|
||||||
|
var resp Resp
|
||||||
|
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||||
|
if err != nil {
|
||||||
|
err = badRequest("malformed request: %v", err)
|
||||||
|
} else {
|
||||||
|
resp, err = handle(r, req)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err != nil {
|
||||||
|
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||||
|
errors.As(err, &terr)
|
||||||
|
w.WriteHeader(terr.status)
|
||||||
|
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.MarshalWrite(w, resp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||||
|
sum := sha256.Sum256([]byte(name))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) runDir(runID string) string {
|
||||||
|
return filepath.Join(s.dir, fileName(runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) artifactDir(runID, name string) string {
|
||||||
|
return filepath.Join(s.runDir(runID), fileName(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) urlDir(r *http.Request) string {
|
||||||
|
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMeta(dir string) (*monolithArtifact, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var meta monolithArtifact
|
||||||
|
return &meta, json.Unmarshal(data, &meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||||
|
data, err := json.Marshal(meta)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||||
|
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.created[dir] {
|
||||||
|
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.lastID++
|
||||||
|
if err := writeMeta(dir, &monolithArtifact{
|
||||||
|
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||||
|
DatabaseID: s.lastID,
|
||||||
|
Name: req.Name,
|
||||||
|
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||||
|
}); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.created[dir] = true
|
||||||
|
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockList struct {
|
||||||
|
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||||
|
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||||
|
var err error
|
||||||
|
switch comp := r.URL.Query().Get("comp"); comp {
|
||||||
|
case "block":
|
||||||
|
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||||
|
case "blocklist":
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||||
|
case "": // Put Blob, which docker/build-push-action uses
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||||
|
_, err := io.Copy(out, r.Body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(name string, body io.Reader) error {
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, body)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||||
|
out, err := os.CreateTemp(dir, "blob")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||||
|
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(out.Name()))
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||||
|
var list blockList
|
||||||
|
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range list.Blocks {
|
||||||
|
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendBlock(out io.Writer, name string) error {
|
||||||
|
in, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
} else if meta.Digest != "" {
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
size, err := io.Copy(hash, blob)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if size != req.Size {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||||
|
}
|
||||||
|
if req.Hash != "" && req.Hash != digest {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||||
|
}
|
||||||
|
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
meta.Size, meta.Digest = size, digest
|
||||||
|
if err := writeMeta(dir, meta); err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||||
|
run := s.runDir(req.WorkflowRunBackendID)
|
||||||
|
entries, err := os.ReadDir(run)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return listArtifactsResponse{}, err
|
||||||
|
}
|
||||||
|
var resp listArtifactsResponse
|
||||||
|
for _, entry := range entries {
|
||||||
|
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||||
|
if err != nil || meta.Digest == "" ||
|
||||||
|
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||||
|
}
|
||||||
|
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||||
|
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||||
|
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil || meta.Digest == "" {
|
||||||
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||||
|
contentType, filename := string(data), meta.Name
|
||||||
|
if !strings.Contains(contentType, "/") {
|
||||||
|
contentType = zipMimeType
|
||||||
|
}
|
||||||
|
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||||
|
filename += ".zip"
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||||
|
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return deleteArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
delete(s.created, dir)
|
||||||
|
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||||
|
now := time.Now()
|
||||||
|
claims, _ := json.Marshal(map[string]any{
|
||||||
|
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||||
|
"ac": `[{"Scope":"","Permission":3}]`,
|
||||||
|
"nbf": now.Unix(),
|
||||||
|
"exp": now.Add(lifetime).Unix(),
|
||||||
|
"jti": rand.Text(),
|
||||||
|
})
|
||||||
|
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
|
||||||
|
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||||
|
createArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
SignedUploadURL string `json:"signedUploadUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeArtifactRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
Size int64 `json:",string"`
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
finalizeArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
ArtifactID int64 `json:"artifactId,string"`
|
||||||
|
}
|
||||||
|
|
||||||
|
listArtifactsRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
NameFilter string
|
||||||
|
IDFilter int64 `json:",string"`
|
||||||
|
}
|
||||||
|
listArtifactsResponse struct {
|
||||||
|
Artifacts []monolithArtifact `json:"artifacts"`
|
||||||
|
}
|
||||||
|
monolithArtifact struct {
|
||||||
|
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||||
|
DatabaseID int64 `json:"databaseId,string"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size int64 `json:"size,string"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||||
|
Digest string `json:"digest"` // empty until the upload is finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
getSignedArtifactURLRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
getSignedArtifactURLResponse struct {
|
||||||
|
SignedURL string `json:"signedUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
createArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactResponse = finalizeArtifactResponse
|
||||||
|
)
|
||||||
+28
-1
@@ -7,7 +7,9 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/rand/v2"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
@@ -158,13 +160,38 @@ func (e Executor) IfBool(conditional bool) Executor {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||||
|
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||||
|
|
||||||
|
// Retry runs the executor up to attempts times until it succeeds.
|
||||||
|
func (e Executor) Retry(attempts int) Executor {
|
||||||
|
return func(ctx context.Context) error {
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
err := e(ctx)
|
||||||
|
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backoff := RetryBackoff()
|
||||||
|
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return err
|
||||||
|
case <-time.After(backoff):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Finally adds an executor to run after other executor
|
// Finally adds an executor to run after other executor
|
||||||
func (e Executor) Finally(finally Executor) Executor {
|
func (e Executor) Finally(finally Executor) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
err := e(ctx)
|
err := e(ctx)
|
||||||
err2 := finally(ctx)
|
err2 := finally(ctx)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,9 +7,11 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -133,10 +135,39 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||||
|
backoff := RetryBackoff
|
||||||
|
RetryBackoff = func() time.Duration { return 0 }
|
||||||
|
t.Cleanup(func() { RetryBackoff = backoff })
|
||||||
|
transient := errors.New("transient")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
failures, calls int
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{failures: 2, calls: 3},
|
||||||
|
{failures: 5, calls: 3, err: transient},
|
||||||
|
} {
|
||||||
|
var calls int
|
||||||
|
err := Executor(func(context.Context) error {
|
||||||
|
calls++
|
||||||
|
if calls <= tc.failures {
|
||||||
|
return transient
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}).Retry(3)(t.Context())
|
||||||
|
require.Equal(t, tc.err, err)
|
||||||
|
require.Equal(t, tc.calls, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
mainErr := errors.New("main failed")
|
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||||
require.EqualError(t, err, "error occurred running finally: cleanup failed (original error: main failed)")
|
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||||
|
require.ErrorIs(t, err, finalErr)
|
||||||
|
if mainErr != nil {
|
||||||
|
require.ErrorIs(t, err, mainErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+246
-16
@@ -8,7 +8,9 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
"net"
|
"net"
|
||||||
@@ -28,6 +30,17 @@ import (
|
|||||||
|
|
||||||
var actionFetches sync.Map
|
var actionFetches sync.Map
|
||||||
|
|
||||||
|
var symlink = (*os.Root).Symlink
|
||||||
|
|
||||||
|
var symlinkFallback = runtime.GOOS == "windows"
|
||||||
|
|
||||||
|
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||||
|
|
||||||
|
type treeFile struct {
|
||||||
|
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
type actionFetch struct {
|
type actionFetch struct {
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
dir, sha string
|
dir, sha string
|
||||||
@@ -86,9 +99,14 @@ func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, s
|
|||||||
record := filepath.Join(input.Dir, "ref")
|
record := filepath.Join(input.Dir, "ref")
|
||||||
data, _ := os.ReadFile(record)
|
data, _ := os.ReadFile(record)
|
||||||
sha, etag, _ := strings.Cut(string(data), "\n")
|
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||||
if !isFullObjectID(sha) || !exists(tree(sha)) {
|
switch {
|
||||||
|
case !isFullObjectID(sha):
|
||||||
sha, etag = "", ""
|
sha, etag = "", ""
|
||||||
} else if input.OfflineMode || isFullObjectID(input.Ref) {
|
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||||
|
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||||
|
case !treeIntact(input.Dir, sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||||
return tree(sha), sha, nil
|
return tree(sha), sha, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,7 +191,7 @@ func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, toke
|
|||||||
newEtag := response.Header.Get("ETag")
|
newEtag := response.Header.Get("ETag")
|
||||||
gz, err := gzip.NewReader(response.Body)
|
gz, err := gzip.NewReader(response.Body)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||||
}
|
}
|
||||||
var netErr net.Error
|
var netErr net.Error
|
||||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||||
@@ -191,18 +209,18 @@ func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, e
|
|||||||
clone := input
|
clone := input
|
||||||
clone.Dir = filepath.Join(input.Dir, "git")
|
clone.Dir = filepath.Join(input.Dir, "git")
|
||||||
sha, _, err := resolveClone(ctx, clone)
|
sha, _, err := resolveClone(ctx, clone)
|
||||||
if err != nil || exists(filepath.Join(input.Dir, sha)) {
|
if err != nil || treeIntact(input.Dir, sha) {
|
||||||
return sha, err
|
return sha, err
|
||||||
}
|
}
|
||||||
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
|
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||||
return sha, err
|
return sha, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||||
temp, err := os.MkdirTemp(dir, "tmp-")
|
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -215,6 +233,9 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
defer root.Close()
|
defer root.Close()
|
||||||
|
|
||||||
sha := ""
|
sha := ""
|
||||||
|
sizes := map[string]int64{}
|
||||||
|
links := map[string]string{}
|
||||||
|
var deferred []string
|
||||||
reader := tar.NewReader(archive)
|
reader := tar.NewReader(archive)
|
||||||
for {
|
for {
|
||||||
header, err := reader.Next()
|
header, err := reader.Next()
|
||||||
@@ -225,7 +246,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
|
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||||
return sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -241,11 +262,10 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
case tar.TypeDir:
|
case tar.TypeDir:
|
||||||
err = root.MkdirAll(name, 0o755)
|
err = root.MkdirAll(name, 0o755)
|
||||||
case tar.TypeReg:
|
case tar.TypeReg:
|
||||||
err = writeTreeFile(root, name, header.Mode, reader)
|
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||||
case tar.TypeSymlink:
|
case tar.TypeSymlink:
|
||||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
links[filepath.Clean(name)] = header.Linkname
|
||||||
err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
deferred = append(deferred, filepath.Clean(name))
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -254,23 +274,233 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
|||||||
if !isFullObjectID(sha) {
|
if !isFullObjectID(sha) {
|
||||||
return "", errors.New("the archive names no commit")
|
return "", errors.New("the archive names no commit")
|
||||||
}
|
}
|
||||||
|
linksByParent := map[string][]string{}
|
||||||
|
if symlinkFallback {
|
||||||
|
for name := range links {
|
||||||
|
linksByParent[filepath.Dir(name)] = append(linksByParent[filepath.Dir(name)], filepath.Base(name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for requireTarget := symlinkFallback; len(deferred) > 0; {
|
||||||
|
unresolved := deferred[:0]
|
||||||
|
for _, name := range deferred {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if requireTarget {
|
||||||
|
if _, err := root.Stat(filepath.Dir(name) + string(filepath.Separator) + links[name]); err != nil {
|
||||||
|
unresolved = append(unresolved, name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := symlink(root, links[name], name); err == nil || !symlinkFallback {
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := copyTreeLink(ctx, root, name, links, linksByParent, sizes); err != nil {
|
||||||
|
return "", fmt.Errorf("materialize symlink %q: %w", filepath.ToSlash(name), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unresolved) == len(deferred) {
|
||||||
|
requireTarget = false
|
||||||
|
}
|
||||||
|
deferred = unresolved
|
||||||
|
}
|
||||||
if err := root.Close(); err != nil {
|
if err := root.Close(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return sha, os.Rename(temp, filepath.Join(dir, sha))
|
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
|
func resolveTreeLink(name string, links map[string]string) (string, bool) {
|
||||||
|
if !filepath.IsLocal(name) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
parts := strings.Split(name, string(filepath.Separator))
|
||||||
|
var resolved []string
|
||||||
|
for count := 0; len(parts) > 0; {
|
||||||
|
part := parts[0]
|
||||||
|
parts = parts[1:]
|
||||||
|
switch part {
|
||||||
|
case "", ".":
|
||||||
|
continue
|
||||||
|
case "..":
|
||||||
|
if len(resolved) == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
resolved = resolved[:len(resolved)-1]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target, ok := links[filepath.Join(append(resolved, part)...)]
|
||||||
|
if !ok {
|
||||||
|
resolved = append(resolved, part)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
if target == "" || os.IsPathSeparator(target[0]) || filepath.VolumeName(target) != "" || count > 255 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
parts = append(strings.Split(filepath.FromSlash(target), string(filepath.Separator)), parts...)
|
||||||
|
}
|
||||||
|
return filepath.Join(append([]string{"."}, resolved...)...), true
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errTreeLinkTarget = errors.New("unmaterializable symlink target")
|
||||||
|
errTreeLinkCycle = fmt.Errorf("cyclic %w", errTreeLinkTarget)
|
||||||
|
)
|
||||||
|
|
||||||
|
func copyTreeLink(ctx context.Context, root *os.Root, name string, links map[string]string, linksByParent map[string][]string, sizes map[string]int64) error {
|
||||||
|
writeLinkText := func(destination, target string) error { // as git writes links it cannot create
|
||||||
|
if err := root.RemoveAll(destination); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for written := range sizes {
|
||||||
|
if written == destination || strings.HasPrefix(written, destination+string(filepath.Separator)) {
|
||||||
|
delete(sizes, written)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
size, err := writeTreeFile(root, destination, 0o644, strings.NewReader(target))
|
||||||
|
sizes[destination] = size
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
active := map[string]bool{}
|
||||||
|
var copyTarget func(string, string) error
|
||||||
|
copyTarget = func(source, destination string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source, ok := resolveTreeLink(source, links)
|
||||||
|
if !ok {
|
||||||
|
return errTreeLinkTarget
|
||||||
|
}
|
||||||
|
file, err := root.Open(source)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return errTreeLinkTarget
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode().IsRegular() {
|
||||||
|
sizes[destination], err = writeTreeFile(root, destination, int64(info.Mode().Perm()), file)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return errTreeLinkTarget
|
||||||
|
}
|
||||||
|
if active[source] {
|
||||||
|
return errTreeLinkCycle
|
||||||
|
}
|
||||||
|
active[source] = true
|
||||||
|
defer delete(active, source)
|
||||||
|
entries, err := file.ReadDir(-1)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
children := map[string]bool{}
|
||||||
|
for _, entry := range entries {
|
||||||
|
children[entry.Name()] = true
|
||||||
|
}
|
||||||
|
for _, child := range linksByParent[source] {
|
||||||
|
children[child] = true
|
||||||
|
}
|
||||||
|
if err := root.MkdirAll(destination, info.Mode().Perm()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for child := range children {
|
||||||
|
childSource, childDestination := filepath.Join(source, child), filepath.Join(destination, child)
|
||||||
|
err := copyTarget(childSource, childDestination)
|
||||||
|
if target, isLink := links[childSource]; isLink && errors.Is(err, errTreeLinkTarget) && !errors.Is(err, errTreeLinkCycle) {
|
||||||
|
err = writeLinkText(childDestination, target)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := copyTarget(name, name); !errors.Is(err, errTreeLinkTarget) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeLinkText(name, links[name])
|
||||||
|
}
|
||||||
|
|
||||||
|
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||||
|
|
||||||
|
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||||
|
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||||
|
files := make([]treeFile, 0, len(sizes))
|
||||||
|
for name, size := range sizes {
|
||||||
|
files = append(files, treeFile{[]byte(name), size})
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tree := filepath.Join(dir, sha)
|
||||||
|
if !exists(tree) {
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||||
|
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||||
|
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||||
|
func treeIntact(dir, sha string) bool {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||||
|
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var files []treeFile
|
||||||
|
if json.Unmarshal(data, &files) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||||
perm := os.FileMode(0o644)
|
perm := os.FileMode(0o644)
|
||||||
if mode&0o111 != 0 {
|
if mode&0o111 != 0 {
|
||||||
perm = 0o755
|
perm = 0o755
|
||||||
}
|
}
|
||||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, err
|
||||||
}
|
}
|
||||||
_, err = io.Copy(file, content)
|
n, err := io.Copy(file, content)
|
||||||
return errors.Join(err, file.Close())
|
return n, errors.Join(err, file.Close())
|
||||||
}
|
}
|
||||||
|
|
||||||
func exists(name string) bool {
|
func exists(name string) bool {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/cgi"
|
"net/http/cgi"
|
||||||
@@ -15,6 +16,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -39,8 +41,10 @@ func TestFetchAction(t *testing.T) {
|
|||||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "update-index", "--chmod=+x", "run.sh"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||||
|
expectedSha := gitRevParse(t, remote, "main")
|
||||||
|
|
||||||
gitPath, err := exec.LookPath("git")
|
gitPath, err := exec.LookPath("git")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -48,6 +52,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
var archiveRequests []http.Header
|
var archiveRequests []http.Header
|
||||||
var mirror *httptest.Server
|
var mirror *httptest.Server
|
||||||
|
archiveCache := map[string][]byte{}
|
||||||
requests, archives, failures := 0, "", 0
|
requests, archives, failures := 0, "", 0
|
||||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
@@ -84,7 +89,12 @@ func TestFetchAction(t *testing.T) {
|
|||||||
if archives != "unprefixed" {
|
if archives != "unprefixed" {
|
||||||
args = append(args, "--prefix=action/")
|
args = append(args, "--prefix=action/")
|
||||||
}
|
}
|
||||||
archive, _ := runGit(request.Context(), remote, nil, args...)
|
archiveKey := strings.Join(args, "\x00")
|
||||||
|
archive, ok := archiveCache[archiveKey]
|
||||||
|
if !ok {
|
||||||
|
archive, _ = runGit(request.Context(), remote, nil, args...)
|
||||||
|
archiveCache[archiveKey] = archive
|
||||||
|
}
|
||||||
writer.Header().Set("ETag", etag)
|
writer.Header().Set("ETag", etag)
|
||||||
_, _ = writer.Write(archive)
|
_, _ = writer.Write(archive)
|
||||||
})
|
})
|
||||||
@@ -107,7 +117,6 @@ func TestFetchAction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||||
expectedSha := gitRevParse(t, remote, "main")
|
|
||||||
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||||
t.Run(scenario, func(t *testing.T) {
|
t.Run(scenario, func(t *testing.T) {
|
||||||
serve("prefixed", 0)
|
serve("prefixed", 0)
|
||||||
@@ -180,12 +189,14 @@ func TestFetchAction(t *testing.T) {
|
|||||||
input := newInput("v1")
|
input := newInput("v1")
|
||||||
tree, sha, err := FetchAction(t.Context(), input)
|
tree, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "v1"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
if runtime.GOOS != "windows" {
|
||||||
|
assert.EqualValues(t, 0o755, info.Mode().Perm())
|
||||||
|
}
|
||||||
link, err := os.Readlink(filepath.Join(tree, "link"))
|
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "action.yml", link)
|
assert.Equal(t, "action.yml", link)
|
||||||
@@ -196,9 +207,10 @@ func TestFetchAction(t *testing.T) {
|
|||||||
|
|
||||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||||
|
expectedSha = gitRevParse(t, remote, "v1")
|
||||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "v1"), movedSha)
|
assert.Equal(t, expectedSha, movedSha)
|
||||||
assert.NotEqual(t, tree, moved)
|
assert.NotEqual(t, tree, moved)
|
||||||
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
|
||||||
@@ -206,10 +218,17 @@ func TestFetchAction(t *testing.T) {
|
|||||||
_, offlineSha, err := FetchAction(t.Context(), input)
|
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, movedSha, offlineSha)
|
assert.Equal(t, movedSha, offlineSha)
|
||||||
|
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||||
|
reused, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, moved, reused)
|
||||||
|
|
||||||
requests, headers := served()
|
requests, headers := served()
|
||||||
assert.Equal(t, 3, requests)
|
assert.Equal(t, 4, requests)
|
||||||
require.Len(t, headers, 3)
|
require.Len(t, headers, 4)
|
||||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||||
assert.Empty(t, headers[0].Get("If-None-Match"))
|
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||||
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||||
@@ -218,7 +237,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||||
serve("prefixed", 0)
|
serve("prefixed", 0)
|
||||||
sha := gitRevParse(t, remote, "main")
|
sha := expectedSha
|
||||||
input := newInput(sha)
|
input := newInput(sha)
|
||||||
for range 2 {
|
for range 2 {
|
||||||
tree, got, err := FetchAction(t.Context(), input)
|
tree, got, err := FetchAction(t.Context(), input)
|
||||||
@@ -233,12 +252,74 @@ func TestFetchAction(t *testing.T) {
|
|||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||||
|
sha := gitRevParse(t, remote, "main")
|
||||||
|
for _, mode := range []string{"prefixed", "none"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
for _, damage := range []func() error{
|
||||||
|
func() error { return os.Truncate(yml, 4) },
|
||||||
|
func() error { return os.Remove(tree + treeSizes) },
|
||||||
|
} {
|
||||||
|
require.NoError(t, damage(), mode)
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, "name: action\n", string(body), mode)
|
||||||
|
}
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 3, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.Truncate(yml, 4))
|
||||||
|
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||||
|
require.NoError(t, err)
|
||||||
|
release := sync.OnceFunc(unlock)
|
||||||
|
defer release()
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||||
|
close(waiting)
|
||||||
|
return AcquireCloneLock(ctx, dir)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||||
|
case <-waiting:
|
||||||
|
}
|
||||||
|
info, err := os.Stat(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||||
|
release()
|
||||||
|
require.NoError(t, <-done)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "name: action\n", string(body))
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||||
serve("redirect", 2)
|
serve("redirect", 2)
|
||||||
input := newInput("main")
|
input := newInput("main")
|
||||||
_, sha, err := FetchAction(t.Context(), input)
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
require.Len(t, headers, 4)
|
require.Len(t, headers, 4)
|
||||||
@@ -251,7 +332,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
serve(mode, 0)
|
serve(mode, 0)
|
||||||
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||||
require.NoError(t, err, mode)
|
require.NoError(t, err, mode)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha, mode)
|
assert.Equal(t, expectedSha, sha, mode)
|
||||||
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||||
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
@@ -266,7 +347,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
input.URL = "https://example.invalid/org/action"
|
input.URL = "https://example.invalid/org/action"
|
||||||
_, sha, err := FetchAction(t.Context(), input)
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
assert.Equal(t, expectedSha, sha)
|
||||||
_, headers := served()
|
_, headers := served()
|
||||||
assert.Len(t, headers, 1)
|
assert.Len(t, headers, 1)
|
||||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
@@ -306,10 +387,176 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
|||||||
require.NoError(t, writer.Close())
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
_, err := extractTree(&archive, dir, false)
|
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||||
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractTreeCopiesDeferredLinks(t *testing.T) {
|
||||||
|
originalSymlink, originalFallback := symlink, symlinkFallback
|
||||||
|
symlink = func(_root *os.Root, _target, _name string) error { return errors.New("symlink privilege unavailable") }
|
||||||
|
symlinkFallback = true
|
||||||
|
t.Cleanup(func() { symlink, symlinkFallback = originalSymlink, originalFallback })
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
links [][2]string
|
||||||
|
files []string
|
||||||
|
text bool
|
||||||
|
}{
|
||||||
|
{name: "file chain", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "dist/index.js"}}, files: []string{"index.js", "entry.js"}},
|
||||||
|
{name: "directory chain", links: [][2]string{{"lib", "bundle"}, {"bundle", "dist"}, {"dist/entry.js", "index.js"}}, files: []string{"lib/index.js", "lib/entry.js", "lib/nested/index.js", "bundle/index.js", "bundle/entry.js", "bundle/nested/index.js"}},
|
||||||
|
{name: "directory in target path", links: [][2]string{{"index.js", "bundle/nested/index.js"}, {"bundle", "dist"}}, files: []string{"index.js", "bundle/nested/index.js"}},
|
||||||
|
{name: "parent after directory link", links: [][2]string{{"index.js", "bundle/../index.js"}, {"bundle", "dist/nested"}}, files: []string{"index.js", "bundle/index.js"}},
|
||||||
|
{name: "outside", links: [][2]string{{"index.js", "../outside.js"}}, text: true},
|
||||||
|
{name: "absolute", links: [][2]string{{"index.js", t.TempDir()}}, text: true},
|
||||||
|
{name: "dangling", links: [][2]string{{"index.js", "missing.js"}}, text: true},
|
||||||
|
{name: "dangling child of directory link", links: [][2]string{{"lib", "dist"}, {"dist/optional", "missing.js"}}, files: []string{"lib/index.js", "lib/nested/index.js"}},
|
||||||
|
{name: "cycle", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "index.js"}}, text: true},
|
||||||
|
{name: "directory cycle", links: [][2]string{{"dist/loop", "."}}, text: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
sha := strings.Repeat("a", 40)
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist/nested", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||||
|
for _, link := range test.links {
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: link[0], Typeflag: tar.TypeSymlink, Linkname: link[1]}))
|
||||||
|
}
|
||||||
|
content := "console.log('action');\n"
|
||||||
|
for _, name := range []string{"dist/index.js", "dist/nested/index.js"} {
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(content))}))
|
||||||
|
_, err := writer.Write([]byte(content))
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, treeIntact(dir, sha))
|
||||||
|
if test.text {
|
||||||
|
for _, link := range test.links {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha, link[0]))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, link[1], string(data))
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, link[0]))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, info.Mode().IsRegular())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range test.files {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha, name))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, content, string(data))
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, name))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, info.Mode().IsRegular())
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||||
|
}
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), nil, 0o644))
|
||||||
|
assert.False(t, treeIntact(dir, sha), name)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), []byte(content), 0o755))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||||
|
}
|
||||||
|
sha := strings.Repeat("a", 40)
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||||
|
_, err := writer.Write([]byte("x"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||||
|
assert.True(t, treeIntact(dir, sha))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractTreeCreatesLinksInPlatformOrder(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("requires symlink privileges")
|
||||||
|
}
|
||||||
|
original, originalFallback := symlink, symlinkFallback
|
||||||
|
var created []string
|
||||||
|
symlink = func(root *os.Root, target, name string) error {
|
||||||
|
created = append(created, name)
|
||||||
|
return root.Symlink(target, name)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { symlink, symlinkFallback = original, originalFallback })
|
||||||
|
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||||
|
for _, entry := range []*tar.Header{
|
||||||
|
{Name: "a", Typeflag: tar.TypeSymlink, Linkname: "b"},
|
||||||
|
{Name: "b", Typeflag: tar.TypeSymlink, Linkname: "dist"},
|
||||||
|
{Name: "dangling", Typeflag: tar.TypeSymlink, Linkname: "missing"},
|
||||||
|
{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||||
|
} {
|
||||||
|
require.NoError(t, writer.WriteHeader(entry))
|
||||||
|
}
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
fallback bool
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{name: "native", want: []string{"a", "b", "dangling"}},
|
||||||
|
{name: "fallback", fallback: true, want: []string{"b", "a", "dangling"}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
symlinkFallback = test.fallback
|
||||||
|
created = nil
|
||||||
|
_, err := extractTree(t.Context(), bytes.NewReader(archive.Bytes()), t.TempDir(), false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, test.want, created)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyTreeLinkCancellationBetweenEntries(t *testing.T) {
|
||||||
|
root, err := os.OpenRoot(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer root.Close()
|
||||||
|
require.NoError(t, root.Mkdir("source", 0o755))
|
||||||
|
for _, name := range []string{"first", "second"} {
|
||||||
|
_, err := writeTreeFile(root, filepath.Join("source", name), 0o644, strings.NewReader("content"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
sizes := map[string]int64{}
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
observed := &treeCopyContext{Context: ctx, check: func() {
|
||||||
|
if len(sizes) == 1 {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
require.ErrorIs(t, copyTreeLink(observed, root, "copy", map[string]string{"copy": "source"}, nil, sizes), context.Canceled)
|
||||||
|
assert.Len(t, sizes, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
type treeCopyContext struct {
|
||||||
|
context.Context
|
||||||
|
check func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ctx *treeCopyContext) Err() error {
|
||||||
|
ctx.check()
|
||||||
|
return ctx.Context.Err()
|
||||||
|
}
|
||||||
+54
-10
@@ -5,11 +5,13 @@
|
|||||||
package git
|
package git
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -26,6 +28,7 @@ import (
|
|||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||||
|
"gitea.com/gitea/runner/internal/pkg/process"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -182,6 +185,7 @@ type NewGitCloneExecutorInput struct {
|
|||||||
// For Gitea
|
// For Gitea
|
||||||
InsecureSkipTLS bool
|
InsecureSkipTLS bool
|
||||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||||
|
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -320,11 +324,20 @@ func cachedRef(ref string) string {
|
|||||||
|
|
||||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||||
if !isFullObjectID(ref) {
|
if !isFullObjectID(ref) {
|
||||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
if strings.ContainsAny(ref, "\x00\n") {
|
||||||
|
for _, revision := range revisions {
|
||||||
|
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||||
return sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||||
|
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||||
|
if isFullObjectID(sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
|
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
|
||||||
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
|
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
|
||||||
@@ -377,6 +390,21 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
|||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||||
|
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||||
|
var config []string
|
||||||
|
if certFile != "" {
|
||||||
|
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||||
|
}
|
||||||
|
if caFile != "" {
|
||||||
|
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||||
|
}
|
||||||
|
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||||
|
config = append(config, prefix+"sslBackend=openssl")
|
||||||
|
}
|
||||||
|
return config
|
||||||
|
}
|
||||||
|
|
||||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||||
func ConfigEnv(config ...string) map[string]string {
|
func ConfigEnv(config ...string) map[string]string {
|
||||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||||
@@ -427,17 +455,20 @@ func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||||
|
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
if input.ClientCertFile != "" {
|
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if dir != "" {
|
if dir != "" {
|
||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
|
cmd.Stdin = stdin
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||||
name, _, _ := strings.Cut(entry, "=")
|
name, _, _ := strings.Cut(entry, "=")
|
||||||
return slices.Contains(LocalEnvVars, name)
|
return slices.Contains(LocalEnvVars, name)
|
||||||
@@ -446,15 +477,28 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
|||||||
cmd.Env = append(cmd.Env, key+"="+value)
|
cmd.Env = append(cmd.Env, key+"="+value)
|
||||||
}
|
}
|
||||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||||
if runtime.GOOS != "windows" {
|
var treeKill *process.TreeKill
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
|
||||||
|
} else {
|
||||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||||
}
|
}
|
||||||
cmd.WaitDelay = time.Second
|
cmd.WaitDelay = time.Second
|
||||||
|
var stdout bytes.Buffer
|
||||||
var stderr strings.Builder
|
var stderr strings.Builder
|
||||||
|
cmd.Stdout = &stdout
|
||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
output, err := cmd.Output()
|
err := cmd.Start()
|
||||||
if err != nil {
|
if err == nil {
|
||||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
if treeKill != nil {
|
||||||
|
if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
|
||||||
|
defer killer.Close()
|
||||||
}
|
}
|
||||||
return output, nil
|
}
|
||||||
|
err = cmd.Wait()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), nil
|
||||||
}
|
}
|
||||||
+58
-10
@@ -29,6 +29,47 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var gitFixtureDir string
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
var err error
|
||||||
|
gitFixtureDir, err = os.MkdirTemp("", "runner-git-fixture-")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
code := 1
|
||||||
|
if err = gitCmd("init", "--quiet", "--template=", "--bare", "--initial-branch=main", filepath.Join(gitFixtureDir, "remote")); err == nil {
|
||||||
|
err = gitCmd("init", "--quiet", "--template=", "--initial-branch=main", filepath.Join(gitFixtureDir, "work"))
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
code = m.Run()
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(gitFixtureDir); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
code = 1
|
||||||
|
}
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSSLConfig(t *testing.T) {
|
||||||
|
for _, testcase := range []struct {
|
||||||
|
prefix, certFile, caFile, executionOS string
|
||||||
|
expected []string
|
||||||
|
}{
|
||||||
|
{"http.https://example.com/.", "cert.pem", "ca.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslCAInfo=ca.pem", "http.https://example.com/.sslBackend=openssl"}},
|
||||||
|
{"http.", "", "ca.pem", "windows", []string{"http.sslCAInfo=ca.pem", "http.sslBackend=openssl"}},
|
||||||
|
{"http.", "cert.pem", "ca.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslCAInfo=ca.pem"}},
|
||||||
|
{"http.", "", "", "Windows", nil},
|
||||||
|
} {
|
||||||
|
t.Run(testcase.prefix+testcase.certFile+testcase.caFile+testcase.executionOS, func(t *testing.T) {
|
||||||
|
assert.Equal(t, testcase.expected, SSLConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.caFile, testcase.executionOS))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFindGitSlug(t *testing.T) {
|
func TestFindGitSlug(t *testing.T) {
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
remoteURL string
|
remoteURL string
|
||||||
@@ -141,9 +182,8 @@ func TestGitFindRef(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
dir := filepath.Join(basedir, name)
|
dir := filepath.Join(basedir, name)
|
||||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
|
require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/master\n"), 0o644))
|
||||||
require.NoError(t, os.RemoveAll(filepath.Join(dir, ".git", "hooks")))
|
|
||||||
for _, command := range tt.commands {
|
for _, command := range tt.commands {
|
||||||
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
||||||
}
|
}
|
||||||
@@ -161,7 +201,8 @@ func TestGitFindRef(t *testing.T) {
|
|||||||
func TestGitCloneExecutor(t *testing.T) {
|
func TestGitCloneExecutor(t *testing.T) {
|
||||||
remoteDir, workDir := newGitRemote(t)
|
remoteDir, workDir := newGitRemote(t)
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".gitattributes"), []byte("*.sh text\n"), 0o644))
|
||||||
|
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||||
@@ -176,10 +217,11 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||||
|
|
||||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||||
t.Setenv("GIT_DIR", t.TempDir())
|
t.Setenv("GIT_DIR", t.TempDir())
|
||||||
|
|
||||||
|
var clones sync.WaitGroup
|
||||||
for name, tt := range map[string]struct {
|
for name, tt := range map[string]struct {
|
||||||
Err error
|
Err error
|
||||||
Ref string
|
Ref string
|
||||||
@@ -207,6 +249,7 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
Ref: fullSha[:4],
|
Ref: fullSha[:4],
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
clones.Go(func() {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
@@ -231,7 +274,9 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||||
})
|
})
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
clones.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||||
@@ -476,9 +521,12 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
|||||||
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
remoteDir, workDir := t.TempDir(), t.TempDir()
|
remoteDir, workDir := t.TempDir(), t.TempDir()
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
require.NoError(t, os.CopyFS(remoteDir, os.DirFS(filepath.Join(gitFixtureDir, "remote"))))
|
||||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
require.NoError(t, os.CopyFS(workDir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
config, err := os.ReadFile(filepath.Join(workDir, ".git", "config"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".git", "config"),
|
||||||
|
fmt.Appendf(config, "\n[remote \"origin\"]\n\turl = %s\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", strconv.Quote(remoteDir)), 0o644))
|
||||||
for _, message := range messages {
|
for _, message := range messages {
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||||
}
|
}
|
||||||
@@ -645,8 +693,8 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
|||||||
|
|
||||||
select {
|
select {
|
||||||
case <-reached:
|
case <-reached:
|
||||||
case <-time.After(10 * time.Second):
|
case err := <-done:
|
||||||
t.Fatal("the executor never reached the remote")
|
t.Fatalf("the executor finished before reaching the remote: %v", err)
|
||||||
}
|
}
|
||||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||||
cancel()
|
cancel()
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package common
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// LookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||||
|
func LookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||||
|
if value, ok := values[name]; ok || !caseInsensitive {
|
||||||
|
return value, ok
|
||||||
|
}
|
||||||
|
for key, value := range values {
|
||||||
|
if strings.EqualFold(key, name) {
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var zero V
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"path"
|
"path"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -20,6 +21,8 @@ import (
|
|||||||
"github.com/moby/moby/api/types/container"
|
"github.com/moby/moby/api/types/container"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var hostOS = runtime.GOOS // replaced in tests
|
||||||
|
|
||||||
// ExitCodeError reports a non-zero process exit code from a container command.
|
// ExitCodeError reports a non-zero process exit code from a container command.
|
||||||
type ExitCodeError int
|
type ExitCodeError int
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
@@ -48,6 +48,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
options := client.ImageBuildOptions{
|
options := client.ImageBuildOptions{
|
||||||
Tags: []string{input.ImageTag},
|
Tags: []string{input.ImageTag},
|
||||||
|
Labels: map[string]string{"com.gitea.runner.image": input.ImageTag}, // own image ID per tag, so removing one never deletes another job's identical build
|
||||||
Remove: true,
|
Remove: true,
|
||||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||||
Dockerfile: input.Dockerfile,
|
Dockerfile: input.Dockerfile,
|
||||||
@@ -74,8 +75,10 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
||||||
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
||||||
|
if err != nil {
|
||||||
return logDockerResponse(logger, resp.Body, err != nil)
|
return err
|
||||||
|
}
|
||||||
|
return logDockerResponse(logger, resp.Body, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,6 +126,10 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||||
|
contextDir, err := filepath.EvalSymlinks(contextDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
tw := tar.NewWriter(w)
|
tw := tar.NewWriter(w)
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
hardlinks := map[uint64]string{}
|
hardlinks := map[uint64]string{}
|
||||||
@@ -130,8 +137,7 @@ func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, incl
|
|||||||
for _, include := range includes {
|
for _, include := range includes {
|
||||||
include = filepath.FromSlash(include)
|
include = filepath.FromSlash(include)
|
||||||
var parents []parentMatch
|
var parents []parentMatch
|
||||||
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
err := filepath.WalkDir(filepath.Join(contextDir, include), func(file string, entry fs.DirEntry, err error) error {
|
||||||
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return leaveOut(ctx, file, err)
|
return leaveOut(ctx, file, err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +214,7 @@ func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string,
|
|||||||
return leaveOut(ctx, file, err)
|
return leaveOut(ctx, file, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
header, err := tar.FileInfoHeader(noNameLookups{info}, filepath.ToSlash(link))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,20 +1,45 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||||
|
var labels string
|
||||||
|
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
writer.Header().Set("API-Version", "1.47")
|
||||||
|
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||||
|
labels = request.URL.Query().Get("labels")
|
||||||
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
|
writer.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(daemon.Close)
|
||||||
|
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||||
|
t.Setenv("DOCKER_CONFIG", t.TempDir())
|
||||||
|
t.Setenv("DOCKER_AUTH_CONFIG", "")
|
||||||
|
|
||||||
|
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||||
|
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||||
|
})(t.Context()), "build rejected")
|
||||||
|
require.JSONEq(t, `{"com.gitea.runner.image":"action:test"}`, labels)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
for name, content := range map[string]string{
|
for name, content := range map[string]string{
|
||||||
@@ -28,7 +53,7 @@ func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile
|
|||||||
|
|
||||||
contexts := []string{dir}
|
contexts := []string{dir}
|
||||||
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||||
contexts = append(contexts, link+string(filepath.Separator))
|
contexts = append(contexts, link, link+string(filepath.Separator))
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, contextDir := range contexts {
|
for _, contextDir := range contexts {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||||
// * appended with license information
|
// * appended with license information
|
||||||
@@ -351,7 +351,7 @@ type containerConfig struct {
|
|||||||
// parse parses the args for the specified command and generates a Config,
|
// parse parses the args for the specified command and generates a Config,
|
||||||
// a HostConfig and returns them with the specified command.
|
// a HostConfig and returns them with the specified command.
|
||||||
// If the specified args are not valid, it will return an error.
|
// If the specified args are not valid, it will return an error.
|
||||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo,unparam // verbatim copy from docker/cli
|
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo // verbatim copy from docker/cli
|
||||||
var (
|
var (
|
||||||
attachStdin = copts.attach.Get("stdin")
|
attachStdin = copts.attach.Get("stdin")
|
||||||
attachStdout = copts.attach.Get("stdout")
|
attachStdout = copts.attach.Get("stdout")
|
||||||
@@ -724,6 +724,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isUserDefinedNetwork(mode container.NetworkMode) bool { // Docker's helper uses the host OS instead of the daemon's.
|
||||||
|
return !mode.IsDefault() && mode != network.NetworkBridge && mode != network.NetworkHost && !mode.IsNone() && !mode.IsContainer()
|
||||||
|
}
|
||||||
|
|
||||||
// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
|
// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
|
||||||
// them with the old --network-alias and --links. If returns an error if conflicting options
|
// them with the old --network-alias and --links. If returns an error if conflicting options
|
||||||
// are found.
|
// are found.
|
||||||
@@ -752,7 +756,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, n := range copts.netMode.Value() {
|
for i, n := range copts.netMode.Value() {
|
||||||
if container.NetworkMode(n.Target).IsUserDefined() {
|
if isUserDefinedNetwork(container.NetworkMode(n.Target)) {
|
||||||
hasUserDefined = true
|
hasUserDefined = true
|
||||||
} else {
|
} else {
|
||||||
hasNonUserDefined = true
|
hasNonUserDefined = true
|
||||||
@@ -814,7 +818,7 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
|||||||
}
|
}
|
||||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||||
// for the default bridge it defines a legacy-link.
|
// for the default bridge it defines a legacy-link.
|
||||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
if isUserDefinedNetwork(container.NetworkMode(n.Target)) && copts.links.Len() > 0 {
|
||||||
n.Links = slices.Clone(copts.links.GetSlice())
|
n.Links = slices.Clone(copts.links.GetSlice())
|
||||||
}
|
}
|
||||||
if copts.ipv4Address != nil {
|
if copts.ipv4Address != nil {
|
||||||
@@ -840,7 +844,7 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
|||||||
if strings.TrimSpace(ep.Target) == "" {
|
if strings.TrimSpace(ep.Target) == "" {
|
||||||
return nil, errors.New("no name set for network")
|
return nil, errors.New("no name set for network")
|
||||||
}
|
}
|
||||||
if !container.NetworkMode(ep.Target).IsUserDefined() {
|
if !isUserDefinedNetwork(container.NetworkMode(ep.Target)) {
|
||||||
if len(ep.Aliases) > 0 {
|
if len(ep.Aliases) > 0 {
|
||||||
return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
|
return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
@@ -5,14 +5,17 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
|
||||||
|
mobyclient "github.com/moby/moby/client"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -21,48 +24,47 @@ func init() {
|
|||||||
logging.SetLevel(logging.LevelDebug)
|
logging.SetLevel(logging.LevelDebug)
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
|
func loadScratchImage(t *testing.T, platform string) string {
|
||||||
// network or emulation since there is nothing to run) and returns its tag, removing it after
|
|
||||||
// the test.
|
|
||||||
func buildScratchImage(t *testing.T, platform string) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
tag := fmt.Sprintf("act-test-exists-%s:latest", strings.TrimPrefix(platform, "linux/"))
|
tag := fmt.Sprintf("act-test-exists-%s-%d:latest", strings.TrimPrefix(platform, "linux/"), time.Now().UnixNano())
|
||||||
cmd := exec.Command("docker", "build", "--platform", platform, "-t", tag, "-")
|
var archive bytes.Buffer
|
||||||
cmd.Stdin = strings.NewReader("FROM scratch\nLABEL act-test=1\n")
|
require.NoError(t, writeFilesTar(t.Context(), &archive, 0, 0,
|
||||||
// Force BuildKit: it records the requested architecture in the image config for a
|
&FileEntry{Name: "config.json", Mode: 0o600, Body: fmt.Sprintf(`{"architecture":%q,"os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`, strings.TrimPrefix(platform, "linux/"))},
|
||||||
// FROM-scratch build, whereas the classic builder ignores --platform and tags it with the
|
&FileEntry{Name: "manifest.json", Mode: 0o600, Body: fmt.Sprintf(`[{"Config":"config.json","RepoTags":[%q],"Layers":[]}]`, tag)},
|
||||||
// host arch, which would break the per-platform existence assertions below.
|
))
|
||||||
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
|
cli, err := GetDockerClient(t.Context())
|
||||||
out, err := cmd.CombinedOutput()
|
require.NoError(t, err)
|
||||||
require.NoError(t, err, string(out))
|
t.Cleanup(func() { _ = cli.Close() })
|
||||||
t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() })
|
response, err := cli.ImageLoad(t.Context(), &archive)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer response.Close()
|
||||||
|
require.NoError(t, logDockerResponse(common.Logger(t.Context()), response, false))
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = cli.ImageRemove(context.WithoutCancel(t.Context()), tag, mobyclient.ImageRemoveOptions{Force: true})
|
||||||
|
})
|
||||||
return tag
|
return tag
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageExistsLocally(t *testing.T) {
|
func TestImageExistsLocally(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
requireDocker(t)
|
requireDocker(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
// a non-existent image is reported absent
|
|
||||||
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, missing)
|
assert.False(t, missing)
|
||||||
|
|
||||||
// Build tiny images for two architectures locally so per-platform existence can be checked
|
amd64Ref := loadScratchImage(t, "linux/amd64")
|
||||||
// offline (formerly pulled node:24-bookworm-slim for amd64 and arm64 over the network).
|
arm64Ref := loadScratchImage(t, "linux/arm64")
|
||||||
amd64Ref := buildScratchImage(t, "linux/amd64")
|
|
||||||
arm64Ref := buildScratchImage(t, "linux/arm64")
|
|
||||||
|
|
||||||
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, amd64Exists)
|
assert.True(t, amd64Exists)
|
||||||
|
|
||||||
// a non-host architecture image is detected for its own architecture
|
|
||||||
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.True(t, arm64Exists)
|
assert.True(t, arm64Exists)
|
||||||
|
|
||||||
// a present image is reported absent for a different platform
|
|
||||||
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
||||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||||
assert.False(t, wrongPlatform)
|
assert.False(t, wrongPlatform)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
@@ -33,6 +33,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
|||||||
defer dockerResponse.Close()
|
defer dockerResponse.Close()
|
||||||
|
|
||||||
scanner := bufio.NewScanner(dockerResponse)
|
scanner := bufio.NewScanner(dockerResponse)
|
||||||
|
scanner.Buffer(nil, 64*1024*1024) // a build echoes long instructions in one record
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Bytes()
|
line := scanner.Bytes()
|
||||||
var msg dockerMessage
|
var msg dockerMessage
|
||||||
@@ -65,7 +66,7 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return scanner.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
@@ -85,14 +85,14 @@ networks:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return createDockerNetwork(ctx, cli, name, opts)
|
return createDockerNetwork(ctx, cli, name, opts)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||||
@@ -157,7 +157,7 @@ func isAddressPoolExhausted(err error) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -167,7 +167,7 @@ func NewDockerNetworkConnectExecutor(networkName, container string) common.Execu
|
|||||||
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -182,14 +182,14 @@ func connectNetwork(ctx context.Context, cli client.APIClient, networkName, cont
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return disconnectNetwork(ctx, cli, networkName, container)
|
return disconnectNetwork(ctx, cli, networkName, container)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||||
@@ -201,14 +201,14 @@ func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, c
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
cli, err := GetDockerClient(ctx)
|
cli, err := GetDockerClient(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
return removeNetwork(ctx, cli, name)
|
return removeNetwork(ctx, cli, name)
|
||||||
}
|
}).IfNot(common.Dryrun)
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// Copyright 2025 The nektos/act Authors. All rights reserved.
|
// Copyright 2025 The nektos/act Authors. All rights reserved.
|
||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||||
|
|
||||||
package container
|
package container
|
||||||
|
|
||||||
|
|||||||
Loaded 50 of 236 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user