mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-12 19:03:45 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b5305c6785 | ||
|
|
a021d3139d | ||
|
|
84c0fcd1fa | ||
|
|
c59caa9689 | ||
|
|
85d007c47f | ||
|
|
d34e4cf11a | ||
|
|
8e999cd039 | ||
|
|
cd748053b0 | ||
|
|
9bb72bbe5c | ||
|
|
79faf5e55d | ||
|
|
993389d428 | ||
|
|
db76fd5159 | ||
|
|
c8b16f4792 | ||
|
|
02446620df | ||
|
|
8c8950d9ef | ||
|
|
b88d111cc5 | ||
|
|
bb6d793cb9 | ||
|
|
95ef17d022 | ||
|
|
ee03687688 | ||
|
|
30e600d8f6 | ||
|
|
664978c819 | ||
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 | ||
|
|
f2fbd67962 | ||
|
|
d6f5b68a0d | ||
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 | ||
|
|
c719a88f36 | ||
|
|
d882311ec6 | ||
|
|
923a0bfb59 | ||
|
|
6f0d4f1b48 | ||
|
|
430f7bb2cf | ||
|
|
befb1b4bb9 | ||
|
|
1a7f09224f | ||
|
|
af2b3f6df5 | ||
|
|
785313cc21 | ||
|
|
41f1e08bec | ||
|
|
4158e05dae | ||
|
|
e356c2ed85 | ||
|
|
ed940db4f0 | ||
|
|
755af87095 | ||
|
|
b9aa47d1a5 | ||
|
|
0d84ec6dab | ||
|
|
b769a9765f | ||
|
|
9c4c66e70f | ||
|
|
42858c61df | ||
|
|
470969dc74 | ||
|
|
be7ddcd15a | ||
|
|
c3c647ef83 | ||
|
|
160e18e705 | ||
|
|
20c8b4abc4 | ||
|
|
ccb9afd7a5 | ||
|
|
49e34813ae | ||
|
|
e0ef637805 | ||
|
|
8502663d0e | ||
|
|
4e8037a93b | ||
|
|
76110557ab | ||
|
|
3bc5cc03f8 | ||
|
|
de961b91eb | ||
|
|
ad9eeb668a | ||
|
|
b741335222 | ||
|
|
20143c9558 |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
|||||||
|
|
||||||
/bin
|
/bin
|
||||||
/dist
|
/dist
|
||||||
|
/.tmp
|
||||||
|
/.git/hooks
|
||||||
|
/.git/logs
|
||||||
|
/.git/rr-cache
|
||||||
/.env
|
/.env
|
||||||
/.runner
|
/.runner
|
||||||
|
/.runner.lock
|
||||||
/config.yaml
|
/config.yaml
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: go-setup
|
||||||
|
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||||
|
id: setup-go
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: false
|
||||||
|
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||||
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||||
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: cache-seeder
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- go.mod
|
||||||
|
- go.sum
|
||||||
|
- .golangci.yml
|
||||||
|
- Makefile
|
||||||
|
- .gitea/actions/go-setup/action.yml
|
||||||
|
- .gitea/workflows/cache-seeder.yml
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
seed:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- run: make deps-tools lint build
|
||||||
|
- name: compile the test binaries without running them
|
||||||
|
run: |
|
||||||
|
make test GOTEST_FLAGS='-race -exec true'
|
||||||
|
go test -tags e2e -exec true ./e2e/...
|
||||||
@@ -19,9 +19,9 @@ jobs:
|
|||||||
timeout-minutes: 5
|
timeout-minutes: 5
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 26
|
||||||
- run: make lint-pr-title
|
- run: make lint-pr-title
|
||||||
env:
|
env:
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
@@ -11,7 +11,6 @@ on:
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: nightly
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
goreleaser:
|
goreleaser:
|
||||||
@@ -35,8 +34,15 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
- name: Import GPG key
|
||||||
|
id: import_gpg
|
||||||
|
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||||
|
with:
|
||||||
|
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||||
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release --nightly
|
args: release --nightly
|
||||||
@@ -48,6 +54,7 @@ jobs:
|
|||||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||||
GORELEASER_FORCE_TOKEN: "gitea"
|
GORELEASER_FORCE_TOKEN: "gitea"
|
||||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||||
|
|
||||||
release-image:
|
release-image:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -80,7 +87,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
passphrase: ${{ secrets.PASSPHRASE }}
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release
|
args: release
|
||||||
@@ -69,7 +69,6 @@ jobs:
|
|||||||
image: catthehacker/ubuntu:act-latest
|
image: catthehacker/ubuntu:act-latest
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: latest
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
@@ -83,7 +82,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
+37
-15
@@ -17,25 +17,24 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||||
- name: pre-pull test images
|
- name: pre-pull test images
|
||||||
env:
|
env:
|
||||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:0e0ff40c39bc087845bfb27465a0df4ea419520094bc35842ff83dd8cbe6f9b6 # renovate: datasource=docker
|
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||||
run: |
|
run: |
|
||||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||||
for attempt in 1 2 3; do
|
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||||
docker pull "$image" && break
|
for attempt in 1 2 3; do
|
||||||
[ "$attempt" = 3 ] || sleep 5
|
docker pull "$image" && break
|
||||||
done
|
[ "$attempt" = 3 ] || sleep 5
|
||||||
|
done
|
||||||
|
fi
|
||||||
docker tag "$image" "${image%@*}"
|
docker tag "$image" "${image%@*}"
|
||||||
done
|
done
|
||||||
- name: lint
|
- name: lint
|
||||||
@@ -58,6 +57,32 @@ jobs:
|
|||||||
make coverage-report
|
make coverage-report
|
||||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
|
windows:
|
||||||
|
name: test on windows
|
||||||
|
runs-on: ubuntu-latest-kvm
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- name: prepare anonymous docker config
|
||||||
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
|
- name: windows vm cache key
|
||||||
|
id: windows-vm
|
||||||
|
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
- name: test on windows
|
||||||
|
id: test-windows
|
||||||
|
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||||
|
env:
|
||||||
|
WINDOWS_TEST_TIMEOUT: 45
|
||||||
|
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||||
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
|
||||||
e2e:
|
e2e:
|
||||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -68,10 +93,7 @@ jobs:
|
|||||||
- gitea/gitea:main-nightly
|
- gitea/gitea:main-nightly
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
- name: e2e compatibility
|
- name: e2e compatibility
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
/gitea-runner
|
/gitea-runner
|
||||||
|
/gitea-runner.exe
|
||||||
.env
|
.env
|
||||||
!/act/runner/testdata/secrets/.env
|
!/act/runner/testdata/secrets/.env
|
||||||
.runner
|
.runner
|
||||||
|
|||||||
@@ -39,6 +39,15 @@ linters:
|
|||||||
deny:
|
deny:
|
||||||
- pkg: golang.org/x/exp
|
- pkg: golang.org/x/exp
|
||||||
desc: it's experimental and unreliable
|
desc: it's experimental and unreliable
|
||||||
|
- pkg: unsafe
|
||||||
|
desc: use typed APIs instead
|
||||||
|
logging:
|
||||||
|
files:
|
||||||
|
- $all
|
||||||
|
- "!**/act/common/logging/**"
|
||||||
|
deny:
|
||||||
|
- pkg: log/slog
|
||||||
|
desc: log through gitea.com/gitea/runner/act/common/logging
|
||||||
nolintlint:
|
nolintlint:
|
||||||
allow-unused: false
|
allow-unused: false
|
||||||
require-explanation: true
|
require-explanation: true
|
||||||
|
|||||||
+5
-22
@@ -28,38 +28,16 @@ builds:
|
|||||||
ignore:
|
ignore:
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: arm
|
goarch: arm
|
||||||
- goos: darwin
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm64
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
flags:
|
flags:
|
||||||
@@ -102,6 +80,7 @@ builds:
|
|||||||
publishers:
|
publishers:
|
||||||
- name: cloudflare-r2
|
- name: cloudflare-r2
|
||||||
checksum: true
|
checksum: true
|
||||||
|
signature: true
|
||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
- glob: ./**.sha256
|
- glob: ./**.sha256
|
||||||
@@ -122,6 +101,10 @@ checksum:
|
|||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
|
|
||||||
|
signs:
|
||||||
|
- artifacts: checksum
|
||||||
|
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||||
|
|
||||||
snapshot:
|
snapshot:
|
||||||
version_template: "{{ .Branch }}-devel"
|
version_template: "{{ .Branch }}-devel"
|
||||||
|
|
||||||
|
|||||||
+44
-11
@@ -23,24 +23,57 @@ depending on the prefix:
|
|||||||
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
|
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
|
||||||
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
|
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
|
||||||
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
|
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
|
||||||
- **Forwarding a command from step output?** Leave the payload alone, it arrived escaped and is
|
- **Forwarding a command from step output?** Preserve its escaping unless masking changes it.
|
||||||
decoded once. Decoding here double-decodes and destroys multi-line.
|
Decoding for display happens in the UI.
|
||||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
- **Interpolating a secret?** Masking runs after escaping. Recognised command payloads are
|
||||||
encoded forms too.
|
normalised before masking to cover partially escaped values. Their original escaping is
|
||||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
|
preserved when no replacement occurs. `AppendSecretMasker` also registers encoded forms for
|
||||||
decodes exactly those two when folding a location into an annotation.
|
other output. Complete nonblank one-character values are masked, while one-byte fragments
|
||||||
|
of multiline values are skipped.
|
||||||
|
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||||
|
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||||
|
|
||||||
## Built-in actions
|
## Logging
|
||||||
|
|
||||||
Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions` in
|
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
|
||||||
`act/runner/step_action_builtin.go`.
|
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
|
||||||
|
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
|
||||||
|
fields.
|
||||||
|
|
||||||
|
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
|
||||||
|
Going around either leaks them.
|
||||||
|
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
|
||||||
|
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
|
||||||
|
job logger.
|
||||||
|
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
|
||||||
|
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
|
||||||
|
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
|
||||||
|
package functions.
|
||||||
|
|
||||||
## End-to-end compatibility tests
|
## End-to-end compatibility tests
|
||||||
|
|
||||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||||
parallel.
|
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||||
|
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||||
|
|
||||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||||
|
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||||
|
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||||
|
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||||
|
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||||
|
Docker and Kubernetes.
|
||||||
|
|
||||||
|
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||||
|
|
||||||
|
## Windows tests
|
||||||
|
|
||||||
|
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
|
||||||
|
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
|
||||||
|
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||||
|
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||||
|
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||||
|
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||||
|
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||||
+11
-7
@@ -1,7 +1,7 @@
|
|||||||
### BUILDER STAGE
|
### BUILDER STAGE
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM golang:1.27-alpine3.23 AS builder
|
FROM golang:1.27-alpine AS builder
|
||||||
|
|
||||||
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
||||||
RUN apk add --no-cache make git
|
RUN apk add --no-cache make git
|
||||||
@@ -12,19 +12,22 @@ ENV GOPROXY=${GOPROXY:-}
|
|||||||
COPY . /opt/src/runner
|
COPY . /opt/src/runner
|
||||||
WORKDIR /opt/src/runner
|
WORKDIR /opt/src/runner
|
||||||
|
|
||||||
RUN make clean && make build
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
make clean && make build
|
||||||
|
|
||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind AS dind
|
FROM docker:29.9.3-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||||
|
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
@@ -39,7 +42,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind-rootless AS dind-rootless
|
FROM docker:29.9.3-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -47,7 +50,8 @@ LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
|||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||||
|
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
@@ -73,7 +77,7 @@ ARG VERSION=dev
|
|||||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
RUN apk add --no-cache tini bash git tzdata
|
RUN apk upgrade --no-cache && apk add --no-cache tini bash git tzdata
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
|
|||||||
@@ -1,31 +1,23 @@
|
|||||||
DIST := dist
|
DIST := dist
|
||||||
EXECUTABLE := gitea-runner
|
|
||||||
DIST_DIRS := $(DIST)/binaries $(DIST)/release
|
|
||||||
GO ?= go
|
GO ?= go
|
||||||
SHASUM ?= shasum -a 256
|
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||||
XGO_PACKAGE ?= src.techknowlogick.com/[email protected] # renovate: datasource=go
|
|
||||||
XGO_VERSION := go-1.27.x
|
|
||||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
|
|
||||||
|
|
||||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
|
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:b96ba465021b1a4c6a1c9e79220c2b9bfad0f29b09dbb528bbfb35411443a3a4 # renovate: datasource=docker
|
||||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
|
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.2-k3s1@sha256:21ea6e0fac4e3930c2ce8aa40ddd41a8b18cb3e9a0222c030dc005e7e88c4332 # renovate: datasource=docker
|
||||||
|
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||||
|
|
||||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
|
||||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
|
||||||
WINDOWS_ARCHS ?= windows/amd64
|
|
||||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||||
|
|
||||||
DOCKER_IMAGE ?= gitea/runner
|
DOCKER_IMAGE ?= gitea/runner
|
||||||
DOCKER_TAG ?= nightly
|
DOCKER_TAG ?= nightly
|
||||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
|
||||||
|
|
||||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||||
|
|
||||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||||
|
|
||||||
@@ -38,22 +30,16 @@ endif
|
|||||||
ifeq ($(HAS_GO), GO)
|
ifeq ($(HAS_GO), GO)
|
||||||
GOPATH ?= $(shell $(GO) env GOPATH)
|
GOPATH ?= $(shell $(GO) env GOPATH)
|
||||||
export PATH := $(GOPATH)/bin:$(PATH)
|
export PATH := $(GOPATH)/bin:$(PATH)
|
||||||
|
|
||||||
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
|
|
||||||
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(OS), Windows_NT)
|
ifeq ($(OS), Windows_NT)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else ifeq ($(OS), Windows)
|
else ifeq ($(OS), Windows)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else
|
else
|
||||||
GOFLAGS := -v
|
GOFLAGS := -v
|
||||||
EXECUTABLE ?= $(EXECUTABLE)
|
|
||||||
GO_ENV_WINDOWS := GOOS=windows
|
GO_ENV_WINDOWS := GOOS=windows
|
||||||
endif
|
endif
|
||||||
|
|
||||||
@@ -108,8 +94,6 @@ fmt-check: ## check the Go code formatting
|
|||||||
.PHONY: deps-tools
|
.PHONY: deps-tools
|
||||||
deps-tools: ## install tool dependencies
|
deps-tools: ## install tool dependencies
|
||||||
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
||||||
$(GO) install $(GXZ_PACKAGE) & \
|
|
||||||
$(GO) install $(XGO_PACKAGE) & \
|
|
||||||
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
||||||
wait
|
wait
|
||||||
|
|
||||||
@@ -167,11 +151,15 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
|||||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||||
|
|
||||||
|
.PHONY: test-windows
|
||||||
|
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||||
|
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||||
|
|
||||||
E2E_CONCURRENCY ?= 8
|
E2E_CONCURRENCY ?= 8
|
||||||
|
|
||||||
.PHONY: test-e2e
|
.PHONY: test-e2e
|
||||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||||
|
|
||||||
.PHONY: install
|
.PHONY: install
|
||||||
install: $(GOFILES) ## install the runner binary via `go install`
|
install: $(GOFILES) ## install the runner binary via `go install`
|
||||||
@@ -187,45 +175,6 @@ $(EXECUTABLE): $(GOFILES)
|
|||||||
deps-backend: ## install backend dependencies
|
deps-backend: ## install backend dependencies
|
||||||
$(GO) mod download
|
$(GO) mod download
|
||||||
|
|
||||||
.PHONY: release
|
|
||||||
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
|
|
||||||
|
|
||||||
$(DIST_DIRS):
|
|
||||||
mkdir -p $(DIST_DIRS)
|
|
||||||
|
|
||||||
.PHONY: release-windows
|
|
||||||
release-windows: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-linux
|
|
||||||
release-linux: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-darwin
|
|
||||||
release-darwin: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-copy
|
|
||||||
release-copy: | $(DIST_DIRS)
|
|
||||||
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
|
|
||||||
|
|
||||||
.PHONY: release-check
|
|
||||||
release-check: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
|
|
||||||
|
|
||||||
.PHONY: release-compress
|
|
||||||
release-compress: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
|
|
||||||
|
|
||||||
.PHONY: docker
|
.PHONY: docker
|
||||||
docker: ## build the docker image
|
docker: ## build the docker image
|
||||||
if ! docker buildx version >/dev/null 2>&1; then \
|
if ! docker buildx version >/dev/null 2>&1; then \
|
||||||
|
|||||||
@@ -6,7 +6,5 @@
|
|||||||
//
|
//
|
||||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||||
//
|
//
|
||||||
// TODO: Authorization
|
|
||||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
|
||||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||||
package artifactcache
|
package artifactcache
|
||||||
+436
-242
File diff suppressed because it is too large.
Load diff
+399
-734
File diff suppressed because it is too large.
Load diff
@@ -15,8 +15,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The cache service v2 API. A client on this version talks twirp to
|
// The cache service v2 API. A client on this version talks twirp to
|
||||||
@@ -44,16 +42,16 @@ const (
|
|||||||
twirpUnauthenticated = "unauthenticated"
|
twirpUnauthenticated = "unauthenticated"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *Handler) registerV2Routes(router *httprouter.Router) {
|
func (h *Handler) registerV2Routes(router *http.ServeMux) {
|
||||||
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
||||||
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
||||||
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
||||||
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
||||||
}
|
}
|
||||||
|
|
||||||
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
||||||
// the upload.
|
// the upload.
|
||||||
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -75,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||||
// existing and silently never saved.
|
// existing and silently never saved.
|
||||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if existing != nil {
|
} else if existing != nil {
|
||||||
@@ -86,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
|
|
||||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||||
owner := hashedToken(bearerToken(r))
|
owner := hashedToken(bearerToken(r))
|
||||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||||
@@ -102,15 +100,17 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache := &Cache{
|
cache := &Cache{
|
||||||
Repo: cred.Repo,
|
Repo: cred.Repo,
|
||||||
Key: req.Key,
|
Ref: cred.Ref,
|
||||||
Version: req.Version,
|
Key: req.Key,
|
||||||
Owner: owner,
|
Version: req.Version,
|
||||||
Size: -1, // the size is only known at finalize time
|
Owner: owner,
|
||||||
CreatedAt: now,
|
Size: -1, // the size is only known at finalize time
|
||||||
UsedAt: now,
|
CreatedAt: now.Unix(),
|
||||||
|
CreatedAtNanos: now.Nanosecond(),
|
||||||
|
UsedAt: now.Unix(),
|
||||||
}
|
}
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -123,7 +123,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -138,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||||
cache = nil // not the reservation this job made, so not this job's to commit
|
cache = nil // not the reservation this job made, so not this job's to commit
|
||||||
}
|
}
|
||||||
if err == nil && cache == nil {
|
if err == nil && cache == nil {
|
||||||
// A retry whose first response was lost finds the entry already committed.
|
// A retry whose first response was lost finds the entry already committed.
|
||||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -172,7 +172,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
return
|
return
|
||||||
@@ -207,8 +207,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
||||||
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
||||||
// in order.
|
// in order.
|
||||||
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||||
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
|
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
@@ -218,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params ht
|
|||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.makeRoom(r.ContentLength)
|
||||||
|
|
||||||
query := r.URL.Query()
|
query := r.URL.Query()
|
||||||
switch strings.ToLower(query.Get("comp")) {
|
switch strings.ToLower(query.Get("comp")) {
|
||||||
|
|||||||
@@ -14,13 +14,12 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// v2Call posts a twirp request to the cache service and returns the decoded response.
|
|
||||||
// Field names are the proto ones, which is what the toolkit's client sends.
|
|
||||||
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -37,7 +36,6 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
|
|||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
|
|
||||||
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
|
|
||||||
func putBlob(t *testing.T, url string, content []byte) int {
|
func putBlob(t *testing.T, url string, content []byte) int {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -47,8 +45,6 @@ func putBlob(t *testing.T, url string, content []byte) int {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode == http.StatusCreated {
|
if resp.StatusCode == http.StatusCreated {
|
||||||
// The Azure SDK client dereferences this header without checking, so a blob upload that
|
|
||||||
// omits it panics the caller rather than failing it.
|
|
||||||
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
||||||
}
|
}
|
||||||
return resp.StatusCode
|
return resp.StatusCode
|
||||||
@@ -67,8 +63,6 @@ func getURL(t *testing.T, url string) []byte {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
|
|
||||||
// with the upload URL it used.
|
|
||||||
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -84,24 +78,22 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
|||||||
}), uploadURL
|
}), uploadURL
|
||||||
}
|
}
|
||||||
|
|
||||||
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
|
|
||||||
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
|
|
||||||
// or to replace a finalized entry.
|
|
||||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||||
|
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||||
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||||
|
|
||||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
assert.NotEmpty(t, finalized["entry_id"])
|
assert.NotEmpty(t, finalized["entry_id"])
|
||||||
|
|
||||||
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
|
|
||||||
// and it is an upload URL only: nothing reads a blob back through it.
|
|
||||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
||||||
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
|
resp, err := http.Get(uploadURL)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
||||||
@@ -114,9 +106,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
assert.Equal(t, content, getURL(t, downloadURL))
|
assert.Equal(t, content, getURL(t, downloadURL))
|
||||||
}
|
}
|
||||||
|
|
||||||
// A large archive is staged as blocks and only put in order by the final block list, so
|
|
||||||
// blocks that arrive out of order must still be assembled the way the client asked.
|
|
||||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||||
@@ -130,7 +121,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
blocks[blockID] = []byte(part)
|
blocks[blockID] = []byte(part)
|
||||||
order = append(order, blockID)
|
order = append(order, blockID)
|
||||||
}
|
}
|
||||||
// Upload in an order that is not the block list order.
|
|
||||||
for _, blockID := range []string{order[2], order[0], order[1]} {
|
for _, blockID := range []string{order[2], order[0], order[1]} {
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
||||||
}
|
}
|
||||||
@@ -165,6 +155,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||||
require.Equal(t, true, saved["ok"])
|
require.Equal(t, true, saved["ok"])
|
||||||
@@ -174,8 +165,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The toolkit serialises with the proto field names; the camelCase spellings of the same
|
|
||||||
// proto JSON mapping are accepted alongside them.
|
|
||||||
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
||||||
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
||||||
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
||||||
@@ -191,9 +180,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, again["ok"])
|
assert.Equal(t, false, again["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// A key that is only a prefix of an existing one is a different entry, so the
|
|
||||||
// reservation check must be exact and not a restore-key prefix match, or the shorter
|
|
||||||
// key would be reported as existing and silently never saved.
|
|
||||||
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
||||||
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
||||||
require.Equal(t, true, reserved["ok"])
|
require.Equal(t, true, reserved["ok"])
|
||||||
@@ -219,7 +205,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The size the client declares is what Commit validates the assembled archive against.
|
|
||||||
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
||||||
@@ -230,8 +215,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// Both API versions are served from one store, so an entry written through v2 is a hit for
|
|
||||||
// a v1 client asking for the same key and version.
|
|
||||||
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
||||||
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -244,7 +227,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.NotEmpty(t, got["archiveLocation"])
|
assert.NotEmpty(t, got["archiveLocation"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The cache of one repository must stay invisible to another, as it does for the v1 API.
|
|
||||||
t.Run("another repository sees nothing", func(t *testing.T) {
|
t.Run("another repository sees nothing", func(t *testing.T) {
|
||||||
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
||||||
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
||||||
@@ -252,9 +234,28 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||||
|
clients := map[string]*http.Client{"x": testClient}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||||
|
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||||
|
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
require.Equal(t, true, created["ok"])
|
||||||
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||||
|
}
|
||||||
|
for _, ref := range []string{"y", "z"} {
|
||||||
|
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||||
|
}
|
||||||
|
for content, client := range clients {
|
||||||
|
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||||
|
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -291,6 +292,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -321,20 +323,12 @@ func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
|||||||
"another job's upload is not this job's to finalize")
|
"another job's upload is not this job's to finalize")
|
||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
reserveV1, err := json.Marshal(&Request{Key: "v1-made", Version: "v1", Size: int64(len(content))})
|
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
|
||||||
require.NoError(t, err)
|
|
||||||
resp, err := testClient.Post(base+"/caches", "application/json", bytes.NewReader(reserveV1))
|
|
||||||
require.NoError(t, err)
|
|
||||||
var madeByV1 struct {
|
|
||||||
CacheID uint64 `json:"cacheId"`
|
|
||||||
}
|
|
||||||
require.NoError(t, json.UnmarshalRead(resp.Body, &madeByV1))
|
|
||||||
resp.Body.Close()
|
|
||||||
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
||||||
fmt.Sprintf("%s/caches/%d", base, madeByV1.CacheID), bytes.NewReader(content))
|
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
||||||
resp, err = testClient.Do(upload)
|
resp, err := testClient.Do(upload)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
Key string `json:"key" `
|
Key string `json:"key" `
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
@@ -11,13 +13,19 @@ type Request struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Cache struct {
|
type Cache struct {
|
||||||
ID uint64 `json:"id" boltholdKey:"ID"`
|
ID uint64 `json:"id"`
|
||||||
Repo string `json:"repo" boltholdIndex:"Repo"`
|
Repo string `json:"repo"`
|
||||||
Key string `json:"key" boltholdIndex:"Key"`
|
Ref string `json:"ref,omitzero"`
|
||||||
Version string `json:"version" boltholdIndex:"Version"`
|
Key string `json:"key"`
|
||||||
Size int64 `json:"cacheSize"`
|
Version string `json:"version"`
|
||||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
Size int64 `json:"cacheSize"`
|
||||||
Complete bool `json:"complete" boltholdIndex:"Complete"`
|
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||||
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
|
Complete bool `json:"complete"`
|
||||||
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
|
UsedAt int64 `json:"usedAt"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cache *Cache) createdAtNano() int64 {
|
||||||
|
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"cmp"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -66,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if cred.InsecureTLS {
|
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||||
proxy.Transport = insecureTransport
|
proxy.Transport = insecureTransport
|
||||||
}
|
}
|
||||||
proxy.ServeHTTP(w, r)
|
proxy.ServeHTTP(w, r)
|
||||||
@@ -74,14 +75,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
||||||
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
||||||
if cred.InsecureTLS && target.Scheme == "https" {
|
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
|
||||||
return true
|
|
||||||
}
|
|
||||||
scheme := r.Header.Get("X-Forwarded-Proto")
|
|
||||||
if scheme == "" {
|
|
||||||
scheme = "http"
|
|
||||||
}
|
|
||||||
return scheme != target.Scheme
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeQuery(target, request string) string {
|
func mergeQuery(target, request string) string {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFrontResultsService(t *testing.T) {
|
func TestFrontResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var gotHost, gotPath, gotProto string
|
var gotHost, gotPath, gotProto string
|
||||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||||
@@ -132,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||||
|
defer gitea.Close()
|
||||||
|
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer handler.Close()
|
||||||
|
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
}
|
||||||
@@ -24,6 +24,10 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
// S3Options configures the bucket that stores completed cache entries.
|
// S3Options configures the bucket that stores completed cache entries.
|
||||||
@@ -47,7 +51,7 @@ type cacheStorage interface {
|
|||||||
Remove(uint64) error
|
Remove(uint64) error
|
||||||
}
|
}
|
||||||
|
|
||||||
const s3PartSize = 5 << 30 // the largest part S3 accepts
|
const s3PartSize = 64 << 20
|
||||||
|
|
||||||
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
||||||
type s3Storage struct {
|
type s3Storage struct {
|
||||||
@@ -69,6 +73,7 @@ func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
|
|||||||
}
|
}
|
||||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||||
transport = transport.Clone()
|
transport = transport.Clone()
|
||||||
|
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
|
||||||
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
||||||
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
||||||
if opts.PathStyle != nil {
|
if opts.PathStyle != nil {
|
||||||
@@ -209,26 +214,43 @@ func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// upload always goes in parts, the only way S3 takes an object over 5 GiB.
|
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
|
||||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) error {
|
if size <= s3PartSize {
|
||||||
|
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
var created struct {
|
var created struct {
|
||||||
UploadID string `xml:"UploadId"`
|
UploadID string `xml:"UploadId"`
|
||||||
}
|
}
|
||||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
if uploadErr != nil && created.UploadID != "" {
|
||||||
|
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
|
||||||
|
}
|
||||||
|
}()
|
||||||
var completion struct {
|
var completion struct {
|
||||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||||
Parts []s3Part `xml:"Part"`
|
Parts []s3Part `xml:"Part"`
|
||||||
}
|
}
|
||||||
for offset := int64(0); offset == 0 || offset < size; offset += s3PartSize {
|
partSize := max(int64(s3PartSize), (size-1)/10000+1)
|
||||||
number, length := len(completion.Parts)+1, min(s3PartSize, size-offset)
|
completion.Parts = make([]s3Part, (size-1)/partSize+1)
|
||||||
query := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {created.UploadID}}
|
workers, workerCtx := errgroup.WithContext(ctx)
|
||||||
header, err := s.call(ctx, http.MethodPut, key, query, io.NewSectionReader(file, offset, length), nil)
|
workers.SetLimit(4)
|
||||||
if err != nil {
|
for part := range completion.Parts {
|
||||||
|
workers.Go(func() (err error) {
|
||||||
|
defer common.RecoverPanic(&err)
|
||||||
|
offset := int64(part) * partSize
|
||||||
|
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
|
||||||
|
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
|
||||||
return err
|
return err
|
||||||
}
|
})
|
||||||
completion.Parts = append(completion.Parts, s3Part{number, header.Get("ETag")})
|
}
|
||||||
|
if err := workers.Wait(); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
body, err := xml.Marshal(completion)
|
body, err := xml.Marshal(completion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,40 +4,135 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
|
||||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||||
backend := s3mem.New()
|
t.Parallel()
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
var objects sync.Map
|
||||||
server := httptest.NewServer(gofakes3.New(backend).Server())
|
var uploads atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
key := strings.TrimPrefix(request.URL.Path, "/cache/")
|
||||||
|
switch {
|
||||||
|
case request.Method == http.MethodPut:
|
||||||
|
if strings.Contains(key, "/blobs/") {
|
||||||
|
uploads.Add(1)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(request.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
objects.Store(key, body)
|
||||||
|
case request.URL.Query().Has("list-type"):
|
||||||
|
_, _ = io.WriteString(writer, "<ListBucketResult>")
|
||||||
|
objects.Range(func(stored, _ any) bool {
|
||||||
|
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
_, _ = io.WriteString(writer, "</ListBucketResult>")
|
||||||
|
default:
|
||||||
|
if body, ok := objects.Load(key); ok {
|
||||||
|
_, _ = writer.Write(body.([]byte))
|
||||||
|
} else {
|
||||||
|
writer.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
||||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
|
assert.EqualValues(t, 1, uploads.Load())
|
||||||
|
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
cache, err := saver.s3.fetch(t.Context(), id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||||
require.Equal(t, true, found["ok"])
|
require.Equal(t, true, found["ok"])
|
||||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
require.NoError(t, backend.ForceDeleteBucket("cache"))
|
for _, cache := range []*Cache{
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||||
|
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||||
|
} {
|
||||||
|
require.NoError(t, saver.s3.publish(cache))
|
||||||
|
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||||
|
}
|
||||||
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
|
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||||
|
require.Equal(t, true, found["ok"])
|
||||||
|
assert.Equal(t, "deps-new", found["matched_key"])
|
||||||
|
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
|
objects.Clear()
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type s3TestTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
return transport(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, failingPart := range []string{"", "2"} {
|
||||||
|
var parts, aborts atomic.Int32
|
||||||
|
var completion struct {
|
||||||
|
Parts []s3Part `xml:"Part"`
|
||||||
|
}
|
||||||
|
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
|
||||||
|
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
|
||||||
|
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
|
||||||
|
switch {
|
||||||
|
case query.Has("uploads"):
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
|
||||||
|
case req.Method == http.MethodPut:
|
||||||
|
parts.Add(1)
|
||||||
|
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
|
||||||
|
if query.Get("partNumber") == failingPart {
|
||||||
|
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
|
||||||
|
}
|
||||||
|
case req.Method == http.MethodPost:
|
||||||
|
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
|
||||||
|
case req.Method == http.MethodDelete:
|
||||||
|
aborts.Add(1)
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
|
||||||
|
assert.EqualValues(t, 3, parts.Load())
|
||||||
|
if failingPart == "" {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
|
||||||
|
assert.Zero(t, aborts.Load())
|
||||||
|
} else {
|
||||||
|
require.ErrorContains(t, err, "InvalidPart")
|
||||||
|
assert.EqualValues(t, 1, aborts.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -32,13 +33,11 @@ func NewStorage(rootDir string) (*Storage, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Exist(id uint64) (bool, error) {
|
func (s *Storage) Exist(id uint64) (bool, error) {
|
||||||
name := s.filename(id)
|
_, err := os.Stat(s.filename(id))
|
||||||
if _, err := os.Stat(name); os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
} else if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
}
|
||||||
return true, nil
|
return err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
||||||
@@ -53,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(file, reader)
|
_, err = io.Copy(file, reader)
|
||||||
|
if err = errors.Join(err, file.Close()); err != nil {
|
||||||
|
_ = os.Remove(name)
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,9 +92,7 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||||
defer func() {
|
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
|
||||||
_ = os.RemoveAll(s.tempDir(id))
|
|
||||||
}()
|
|
||||||
|
|
||||||
name := s.filename(id)
|
name := s.filename(id)
|
||||||
tempNames, err := s.tempNames(id)
|
tempNames, err := s.tempNames(id)
|
||||||
@@ -106,55 +104,50 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written, err := assemble(name, tempNames)
|
written, err := assemble(name, tempNames)
|
||||||
if err != nil {
|
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||||
return 0, err
|
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||||
}
|
}
|
||||||
// If size is less than 0, it means the size is unknown.
|
if err != nil {
|
||||||
// We can't check the size of the file, just skip the check.
|
|
||||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
|
||||||
if size >= 0 && written != size {
|
|
||||||
_ = os.Remove(name)
|
_ = os.Remove(name)
|
||||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
return 0, err
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
|
||||||
func assemble(name string, tempNames []string) (int64, error) {
|
func assemble(name string, tempNames []string) (int64, error) {
|
||||||
if len(tempNames) == 1 {
|
if len(tempNames) == 0 {
|
||||||
info, err := os.Stat(tempNames[0])
|
return 0, errors.New("nothing was uploaded")
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return info.Size(), os.Rename(tempNames[0], name)
|
|
||||||
}
|
}
|
||||||
|
if err := os.Rename(tempNames[0], name); err != nil {
|
||||||
file, err := os.Create(name)
|
return 0, err
|
||||||
|
}
|
||||||
|
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
for _, v := range tempNames[1:] {
|
||||||
var written int64
|
part, err := os.Open(v)
|
||||||
for _, v := range tempNames {
|
|
||||||
f, err := os.Open(v)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
n, err := io.Copy(file, f)
|
_, err = io.Copy(file, part)
|
||||||
_ = f.Close()
|
_ = part.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written += n
|
_ = os.Remove(v)
|
||||||
}
|
}
|
||||||
return written, nil
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return info.Size(), file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||||
name := s.filename(id)
|
http.ServeFile(w, r, s.filename(id))
|
||||||
http.ServeFile(w, r, name)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
||||||
|
|||||||
+39
-38
@@ -13,14 +13,13 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileContainerResourceURL struct {
|
type FileContainerResourceURL struct {
|
||||||
@@ -67,39 +66,44 @@ func writeJSON(w http.ResponseWriter, value any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploads(router *httprouter.Router, baseDir string) {
|
func uploads(router *http.ServeMux, baseDir string) {
|
||||||
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
writeJSON(w, FileContainerResourceURL{
|
writeJSON(w, FileContainerResourceURL{
|
||||||
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
|
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
if req.Header.Get("Content-Encoding") == "gzip" {
|
if req.Header.Get("Content-Encoding") == "gzip" {
|
||||||
itemPath += gzipExtension
|
itemPath += gzipExtension
|
||||||
}
|
}
|
||||||
|
|
||||||
safeRunPath := safeResolve(baseDir, runID)
|
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
|
||||||
safePath := safeResolve(safeRunPath, itemPath)
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
var start int64
|
||||||
appendUpload := req.Header.Get("Content-Range")
|
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if start == 0 {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
}
|
}
|
||||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
if req.Body == nil {
|
if req.Body == nil {
|
||||||
panic(errors.New("no body given"))
|
panic(errors.New("no body given"))
|
||||||
}
|
}
|
||||||
@@ -114,21 +118,19 @@ func uploads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
|
||||||
writeJSON(w, ResponseMessage{
|
writeJSON(w, ResponseMessage{
|
||||||
Message: "success",
|
Message: "success",
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloads(router *httprouter.Router, baseDir string) {
|
func downloads(router *http.ServeMux, baseDir string) {
|
||||||
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
runID := req.PathValue("runId")
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, runID)
|
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
entries, err := os.ReadDir(safePath)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,13 +148,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
container := params.ByName("container")
|
container := req.PathValue("container")
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
||||||
|
|
||||||
var files []ContainerItem
|
var files []ContainerItem
|
||||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !entry.IsDir() {
|
if !entry.IsDir() {
|
||||||
rel, err := filepath.Rel(safePath, path)
|
rel, err := filepath.Rel(safePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -161,20 +166,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
|
|
||||||
// if it was upload as gzip
|
// if it was upload as gzip
|
||||||
rel = strings.TrimSuffix(rel, gzipExtension)
|
rel = strings.TrimSuffix(rel, gzipExtension)
|
||||||
path := filepath.Join(itemPath, rel)
|
|
||||||
|
|
||||||
rel = filepath.ToSlash(rel)
|
|
||||||
path = filepath.ToSlash(path)
|
|
||||||
|
|
||||||
files = append(files, ContainerItem{
|
files = append(files, ContainerItem{
|
||||||
Path: path,
|
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||||
ItemType: "file",
|
ItemType: "file",
|
||||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
|
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,10 +184,8 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
path := params.ByName("path")[1:]
|
safePath := safeResolve(baseDir, req.PathValue("path"))
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, path)
|
|
||||||
|
|
||||||
file, err := os.Open(safePath)
|
file, err := os.Open(safePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -214,11 +213,12 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
return cancel
|
return cancel
|
||||||
}
|
}
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
|
|
||||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: net.JoinHostPort(addr, port),
|
Addr: net.JoinHostPort(addr, port),
|
||||||
@@ -230,7 +230,8 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
go func() {
|
go func() {
|
||||||
logger.Infof("Start server on http://%s", server.Addr)
|
logger.Infof("Start server on http://%s", server.Addr)
|
||||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
logger.Fatal(err)
|
logger.Error(err)
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
+114
-10
@@ -19,16 +19,16 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFlow(t *testing.T) {
|
func TestArtifactFlow(t *testing.T) {
|
||||||
artifactPath := t.TempDir()
|
artifactPath := t.TempDir()
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
server := httptest.NewServer(router)
|
server := httptest.NewServer(router)
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -49,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
return resp.StatusCode, data
|
return resp.StatusCode, data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("no-artifacts", func(t *testing.T) {
|
||||||
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
var list NamedFileContainerResourceURLResponse
|
||||||
|
require.NoError(t, json.Unmarshal(data, &list))
|
||||||
|
require.Zero(t, list.Count)
|
||||||
|
require.Empty(t, list.Value)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("upload-and-download", func(t *testing.T) {
|
t.Run("upload-and-download", func(t *testing.T) {
|
||||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||||
|
|
||||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
@@ -82,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(data, &items))
|
require.NoError(t, json.Unmarshal(data, &items))
|
||||||
require.Len(t, items.Value, 1)
|
require.Len(t, items.Value, 1)
|
||||||
require.Equal(t, "file", items.Value[0].ItemType)
|
require.Equal(t, "file", items.Value[0].ItemType)
|
||||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
require.Equal(t, item, items.Value[0].Path)
|
||||||
|
|
||||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(stored))
|
require.Equal(t, content, string(stored))
|
||||||
})
|
})
|
||||||
@@ -96,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
t.Run("content-range", func(t *testing.T) {
|
t.Run("content-range", func(t *testing.T) {
|
||||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
|
||||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
for range 2 {
|
||||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "first-second", string(stored))
|
require.Equal(t, "first-second", string(stored))
|
||||||
|
|
||||||
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||||
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, stored)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||||
@@ -157,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("v4", func(t *testing.T) {
|
||||||
|
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||||
|
call := func(method, body string) (int, map[string]any) {
|
||||||
|
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||||
|
var out map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||||
|
return status, out
|
||||||
|
}
|
||||||
|
put := func(rawURL, body string) int {
|
||||||
|
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
download := func() (string, http.Header) {
|
||||||
|
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, signed)
|
||||||
|
resp, err := client.Get(signed["signedUrl"].(string))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(body), resp.Header
|
||||||
|
}
|
||||||
|
|
||||||
|
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
upload := created["signedUploadUrl"].(string)
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||||
|
for attempt := range 2 {
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusBadRequest, status)
|
||||||
|
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status, finalized)
|
||||||
|
id := finalized["artifactId"].(string)
|
||||||
|
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
require.Equal(t, id, refinalized["artifactId"])
|
||||||
|
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusConflict, status)
|
||||||
|
require.Equal(t, "already_exists", conflict["code"])
|
||||||
|
|
||||||
|
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
artifacts := listed["artifacts"].([]any)
|
||||||
|
require.Len(t, artifacts, 1)
|
||||||
|
artifact := artifacts[0].(map[string]any)
|
||||||
|
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||||
|
delete(artifact, "createdAt")
|
||||||
|
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||||
|
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||||
|
require.Empty(t, other["artifacts"])
|
||||||
|
|
||||||
|
body, header := download()
|
||||||
|
require.Equal(t, "hello world", body)
|
||||||
|
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusNotFound, status)
|
||||||
|
require.Equal(t, "not_found", missing["code"])
|
||||||
|
|
||||||
|
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||||
|
require.Equal(t, http.StatusCreated, status)
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
body, header = download()
|
||||||
|
require.Equal(t, "gz", body)
|
||||||
|
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
rerun := http.NewServeMux()
|
||||||
|
registerV4(rerun, artifactPath)
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||||
|
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSafeResolve(t *testing.T) {
|
func TestSafeResolve(t *testing.T) {
|
||||||
@@ -177,7 +281,7 @@ func TestSafeResolve(t *testing.T) {
|
|||||||
|
|
||||||
for name, tc := range tests {
|
for name, tc := range tests {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||||
|
dir string
|
||||||
|
mu sync.Mutex
|
||||||
|
lastID int64
|
||||||
|
created map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
zipMimeType = "application/zip"
|
||||||
|
maxTwirpBytes = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerV4(router *http.ServeMux, baseDir string) {
|
||||||
|
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||||
|
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||||
|
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
type twirpError struct {
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *twirpError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
func notFound(name string) error {
|
||||||
|
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(format string, args ...any) error {
|
||||||
|
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req Req
|
||||||
|
var resp Resp
|
||||||
|
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||||
|
if err != nil {
|
||||||
|
err = badRequest("malformed request: %v", err)
|
||||||
|
} else {
|
||||||
|
resp, err = handle(r, req)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err != nil {
|
||||||
|
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||||
|
errors.As(err, &terr)
|
||||||
|
w.WriteHeader(terr.status)
|
||||||
|
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.MarshalWrite(w, resp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||||
|
sum := sha256.Sum256([]byte(name))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) runDir(runID string) string {
|
||||||
|
return filepath.Join(s.dir, fileName(runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) artifactDir(runID, name string) string {
|
||||||
|
return filepath.Join(s.runDir(runID), fileName(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) urlDir(r *http.Request) string {
|
||||||
|
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMeta(dir string) (*monolithArtifact, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var meta monolithArtifact
|
||||||
|
return &meta, json.Unmarshal(data, &meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||||
|
data, err := json.Marshal(meta)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||||
|
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.created[dir] {
|
||||||
|
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.lastID++
|
||||||
|
if err := writeMeta(dir, &monolithArtifact{
|
||||||
|
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||||
|
DatabaseID: s.lastID,
|
||||||
|
Name: req.Name,
|
||||||
|
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||||
|
}); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.created[dir] = true
|
||||||
|
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockList struct {
|
||||||
|
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||||
|
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||||
|
var err error
|
||||||
|
switch comp := r.URL.Query().Get("comp"); comp {
|
||||||
|
case "block":
|
||||||
|
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||||
|
case "blocklist":
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||||
|
case "": // Put Blob, which docker/build-push-action uses
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||||
|
_, err := io.Copy(out, r.Body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(name string, body io.Reader) error {
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, body)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||||
|
out, err := os.CreateTemp(dir, "blob")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||||
|
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(out.Name()))
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||||
|
var list blockList
|
||||||
|
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range list.Blocks {
|
||||||
|
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendBlock(out io.Writer, name string) error {
|
||||||
|
in, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
} else if meta.Digest != "" {
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
size, err := io.Copy(hash, blob)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if size != req.Size {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||||
|
}
|
||||||
|
if req.Hash != "" && req.Hash != digest {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||||
|
}
|
||||||
|
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
meta.Size, meta.Digest = size, digest
|
||||||
|
if err := writeMeta(dir, meta); err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||||
|
run := s.runDir(req.WorkflowRunBackendID)
|
||||||
|
entries, err := os.ReadDir(run)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return listArtifactsResponse{}, err
|
||||||
|
}
|
||||||
|
var resp listArtifactsResponse
|
||||||
|
for _, entry := range entries {
|
||||||
|
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||||
|
if err != nil || meta.Digest == "" ||
|
||||||
|
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||||
|
}
|
||||||
|
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||||
|
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||||
|
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil || meta.Digest == "" {
|
||||||
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||||
|
contentType, filename := string(data), meta.Name
|
||||||
|
if !strings.Contains(contentType, "/") {
|
||||||
|
contentType = zipMimeType
|
||||||
|
}
|
||||||
|
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||||
|
filename += ".zip"
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||||
|
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return deleteArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
delete(s.created, dir)
|
||||||
|
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||||
|
now := time.Now()
|
||||||
|
claims, _ := json.Marshal(map[string]any{
|
||||||
|
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||||
|
"ac": `[{"Scope":"","Permission":3}]`,
|
||||||
|
"nbf": now.Unix(),
|
||||||
|
"exp": now.Add(lifetime).Unix(),
|
||||||
|
"jti": rand.Text(),
|
||||||
|
})
|
||||||
|
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
|
||||||
|
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||||
|
createArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
SignedUploadURL string `json:"signedUploadUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeArtifactRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
Size int64 `json:",string"`
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
finalizeArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
ArtifactID int64 `json:"artifactId,string"`
|
||||||
|
}
|
||||||
|
|
||||||
|
listArtifactsRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
NameFilter string
|
||||||
|
IDFilter int64 `json:",string"`
|
||||||
|
}
|
||||||
|
listArtifactsResponse struct {
|
||||||
|
Artifacts []monolithArtifact `json:"artifacts"`
|
||||||
|
}
|
||||||
|
monolithArtifact struct {
|
||||||
|
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||||
|
DatabaseID int64 `json:"databaseId,string"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size int64 `json:"size,string"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||||
|
Digest string `json:"digest"` // empty until the upload is finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
getSignedArtifactURLRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
getSignedArtifactURLResponse struct {
|
||||||
|
SignedURL string `json:"signedUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
createArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactResponse = finalizeArtifactResponse
|
||||||
|
)
|
||||||
@@ -4,15 +4,15 @@
|
|||||||
package common
|
package common
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDryrunContext(t *testing.T) {
|
func TestDryrunContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Dryrun(ctx) {
|
if Dryrun(ctx) {
|
||||||
t.Fatal("plain context should not be dryrun")
|
t.Fatal("plain context should not be dryrun")
|
||||||
}
|
}
|
||||||
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestJobErrorContainer(t *testing.T) {
|
func TestJobErrorContainer(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
err := errors.New("job failed")
|
err := errors.New("job failed")
|
||||||
|
|
||||||
SetJobError(ctx, err)
|
SetJobError(ctx, err)
|
||||||
@@ -40,34 +40,27 @@ func TestJobErrorContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoggerAndHookContext(t *testing.T) {
|
func TestLoggerAndHandlerContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Logger(ctx) != logrus.StandardLogger() {
|
if Logger(ctx) != logging.Default() {
|
||||||
t.Fatal("plain context should use standard logger")
|
t.Fatal("plain context should use the default logger")
|
||||||
}
|
}
|
||||||
if LoggerHook(ctx) != nil {
|
if LoggerHandler(ctx) != nil {
|
||||||
t.Fatal("plain context should not have a logger hook")
|
t.Fatal("plain context should not have a logger handler")
|
||||||
|
}
|
||||||
|
if StepIDs(ctx) != nil {
|
||||||
|
t.Fatal("plain context should not have step IDs")
|
||||||
}
|
}
|
||||||
|
|
||||||
logger := logrus.New()
|
logger := logging.New(logtest.New())
|
||||||
ctx = WithLogger(ctx, logger)
|
ctx = WithLogger(ctx, logger)
|
||||||
if Logger(ctx) != logger {
|
if Logger(ctx) != logger {
|
||||||
t.Fatal("WithLogger should set logger")
|
t.Fatal("WithLogger should set logger")
|
||||||
}
|
}
|
||||||
|
|
||||||
hook := testHook{}
|
handler := logtest.New()
|
||||||
ctx = WithLoggerHook(ctx, hook)
|
ctx = WithLoggerHandler(ctx, handler)
|
||||||
if LoggerHook(ctx) != hook {
|
if LoggerHandler(ctx) != handler {
|
||||||
t.Fatal("WithLoggerHook should set hook")
|
t.Fatal("WithLoggerHandler should set handler")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type testHook struct{}
|
|
||||||
|
|
||||||
func (testHook) Levels() []logrus.Level {
|
|
||||||
return logrus.AllLevels
|
|
||||||
}
|
|
||||||
|
|
||||||
func (testHook) Fire(*logrus.Entry) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -14,13 +14,8 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
|
|||||||
|
|
||||||
// Dryrun returns true if the current context is dryrun
|
// Dryrun returns true if the current context is dryrun
|
||||||
func Dryrun(ctx context.Context) bool {
|
func Dryrun(ctx context.Context) bool {
|
||||||
val := ctx.Value(dryrunContextKeyVal)
|
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
|
||||||
if val != nil {
|
return dryrun
|
||||||
if dryrun, ok := val.(bool); ok {
|
|
||||||
return dryrun
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithDryrun adds a value to the context for dryrun
|
// WithDryrun adds a value to the context for dryrun
|
||||||
|
|||||||
+49
-29
@@ -7,9 +7,11 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/rand/v2"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
"time"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Executor define contract for the steps of a workflow
|
// Executor define contract for the steps of a workflow
|
||||||
@@ -21,8 +23,7 @@ type Conditional func(ctx context.Context) bool
|
|||||||
// NewInfoExecutor is an executor that logs messages
|
// NewInfoExecutor is an executor that logs messages
|
||||||
func NewInfoExecutor(format string, args ...any) Executor {
|
func NewInfoExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Infof(format, args...)
|
||||||
logger.Infof(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -30,8 +31,7 @@ func NewInfoExecutor(format string, args ...any) Executor {
|
|||||||
// NewDebugExecutor is an executor that logs messages
|
// NewDebugExecutor is an executor that logs messages
|
||||||
func NewDebugExecutor(format string, args ...any) Executor {
|
func NewDebugExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Debugf(format, args...)
|
||||||
logger.Debugf(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -39,9 +39,7 @@ func NewDebugExecutor(format string, args ...any) Executor {
|
|||||||
// NewPipelineExecutor creates a new executor from a series of other executors
|
// NewPipelineExecutor creates a new executor from a series of other executors
|
||||||
func NewPipelineExecutor(executors ...Executor) Executor {
|
func NewPipelineExecutor(executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
return func(ctx context.Context) error {
|
return NewErrorExecutor(nil)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
var rtn Executor
|
var rtn Executor
|
||||||
for _, executor := range executors {
|
for _, executor := range executors {
|
||||||
@@ -61,6 +59,13 @@ func NewErrorExecutor(err error) Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
|
||||||
|
if recovered := recover(); recovered != nil {
|
||||||
|
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
|
||||||
|
*err = fmt.Errorf("panic: %v", recovered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewParallelExecutor creates a new executor from a parallel of other executors
|
// NewParallelExecutor creates a new executor from a parallel of other executors
|
||||||
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
@@ -74,34 +79,29 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
|||||||
errs := make(chan error, len(executors))
|
errs := make(chan error, len(executors))
|
||||||
|
|
||||||
if 1 > parallel {
|
if 1 > parallel {
|
||||||
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
||||||
parallel = 1
|
parallel = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||||
|
|
||||||
for i := 0; i < parallel; i++ {
|
for workerID := range parallel {
|
||||||
go func(workerID int, work <-chan Executor, errs chan<- error) {
|
go func() {
|
||||||
log.Debugf("Worker %d started", workerID)
|
logging.Debugf("Worker %d started", workerID)
|
||||||
taskCount := 0
|
taskCount := 0
|
||||||
for executor := range work {
|
for executor := range work {
|
||||||
taskCount++
|
taskCount++
|
||||||
log.Debugf("Worker %d executing task %d", workerID, taskCount)
|
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
|
||||||
// Recover from panics in executors to avoid crashing the worker
|
// Recover from panics in executors to avoid crashing the worker
|
||||||
// goroutine which would leave the runner process hung.
|
// goroutine which would leave the runner process hung.
|
||||||
// https://gitea.com/gitea/runner/issues/371
|
// https://gitea.com/gitea/runner/issues/371
|
||||||
errs <- func() (err error) {
|
errs <- func() (err error) {
|
||||||
defer func() {
|
defer RecoverPanic(&err)
|
||||||
if r := recover(); r != nil {
|
|
||||||
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
|
|
||||||
err = fmt.Errorf("panic: %v", r)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return executor(ctx)
|
return executor(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||||
}(i, work, errs)
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range executors {
|
for i := range executors {
|
||||||
@@ -150,12 +150,7 @@ func (e Executor) If(conditional Conditional) Executor {
|
|||||||
|
|
||||||
// IfNot only runs this executor if conditional is true
|
// IfNot only runs this executor if conditional is true
|
||||||
func (e Executor) IfNot(conditional Conditional) Executor {
|
func (e Executor) IfNot(conditional Conditional) Executor {
|
||||||
return func(ctx context.Context) error {
|
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
|
||||||
if !conditional(ctx) {
|
|
||||||
return e(ctx)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IfBool only runs this executor if conditional is true
|
// IfBool only runs this executor if conditional is true
|
||||||
@@ -165,13 +160,38 @@ func (e Executor) IfBool(conditional bool) Executor {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||||
|
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||||
|
|
||||||
|
// Retry runs the executor up to attempts times until it succeeds.
|
||||||
|
func (e Executor) Retry(attempts int) Executor {
|
||||||
|
return func(ctx context.Context) error {
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
err := e(ctx)
|
||||||
|
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backoff := RetryBackoff()
|
||||||
|
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return err
|
||||||
|
case <-time.After(backoff):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Finally adds an executor to run after other executor
|
// Finally adds an executor to run after other executor
|
||||||
func (e Executor) Finally(finally Executor) Executor {
|
func (e Executor) Finally(finally Executor) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
err := e(ctx)
|
err := e(ctx)
|
||||||
err2 := finally(ctx)
|
err2 := finally(ctx)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+90
-105
@@ -7,53 +7,36 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewWorkflow(t *testing.T) {
|
func TestNewWorkflow(t *testing.T) {
|
||||||
assert := assert.New(t)
|
ctx := t.Context()
|
||||||
|
require.NoError(t, NewPipelineExecutor()(ctx))
|
||||||
|
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// empty
|
|
||||||
emptyWorkflow := NewPipelineExecutor()
|
|
||||||
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// error case
|
|
||||||
errorWorkflow := NewErrorExecutor(errors.New("test error"))
|
|
||||||
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// multiple success case
|
|
||||||
runcount := 0
|
runcount := 0
|
||||||
successWorkflow := NewPipelineExecutor(
|
increment := func(_ctx context.Context) error {
|
||||||
func(ctx context.Context) error {
|
runcount++
|
||||||
runcount++
|
return nil
|
||||||
return nil
|
}
|
||||||
},
|
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
|
||||||
func(ctx context.Context) error {
|
assert.Equal(t, 2, runcount)
|
||||||
runcount++
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
assert.Equal(2, runcount)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
|
||||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
|
||||||
// find the gate already open so the last one still finishes with no partner left.
|
|
||||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||||
var counted, active, peak atomic.Int32
|
var counted, active, peak atomic.Int32
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
reached := make(chan struct{})
|
reached := make(chan struct{})
|
||||||
|
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
counted.Add(1)
|
counted.Add(1)
|
||||||
running := active.Add(1)
|
running := active.Add(1)
|
||||||
for {
|
for {
|
||||||
@@ -72,93 +55,72 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutor(t *testing.T) {
|
func TestNewParallelExecutor(t *testing.T) {
|
||||||
ctx := context.Background()
|
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
exec, count, maxActive := concurrencyProbe(2)
|
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
|
||||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
assert.Equal(t, int32(3), count.Load())
|
||||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
|
||||||
|
})
|
||||||
// parallelism below 1 falls back to a single worker
|
}
|
||||||
exec, count, maxActive = concurrencyProbe(1)
|
|
||||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
|
||||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
require.NoError(t, NewParallelExecutor(2)(ctx))
|
require.NoError(t, NewParallelExecutor(2)(ctx))
|
||||||
|
|
||||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
canceledCtx, cancel := context.WithCancel(t.Context())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
err := NewParallelExecutor(2)(canceledCtx)
|
err := NewParallelExecutor(2)(canceledCtx)
|
||||||
assert.ErrorIs(err, context.Canceled)
|
assert.ErrorIs(err, context.Canceled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorFailed(t *testing.T) {
|
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
|
||||||
assert := assert.New(t)
|
for name, tc := range map[string]struct {
|
||||||
|
canceled bool
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
parallel, tasks int
|
||||||
cancel()
|
}{
|
||||||
|
"failure": {false, 2, 5},
|
||||||
count := 0
|
"canceled": {true, 2, 5},
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
"canceled single task": {true, 1, 1},
|
||||||
count++
|
} {
|
||||||
return errors.New("fake error")
|
t.Run(name, func(t *testing.T) {
|
||||||
})
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
err := NewParallelExecutor(1, errorWorkflow)(ctx)
|
defer cancel()
|
||||||
assert.Equal(1, count)
|
if tc.canceled {
|
||||||
assert.ErrorIs(context.Canceled, err)
|
cancel()
|
||||||
}
|
}
|
||||||
|
var calls atomic.Int32
|
||||||
func TestNewParallelExecutorCanceled(t *testing.T) {
|
executors := make([]Executor, tc.tasks)
|
||||||
assert := assert.New(t)
|
for i := range executors {
|
||||||
|
executors[i] = func(_ctx context.Context) error {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
calls.Add(1)
|
||||||
cancel()
|
if i == tc.tasks/2 {
|
||||||
|
return errors.New("fake error")
|
||||||
errExpected := errors.New("fake error")
|
}
|
||||||
|
return nil
|
||||||
var count atomic.Int32
|
}
|
||||||
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return errExpected
|
|
||||||
})
|
|
||||||
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
|
|
||||||
assert.Equal(int32(3), count.Load())
|
|
||||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
|
||||||
var successCount atomic.Int32
|
|
||||||
executors := make([]Executor, 5)
|
|
||||||
for i := range executors {
|
|
||||||
executors[i] = func(ctx context.Context) error {
|
|
||||||
if i == 2 {
|
|
||||||
return errors.New("fake error")
|
|
||||||
}
|
}
|
||||||
successCount.Add(1)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
|
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
|
||||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
if tc.canceled {
|
||||||
|
require.ErrorIs(t, err, context.Canceled)
|
||||||
|
} else {
|
||||||
|
require.EqualError(t, err, "fake error")
|
||||||
|
}
|
||||||
|
assert.Equal(t, int32(tc.tasks), calls.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
var calls []string
|
var calls []string
|
||||||
record := func(name string) Executor {
|
record := func(name string) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
calls = append(calls, name)
|
calls = append(calls, name)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -170,19 +132,42 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
||||||
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
||||||
|
|
||||||
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
|
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||||
if !reflect.DeepEqual(calls, want) {
|
}
|
||||||
t.Fatalf("calls = %v, want %v", calls, want)
|
|
||||||
|
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||||
|
backoff := RetryBackoff
|
||||||
|
RetryBackoff = func() time.Duration { return 0 }
|
||||||
|
t.Cleanup(func() { RetryBackoff = backoff })
|
||||||
|
transient := errors.New("transient")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
failures, calls int
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{failures: 2, calls: 3},
|
||||||
|
{failures: 5, calls: 3, err: transient},
|
||||||
|
} {
|
||||||
|
var calls int
|
||||||
|
err := Executor(func(context.Context) error {
|
||||||
|
calls++
|
||||||
|
if calls <= tc.failures {
|
||||||
|
return transient
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}).Retry(3)(t.Context())
|
||||||
|
require.Equal(t, tc.err, err)
|
||||||
|
require.Equal(t, tc.calls, calls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
mainErr := errors.New("main failed")
|
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
|
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||||
require.Error(t, err)
|
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||||
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
|
require.ErrorIs(t, err, finalErr)
|
||||||
t.Fatalf("finally error = %q, want both cleanup and original error", err)
|
if mainErr != nil {
|
||||||
|
require.ErrorIs(t, err, mainErr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+182
-76
@@ -5,13 +5,18 @@
|
|||||||
package git
|
package git
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -21,16 +26,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||||
|
"gitea.com/gitea/runner/internal/pkg/process"
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
|
||||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
|
||||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||||
|
|
||||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||||
@@ -44,12 +47,29 @@ var (
|
|||||||
ErrShortRef = errors.New("short SHA references are not supported")
|
ErrShortRef = errors.New("short SHA references are not supported")
|
||||||
)
|
)
|
||||||
|
|
||||||
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
|
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
|
||||||
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
|
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
|
||||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
dir = filepath.Clean(dir)
|
||||||
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
|
if runtime.GOOS == "plan9" {
|
||||||
func AcquireCloneLock(dir string) func() {
|
return cloneLocks.Lock(dir), nil
|
||||||
return cloneLocks.Lock(dir)
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
release, err := lock.TryLock(dir)
|
||||||
|
if err == nil {
|
||||||
|
return func() { _ = release() }, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, lock.ErrLocked) {
|
||||||
|
return cloneLocks.Lock(dir), nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Error struct {
|
type Error struct {
|
||||||
@@ -62,25 +82,25 @@ func (e *Error) Unwrap() error { return e.err }
|
|||||||
func (e *Error) Commit() string { return e.commit }
|
func (e *Error) Commit() string { return e.commit }
|
||||||
|
|
||||||
// FindGitRevision get the current git revision
|
// FindGitRevision get the current git revision
|
||||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
func FindGitRevision(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||||
return "", "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if len(sha) < 7 {
|
if len(sha) < 7 {
|
||||||
return "", "", errors.New("HEAD could not be resolved")
|
return "", errors.New("HEAD could not be resolved")
|
||||||
}
|
}
|
||||||
logger.Debugf("Found revision: %s", sha)
|
logger.Debugf("Found revision: %s", sha)
|
||||||
return sha[:7], sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindGitRef get the current git ref
|
// FindGitRef get the current git ref
|
||||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
logger.Debugf("Loading revision from git directory")
|
logger.Debugf("Loading revision from git directory")
|
||||||
_, revision, err := FindGitRevision(ctx, file)
|
revision, err := FindGitRevision(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -109,13 +129,12 @@ func FindGitRef(ctx context.Context, file string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FindGithubRepo get the repo
|
// FindGithubRepo get the repo
|
||||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
func FindGithubRepo(ctx context.Context, file string) (string, error) {
|
||||||
url, err := findGitRemoteURL(ctx, file)
|
url, err := findGitRemoteURL(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
_, slug := findGitSlug(url, githubInstance)
|
return findGitSlug(url)
|
||||||
return slug, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||||
@@ -130,25 +149,25 @@ func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
|||||||
return url, nil
|
return url, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitSlug(url, githubInstance string) (string, string) {
|
func findGitSlug(remoteURL string) (string, error) {
|
||||||
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
|
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
|
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if githubInstance != "github.com" {
|
|
||||||
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
|
|
||||||
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
|
|
||||||
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return "", url
|
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
|
||||||
|
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(remoteURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("cannot parse the origin remote URL")
|
||||||
|
}
|
||||||
|
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
|
||||||
|
owner = path.Base(owner)
|
||||||
|
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
|
||||||
|
return "", errors.New("cannot determine owner and repository from the origin remote URL")
|
||||||
|
}
|
||||||
|
return owner + "/" + repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
||||||
@@ -163,17 +182,15 @@ type NewGitCloneExecutorInput struct {
|
|||||||
// 0 for full clone.
|
// 0 for full clone.
|
||||||
Depth int
|
Depth int
|
||||||
|
|
||||||
// Quiet drops the informational clone line to debug level, for callers that log their own
|
|
||||||
// download summary (the setup section's action report).
|
|
||||||
Quiet bool
|
|
||||||
|
|
||||||
// For Gitea
|
// For Gitea
|
||||||
InsecureSkipTLS bool
|
InsecureSkipTLS bool
|
||||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||||
|
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||||
|
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneIfRequired reports whether an existing local clone was reused.
|
// CloneIfRequired reports whether an existing local clone was reused.
|
||||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) {
|
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
|
||||||
var origin string
|
var origin string
|
||||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -182,6 +199,9 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
if err == nil && origin == input.URL {
|
if err == nil && origin == input.URL {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
|
||||||
|
return false, ctxErr
|
||||||
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||||
@@ -203,32 +223,18 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
if input.Quiet {
|
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||||
logger.Debugf("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
} else {
|
|
||||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
}
|
|
||||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||||
defer AcquireCloneLock(input.Dir)()
|
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
reused, err := CloneIfRequired(ctx, input, logger)
|
resolved, reused, err := resolveClone(ctx, input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
|
||||||
if err := fetch(ctx, input); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||||
}
|
}
|
||||||
@@ -242,6 +248,28 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
|
||||||
|
logger := common.Logger(ctx)
|
||||||
|
reused, err := CloneIfRequired(ctx, input, logger)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||||
|
if err := fetch(ctx, input); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||||
|
}
|
||||||
|
return resolved, reused, err
|
||||||
|
}
|
||||||
|
|
||||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||||
@@ -282,7 +310,7 @@ func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
args := []string{"fetch", "--force", "--prune"}
|
args := []string{"fetch", "--force", "--no-prune"}
|
||||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||||
args = append(args, "--depth", "1", "--no-tags")
|
args = append(args, "--depth", "1", "--no-tags")
|
||||||
}
|
}
|
||||||
@@ -296,9 +324,18 @@ func cachedRef(ref string) string {
|
|||||||
|
|
||||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||||
if !isFullObjectID(ref) {
|
if !isFullObjectID(ref) {
|
||||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
if strings.ContainsAny(ref, "\x00\n") {
|
||||||
return sha, nil
|
for _, revision := range revisions {
|
||||||
|
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||||
|
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||||
|
if isFullObjectID(sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -353,6 +390,21 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
|||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||||
|
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||||
|
var config []string
|
||||||
|
if certFile != "" {
|
||||||
|
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||||
|
}
|
||||||
|
if caFile != "" {
|
||||||
|
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||||
|
}
|
||||||
|
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||||
|
config = append(config, prefix+"sslBackend=openssl")
|
||||||
|
}
|
||||||
|
return config
|
||||||
|
}
|
||||||
|
|
||||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||||
func ConfigEnv(config ...string) map[string]string {
|
func ConfigEnv(config ...string) map[string]string {
|
||||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||||
@@ -364,18 +416,59 @@ func ConfigEnv(config ...string) map[string]string {
|
|||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
||||||
|
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var files, repos []string
|
||||||
|
for entry := range strings.SplitSeq(string(listed), "\x00") {
|
||||||
|
tag, file, ok := strings.Cut(entry, " ")
|
||||||
|
if tag != "?" {
|
||||||
|
var meta string
|
||||||
|
meta, file, ok = strings.Cut(file, "\t")
|
||||||
|
if strings.HasPrefix(meta, "160000 ") {
|
||||||
|
file += "/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
|
||||||
|
repos = append(repos, repo)
|
||||||
|
} else if ok {
|
||||||
|
files = append(files, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(repos)
|
||||||
|
for _, repo := range slices.Compact(repos) {
|
||||||
|
files = append(files, repo)
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
|
||||||
|
continue // git lists an uninitialized submodule as "./" from inside it
|
||||||
|
}
|
||||||
|
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, file := range nested {
|
||||||
|
files = append(files, repo+"/"+file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||||
|
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
if input.ClientCertFile != "" {
|
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if dir != "" {
|
if dir != "" {
|
||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
|
cmd.Stdin = stdin
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||||
name, _, _ := strings.Cut(entry, "=")
|
name, _, _ := strings.Cut(entry, "=")
|
||||||
return slices.Contains(LocalEnvVars, name)
|
return slices.Contains(LocalEnvVars, name)
|
||||||
@@ -384,15 +477,28 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
|||||||
cmd.Env = append(cmd.Env, key+"="+value)
|
cmd.Env = append(cmd.Env, key+"="+value)
|
||||||
}
|
}
|
||||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||||
if runtime.GOOS != "windows" {
|
var treeKill *process.TreeKill
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
|
||||||
|
} else {
|
||||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||||
}
|
}
|
||||||
cmd.WaitDelay = time.Second
|
cmd.WaitDelay = time.Second
|
||||||
|
var stdout bytes.Buffer
|
||||||
var stderr strings.Builder
|
var stderr strings.Builder
|
||||||
|
cmd.Stdout = &stdout
|
||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
output, err := cmd.Output()
|
err := cmd.Start()
|
||||||
if err != nil {
|
if err == nil {
|
||||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
if treeKill != nil {
|
||||||
|
if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
|
||||||
|
defer killer.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = cmd.Wait()
|
||||||
}
|
}
|
||||||
return output, nil
|
if err != nil {
|
||||||
|
return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), nil
|
||||||
}
|
}
|
||||||
+256
-282
File diff suppressed because it is too large.
Load diff
@@ -14,13 +14,8 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
|
|||||||
|
|
||||||
// JobError returns the job error for current context if any
|
// JobError returns the job error for current context if any
|
||||||
func JobError(ctx context.Context) error {
|
func JobError(ctx context.Context) error {
|
||||||
val := ctx.Value(jobErrorContextKeyVal)
|
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
|
||||||
if val != nil {
|
return container["error"]
|
||||||
if container, ok := val.(map[string]error); ok {
|
|
||||||
return container["error"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetJobError(ctx context.Context, err error) {
|
func SetJobError(ctx context.Context, err error) {
|
||||||
@@ -31,6 +26,5 @@ func SetJobError(ctx context.Context, err error) {
|
|||||||
|
|
||||||
// WithJobErrorContainer adds a value to the context as a container for an error
|
// WithJobErrorContainer adds a value to the context as a container for an error
|
||||||
func WithJobErrorContainer(ctx context.Context) context.Context {
|
func WithJobErrorContainer(ctx context.Context) context.Context {
|
||||||
container := map[string]error{}
|
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
|
||||||
return context.WithValue(ctx, jobErrorContextKeyVal, container)
|
|
||||||
}
|
}
|
||||||
@@ -26,9 +26,7 @@ type lineWriter struct {
|
|||||||
|
|
||||||
// NewLineWriter creates a new instance of a line writer
|
// NewLineWriter creates a new instance of a line writer
|
||||||
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
||||||
w := new(lineWriter)
|
return &lineWriter{handlers: handlers}
|
||||||
w.handlers = handlers
|
|
||||||
return w
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
||||||
@@ -39,24 +37,15 @@ func FlushWriter(w io.Writer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lw *lineWriter) Write(p []byte) (n int, err error) {
|
func (lw *lineWriter) Write(p []byte) (int, error) {
|
||||||
pBuf := bytes.NewBuffer(p)
|
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
|
||||||
written := 0
|
_, _ = lw.buffer.Write(line)
|
||||||
for {
|
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||||
line, err := pBuf.ReadString('\n')
|
lw.handleLine(lw.buffer.String())
|
||||||
w, _ := lw.buffer.WriteString(line)
|
lw.buffer.Reset()
|
||||||
written += w
|
|
||||||
if err != nil {
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return written, err
|
|
||||||
}
|
}
|
||||||
lw.handleLine(lw.buffer.String())
|
|
||||||
lw.buffer.Reset()
|
|
||||||
}
|
}
|
||||||
|
return len(p), nil
|
||||||
return written, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
||||||
@@ -72,8 +61,7 @@ func (lw *lineWriter) Flush() {
|
|||||||
|
|
||||||
func (lw *lineWriter) handleLine(line string) {
|
func (lw *lineWriter) handleLine(line string) {
|
||||||
for _, h := range lw.handlers {
|
for _, h := range lw.handlers {
|
||||||
ok := h(line)
|
if !h(line) {
|
||||||
if !ok {
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,24 +9,24 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLineWriter(t *testing.T) {
|
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
|
||||||
lines := make([]string, 0)
|
var lines []string
|
||||||
lineHandler := func(s string) bool {
|
lineWriter := NewLineWriter(func(s string) bool {
|
||||||
lines = append(lines, s)
|
lines = append(lines, s)
|
||||||
return true
|
return true
|
||||||
}
|
})
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
write := func(s string) {
|
write := func(s string) {
|
||||||
n, err := lineWriter.Write([]byte(s))
|
n, err := lineWriter.Write([]byte(s))
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(len(s), n, s)
|
assert.Equal(len(s), n, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
write("")
|
||||||
write("hello")
|
write("hello")
|
||||||
write(" ")
|
write(" ")
|
||||||
write("world!!\nextra")
|
write("world!!\nextra")
|
||||||
@@ -34,39 +34,17 @@ func TestLineWriter(t *testing.T) {
|
|||||||
write(" line\n")
|
write(" line\n")
|
||||||
write("no newline here...")
|
write("no newline here...")
|
||||||
|
|
||||||
assert.Len(lines, 4)
|
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
|
||||||
assert.Equal("hello world!!\n", lines[0])
|
assert.Equal(expected, lines)
|
||||||
assert.Equal("extra line\n", lines[1])
|
|
||||||
assert.Equal(" and another\n", lines[2])
|
|
||||||
assert.Equal("last line\n", lines[3])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLineWriterFlush(t *testing.T) {
|
|
||||||
lines := make([]string, 0)
|
|
||||||
lineHandler := func(s string) bool {
|
|
||||||
lines = append(lines, s)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
|
||||||
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
|
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
|
|
||||||
|
|
||||||
// Only the newline-terminated line is emitted before flushing.
|
|
||||||
assert.Equal([]string{"complete line\n"}, lines)
|
|
||||||
|
|
||||||
// Flushing emits the buffered, not-yet-terminated trailing line.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
|
expected = append(expected, "no newline here...")
|
||||||
|
assert.Equal(expected, lines)
|
||||||
|
|
||||||
// Flushing again is a no-op: nothing is buffered.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Len(lines, 2)
|
assert.Equal(expected, lines)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
||||||
// FlushWriter must be a safe no-op for writers that do not buffer lines.
|
|
||||||
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
||||||
}
|
}
|
||||||
+30
-25
@@ -7,46 +7,51 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
type loggerContextKey string
|
type loggerContextKey string
|
||||||
|
|
||||||
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
|
const loggerContextKeyVal = loggerContextKey("logging.Logger")
|
||||||
|
|
||||||
// Logger returns the appropriate logger for current context
|
// Logger returns the appropriate logger for current context
|
||||||
func Logger(ctx context.Context) logrus.FieldLogger {
|
func Logger(ctx context.Context) *logging.Logger {
|
||||||
val := ctx.Value(loggerContextKeyVal)
|
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
|
||||||
if val != nil {
|
return logger
|
||||||
if logger, ok := val.(logrus.FieldLogger); ok {
|
|
||||||
return logger
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return logrus.StandardLogger()
|
return logging.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLogger adds a value to the context for the logger
|
// WithLogger adds a value to the context for the logger
|
||||||
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
|
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
|
||||||
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
||||||
}
|
}
|
||||||
|
|
||||||
type loggerHookKey string
|
type loggerHandlerKey string
|
||||||
|
|
||||||
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
|
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
|
||||||
|
|
||||||
// LoggerHook returns the appropriate logger hook for current context
|
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
|
||||||
// the hook affects job logger, not global logger
|
// It affects the job logger, not the global logger.
|
||||||
func LoggerHook(ctx context.Context) logrus.Hook {
|
func LoggerHandler(ctx context.Context) logging.Handler {
|
||||||
val := ctx.Value(loggerHookKeyVal)
|
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
|
||||||
if val != nil {
|
return h
|
||||||
if hook, ok := val.(logrus.Hook); ok {
|
|
||||||
return hook
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLoggerHook adds a value to the context for the logger hook
|
// WithLoggerHandler adds the extra handler for the job logger to the context
|
||||||
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
|
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
|
||||||
return context.WithValue(ctx, loggerHookKeyVal, hook)
|
return context.WithValue(ctx, loggerHandlerKeyVal, h)
|
||||||
|
}
|
||||||
|
|
||||||
|
type stepIDsContextKey struct{}
|
||||||
|
|
||||||
|
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
|
||||||
|
func StepIDs(ctx context.Context) []string {
|
||||||
|
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithStepIDs replaces the step IDs in the context
|
||||||
|
func WithStepIDs(ctx context.Context, ids []string) context.Context {
|
||||||
|
return context.WithValue(ctx, stepIDsContextKey{}, ids)
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ansiReset = "\x1b[0m"
|
||||||
|
ansiRed = "\x1b[31m"
|
||||||
|
ansiYellow = "\x1b[33m"
|
||||||
|
ansiCyan = "\x1b[36m"
|
||||||
|
ansiDimCyan = "\x1b[2;36m"
|
||||||
|
ansiDimWhite = "\x1b[2;37m"
|
||||||
|
)
|
||||||
|
|
||||||
|
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
|
||||||
|
var consoleStart = time.Now()
|
||||||
|
|
||||||
|
// console is the state shared by every consoleHandler derived from one another.
|
||||||
|
type console struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
out io.Writer
|
||||||
|
terminal bool
|
||||||
|
opts TextOptions
|
||||||
|
reportCaller bool
|
||||||
|
level slog.LevelVar
|
||||||
|
}
|
||||||
|
|
||||||
|
func newConsole(out io.Writer) *console {
|
||||||
|
c := &console{}
|
||||||
|
c.setOutput(out)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *console) setOutput(w io.Writer) {
|
||||||
|
terminal := checkIfTerminal(w)
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.out, c.terminal = w, terminal
|
||||||
|
}
|
||||||
|
|
||||||
|
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
|
||||||
|
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
|
||||||
|
// and a colored prefix with elapsed or full time on a terminal.
|
||||||
|
type consoleHandler struct {
|
||||||
|
c *console
|
||||||
|
attrs []slog.Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||||
|
return level >= h.c.level.Level()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||||
|
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
|
||||||
|
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
|
||||||
|
|
||||||
|
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||||
|
c := h.c
|
||||||
|
c.mu.Lock()
|
||||||
|
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
|
||||||
|
for _, a := range h.attrs {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
}
|
||||||
|
r.Attrs(func(a slog.Attr) bool {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
|
||||||
|
var caller *runtime.Frame
|
||||||
|
if reportCaller && r.PC != 0 {
|
||||||
|
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
|
||||||
|
caller = &frame
|
||||||
|
}
|
||||||
|
prefixFieldClashes(data, caller != nil)
|
||||||
|
|
||||||
|
keys := slices.Sorted(maps.Keys(data))
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
if terminal && !opts.DisableColors {
|
||||||
|
printColored(&b, opts, r, caller, keys, data)
|
||||||
|
} else {
|
||||||
|
printPlain(&b, opts, r, caller, keys, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
line := b.Bytes()
|
||||||
|
if opts.Filter != nil {
|
||||||
|
line = opts.Filter(line)
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
_, err := c.out.Write(line)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
|
||||||
|
func prefixFieldClashes(data map[string]any, reportCaller bool) {
|
||||||
|
for _, key := range []string{"time", "level", "msg"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
delete(data, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reportCaller {
|
||||||
|
for _, key := range []string{"func", "file"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
if !opts.DisableTimestamp {
|
||||||
|
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
|
||||||
|
if r.Message != "" {
|
||||||
|
appendKeyValue(b, "msg", r.Message)
|
||||||
|
}
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if funcVal != "" {
|
||||||
|
appendKeyValue(b, "func", funcVal)
|
||||||
|
}
|
||||||
|
if fileVal != "" {
|
||||||
|
appendKeyValue(b, "file", fileVal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
appendKeyValue(b, k, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
|
||||||
|
if opts.CallerPrettyfier != nil {
|
||||||
|
return opts.CallerPrettyfier(caller)
|
||||||
|
}
|
||||||
|
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
msg := strings.TrimSuffix(r.Message, "\n")
|
||||||
|
|
||||||
|
var callerStr string
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if opts.CallerPrettyfier == nil && caller.Function != "" {
|
||||||
|
funcVal += "()"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case fileVal == "":
|
||||||
|
callerStr = funcVal
|
||||||
|
case funcVal == "":
|
||||||
|
callerStr = fileVal
|
||||||
|
default:
|
||||||
|
callerStr = fileVal + " " + funcVal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
level := Level(r.Level)
|
||||||
|
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
|
||||||
|
switch {
|
||||||
|
case opts.DisableTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
|
||||||
|
case !opts.FullTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
b.WriteString(colorize(level, k))
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainLevel spells Warn the way logrus did, as "warning".
|
||||||
|
func plainLevel(l Level) string {
|
||||||
|
if s := l.String(); s != "warn" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "warning"
|
||||||
|
}
|
||||||
|
|
||||||
|
func colorize(l Level, s string) string {
|
||||||
|
var color string
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
color = ansiDimWhite
|
||||||
|
case l <= LevelDebug:
|
||||||
|
color = ansiDimCyan
|
||||||
|
case l <= LevelInfo:
|
||||||
|
color = ansiCyan
|
||||||
|
case l <= LevelWarn:
|
||||||
|
color = ansiYellow
|
||||||
|
default:
|
||||||
|
color = ansiRed
|
||||||
|
}
|
||||||
|
return color + s + ansiReset
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendKeyValue(b *bytes.Buffer, key string, value any) {
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
b.WriteString(key)
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendValue(b *bytes.Buffer, value any) {
|
||||||
|
switch v := value.(type) {
|
||||||
|
case string:
|
||||||
|
appendString(b, v)
|
||||||
|
case []byte:
|
||||||
|
appendString(b, string(v))
|
||||||
|
case error:
|
||||||
|
appendFormatted(b, v, v.Error)
|
||||||
|
case fmt.Stringer:
|
||||||
|
appendFormatted(b, v, v.String)
|
||||||
|
default:
|
||||||
|
appendString(b, fmt.Sprint(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
|
||||||
|
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
|
||||||
|
appendString(b, "<nil>")
|
||||||
|
} else {
|
||||||
|
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
appendString(b, get())
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendString(b *bytes.Buffer, s string) {
|
||||||
|
if needsQuoting(s) {
|
||||||
|
b.WriteString(strconv.Quote(s))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.WriteString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsQuoting(s string) bool {
|
||||||
|
for i := range len(s) {
|
||||||
|
c := s[i]
|
||||||
|
switch {
|
||||||
|
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
|
||||||
|
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
|
||||||
|
t.Helper()
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
|
||||||
|
c.level.Set(slog.LevelDebug)
|
||||||
|
build(New(NewSlogHandler(&consoleHandler{c: c})))
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlain(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
|
||||||
|
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
|
||||||
|
l.With("msg", "clash", "empty", "").Info("")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
|
||||||
|
"level=info empty= fields.msg=clash\n", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
|
||||||
|
opts := TextOptions{
|
||||||
|
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
|
||||||
|
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
|
||||||
|
},
|
||||||
|
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
|
||||||
|
}
|
||||||
|
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
|
||||||
|
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleColored(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
|
||||||
|
l.With("job", "j").Warn("careful")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
|
||||||
|
|
||||||
|
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
|
||||||
|
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
|
||||||
|
|
||||||
|
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
|
||||||
|
|
||||||
|
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Equal(t, "level=info msg=x\n", plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleLevelAndOutput(t *testing.T) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
h := &consoleHandler{c: c}
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
|
||||||
|
c.level.Set(slog.LevelError)
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
|
||||||
|
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
|
||||||
|
|
||||||
|
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
|
||||||
|
rec.AddAttrs(slog.String("c", "3"))
|
||||||
|
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
|
||||||
|
assert.Contains(t, out.String(), "a=1 c=3")
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TextOptions describes how the process-wide Logger writes a record as text.
|
||||||
|
type TextOptions struct {
|
||||||
|
DisableColors bool
|
||||||
|
DisableTimestamp bool
|
||||||
|
FullTimestamp bool
|
||||||
|
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
|
||||||
|
CallerPrettyfier func(*runtime.Frame) (function, file string)
|
||||||
|
// Filter rewrites each formatted line, for example to mask secrets.
|
||||||
|
Filter func(line []byte) []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
std atomic.Pointer[Logger]
|
||||||
|
stdState = newConsole(os.Stderr)
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default returns the process-wide Logger.
|
||||||
|
func Default() *Logger { return std.Load() }
|
||||||
|
|
||||||
|
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
|
||||||
|
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
|
||||||
|
// initial console Logger.
|
||||||
|
func SetDefault(l *Logger) { std.Store(l) }
|
||||||
|
|
||||||
|
// SetLevel sets the minimum level of the initial process-wide Logger.
|
||||||
|
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
|
||||||
|
|
||||||
|
// GetLevel returns the minimum level of the initial process-wide Logger.
|
||||||
|
func GetLevel() Level { return Level(stdState.level.Level()) }
|
||||||
|
|
||||||
|
// SetOutput sets where the initial process-wide Logger writes.
|
||||||
|
func SetOutput(w io.Writer) { stdState.setOutput(w) }
|
||||||
|
|
||||||
|
// Output returns where the initial process-wide Logger writes.
|
||||||
|
func Output() io.Writer {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
return stdState.out
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetTextFormat sets the text format of the initial process-wide Logger.
|
||||||
|
func SetTextFormat(o TextOptions) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.opts = o
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
|
||||||
|
func SetReportCaller(on bool) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.reportCaller = on
|
||||||
|
}
|
||||||
|
|
||||||
|
func Enabled(level Level) bool { return Default().Enabled(level) }
|
||||||
|
|
||||||
|
func With(args ...any) *Logger { return Default().With(args...) }
|
||||||
|
func WithError(err error) *Logger { return Default().WithError(err) }
|
||||||
|
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
|
||||||
|
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
|
||||||
|
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
|
||||||
|
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
|
||||||
|
func Debug(args ...any) { Default().log(LevelDebug, args) }
|
||||||
|
func Info(args ...any) { Default().log(LevelInfo, args) }
|
||||||
|
func Warn(args ...any) { Default().log(LevelWarn, args) }
|
||||||
|
func Error(args ...any) { Default().log(LevelError, args) }
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Attr is a structured key/value pair attached to a log record.
|
||||||
|
type Attr struct {
|
||||||
|
Key string
|
||||||
|
Value any
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
|
||||||
|
// Logger.With are accumulated by the Handler.
|
||||||
|
type Record struct {
|
||||||
|
Time time.Time
|
||||||
|
Level Level
|
||||||
|
Message string
|
||||||
|
PC uintptr // program counter of the logging call, 0 if unknown
|
||||||
|
Attrs []Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attr returns the value of the last attribute named key.
|
||||||
|
func (r Record) Attr(key string) (any, bool) {
|
||||||
|
for _, a := range slices.Backward(r.Attrs) {
|
||||||
|
if a.Key == key {
|
||||||
|
return a.Value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
|
||||||
|
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
|
||||||
|
type Handler interface {
|
||||||
|
Enabled(ctx context.Context, level Level) bool
|
||||||
|
Handle(ctx context.Context, r Record) error
|
||||||
|
// WithAttrs returns a Handler that adds attrs to every record it handles.
|
||||||
|
WithAttrs(attrs []Attr) Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
type multiHandler []Handler
|
||||||
|
|
||||||
|
// NewMultiHandler fans every record out to all handlers that have its level enabled.
|
||||||
|
func NewMultiHandler(handlers ...Handler) Handler {
|
||||||
|
return multiHandler(handlers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, level) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
var errs []error
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, r.Level) {
|
||||||
|
errs = append(errs, h.Handle(ctx, r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
out := make(multiHandler, len(m))
|
||||||
|
for i, h := range m {
|
||||||
|
out[i] = h.WithAttrs(attrs)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
type discardHandler struct{}
|
||||||
|
|
||||||
|
// DiscardHandler drops every record.
|
||||||
|
var DiscardHandler Handler = discardHandler{}
|
||||||
|
|
||||||
|
func (discardHandler) Enabled(context.Context, Level) bool { return false }
|
||||||
|
func (discardHandler) Handle(context.Context, Record) error { return nil }
|
||||||
|
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
|
||||||
|
|
||||||
|
type levelHandler struct {
|
||||||
|
h Handler
|
||||||
|
minLevel Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLevelHandler passes only records at or above minLevel on to h.
|
||||||
|
func NewLevelHandler(h Handler, minLevel Level) Handler {
|
||||||
|
return levelHandler{h: h, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
return level >= l.minLevel && l.h.Enabled(ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
if r.Level < l.minLevel {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return l.h.Handle(ctx, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Level is a log severity. The values match log/slog, with Trace added below Debug.
|
||||||
|
type Level int
|
||||||
|
|
||||||
|
const (
|
||||||
|
LevelTrace Level = -8
|
||||||
|
LevelDebug Level = -4
|
||||||
|
LevelInfo Level = 0
|
||||||
|
LevelWarn Level = 4
|
||||||
|
LevelError Level = 8
|
||||||
|
)
|
||||||
|
|
||||||
|
func (l Level) String() string {
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
return "trace"
|
||||||
|
case l <= LevelDebug:
|
||||||
|
return "debug"
|
||||||
|
case l <= LevelInfo:
|
||||||
|
return "info"
|
||||||
|
case l <= LevelWarn:
|
||||||
|
return "warn"
|
||||||
|
default:
|
||||||
|
return "error"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
|
||||||
|
// the Logger has no level above it.
|
||||||
|
func ParseLevel(s string) (Level, error) {
|
||||||
|
switch strings.ToLower(s) {
|
||||||
|
case "trace":
|
||||||
|
return LevelTrace, nil
|
||||||
|
case "debug":
|
||||||
|
return LevelDebug, nil
|
||||||
|
case "info":
|
||||||
|
return LevelInfo, nil
|
||||||
|
case "warn", "warning":
|
||||||
|
return LevelWarn, nil
|
||||||
|
case "error", "fatal", "panic":
|
||||||
|
return LevelError, nil
|
||||||
|
}
|
||||||
|
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const badKey = "!BADKEY"
|
||||||
|
|
||||||
|
// Logger writes printf-style messages with structured attributes to a Handler.
|
||||||
|
type Logger struct {
|
||||||
|
h Handler
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(h Handler) *Logger {
|
||||||
|
return &Logger{h: h, ctx: context.Background()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
|
||||||
|
func (l *Logger) With(args ...any) *Logger {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) WithError(err error) *Logger {
|
||||||
|
return l.With("error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithContext returns a Logger that passes ctx to its Handler.
|
||||||
|
func (l *Logger) WithContext(ctx context.Context) *Logger {
|
||||||
|
return &Logger{h: l.h, ctx: ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Enabled(level Level) bool {
|
||||||
|
return l.h.Enabled(l.ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
|
||||||
|
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
|
||||||
|
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
|
||||||
|
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
|
||||||
|
|
||||||
|
// Trace and the methods below format their arguments like fmt.Sprint.
|
||||||
|
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
|
||||||
|
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
|
||||||
|
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
|
||||||
|
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
|
||||||
|
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
|
||||||
|
|
||||||
|
func (l *Logger) logf(level Level, format string, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) log(level Level, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprint(args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// emit drops the Handler error, as a logger has nowhere to report it.
|
||||||
|
func (l *Logger) emit(pc uintptr, level Level, msg string) {
|
||||||
|
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
|
||||||
|
func callerPC(skip int) uintptr {
|
||||||
|
var pcs [1]uintptr
|
||||||
|
runtime.Callers(skip, pcs[:])
|
||||||
|
return pcs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func argsToAttrs(args []any) []Attr {
|
||||||
|
attrs := make([]Attr, 0, len(args)/2+1)
|
||||||
|
for len(args) > 0 {
|
||||||
|
switch k := args[0].(type) {
|
||||||
|
case Attr:
|
||||||
|
attrs = append(attrs, k)
|
||||||
|
args = args[1:]
|
||||||
|
case string:
|
||||||
|
if len(args) == 1 {
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = nil
|
||||||
|
} else {
|
||||||
|
attrs = append(attrs, Attr{Key: k, Value: args[1]})
|
||||||
|
args = args[2:]
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return attrs
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseLevel(t *testing.T) {
|
||||||
|
for in, want := range map[string]logging.Level{
|
||||||
|
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
|
||||||
|
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
|
||||||
|
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
|
||||||
|
} {
|
||||||
|
got, err := logging.ParseLevel(in)
|
||||||
|
require.NoError(t, err, in)
|
||||||
|
assert.Equal(t, want, got, in)
|
||||||
|
}
|
||||||
|
_, err := logging.ParseLevel("loud")
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, "warn", logging.LevelWarn.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingStringer struct{ n *int }
|
||||||
|
|
||||||
|
func (c countingStringer) String() string { *c.n++; return "x" }
|
||||||
|
|
||||||
|
func TestLogger(t *testing.T) {
|
||||||
|
h := logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
assert.Nil(t, h.LastRecord())
|
||||||
|
l := logging.New(h)
|
||||||
|
|
||||||
|
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
|
||||||
|
l.Info("a", 1, 2, "b")
|
||||||
|
l.With("k").With(7).Debug("x")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 3)
|
||||||
|
assert.Equal(t, "step 2", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
assert.False(t, recs[0].Time.IsZero())
|
||||||
|
job, _ := recs[0].Attr("job")
|
||||||
|
assert.Equal(t, "j1", job)
|
||||||
|
errv, _ := recs[0].Attr("error")
|
||||||
|
require.EqualError(t, errv.(error), "boom")
|
||||||
|
assert.Equal(t, "a1 2b", recs[1].Message)
|
||||||
|
bad, _ := recs[2].Attr("!BADKEY")
|
||||||
|
assert.Equal(t, 7, bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
|
||||||
|
n := 0
|
||||||
|
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
|
||||||
|
l.Debugf("%v", countingStringer{&n})
|
||||||
|
assert.Zero(t, n)
|
||||||
|
assert.False(t, l.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiHandler(t *testing.T) {
|
||||||
|
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
|
||||||
|
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
|
||||||
|
l.Info("i")
|
||||||
|
l.Error("e")
|
||||||
|
assert.Len(t, all.AllRecords(), 2)
|
||||||
|
require.Len(t, warnOnly.AllRecords(), 1)
|
||||||
|
v, _ := warnOnly.LastRecord().Attr("a")
|
||||||
|
assert.Equal(t, 1, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
|
||||||
|
t.Helper()
|
||||||
|
require.NotZero(t, pc)
|
||||||
|
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCallerPC(t *testing.T) {
|
||||||
|
l, h := logtest.NewLogger()
|
||||||
|
l.Infof("method")
|
||||||
|
l.Info("method")
|
||||||
|
|
||||||
|
prev := logging.Default()
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
logging.SetDefault(l)
|
||||||
|
logging.Infof("global")
|
||||||
|
logging.Error("global")
|
||||||
|
|
||||||
|
require.Len(t, h.AllRecords(), 4)
|
||||||
|
for _, r := range h.AllRecords() {
|
||||||
|
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultLevel(t *testing.T) {
|
||||||
|
prev := logging.GetLevel()
|
||||||
|
t.Cleanup(func() { logging.SetLevel(prev) })
|
||||||
|
|
||||||
|
logging.SetLevel(logging.LevelWarn)
|
||||||
|
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
|
||||||
|
assert.True(t, logging.Enabled(logging.LevelError))
|
||||||
|
assert.False(t, logging.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogHandler(t *testing.T) {
|
||||||
|
var buf strings.Builder
|
||||||
|
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
|
||||||
|
|
||||||
|
assert.False(t, l.Enabled(logging.LevelTrace))
|
||||||
|
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
|
||||||
|
|
||||||
|
out := buf.String()
|
||||||
|
assert.Contains(t, out, "level=ERROR")
|
||||||
|
assert.Contains(t, out, `msg="fail x"`)
|
||||||
|
assert.Contains(t, out, "job=j1")
|
||||||
|
assert.Contains(t, out, "error=boom")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogBridge(t *testing.T) {
|
||||||
|
h := logtest.SetDefault(t)
|
||||||
|
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
|
||||||
|
_, _, line, _ := runtime.Caller(0)
|
||||||
|
slog.Debug("dropped")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 1)
|
||||||
|
assert.Equal(t, "from a library", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
lib, _ := recs[0].Attr("lib")
|
||||||
|
assert.Equal(t, "actionslib", lib)
|
||||||
|
n, _ := recs[0].Attr("n")
|
||||||
|
assert.EqualValues(t, 1, n)
|
||||||
|
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
// Package logtest provides a Handler that keeps records in memory for assertions.
|
||||||
|
package logtest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
)
|
||||||
|
|
||||||
|
type store struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
records []logging.Record
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
|
||||||
|
type Handler struct {
|
||||||
|
store *store
|
||||||
|
attrs []logging.Attr
|
||||||
|
minLevel logging.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New records from LevelInfo up, like the logger of logrus's hooks/test.
|
||||||
|
func New() *Handler {
|
||||||
|
return NewWithLevel(logging.LevelInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
|
||||||
|
func NewLogger() (*logging.Logger, *Handler) {
|
||||||
|
h := New()
|
||||||
|
return logging.New(h), h
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWithLevel(minLevel logging.Level) *Handler {
|
||||||
|
return &Handler{store: &store{}, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
|
||||||
|
|
||||||
|
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
|
||||||
|
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
|
||||||
|
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
h.store.records = append(h.store.records, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
|
||||||
|
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) AllRecords() []logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
return slices.Clone(h.store.records)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LastRecord returns nil when nothing was logged.
|
||||||
|
func (h *Handler) LastRecord() *logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
if len(h.store.records) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r := h.store.records[len(h.store.records)-1]
|
||||||
|
return &r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
|
||||||
|
func SetDefault(t testing.TB) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
prev := logging.Default()
|
||||||
|
logger, h := NewLogger()
|
||||||
|
logging.SetDefault(logger)
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
return h
|
||||||
|
}
|
||||||
Loaded 50 of 315 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user