Compare commits

..
Author SHA1 Message Date
Lunny Xiao de9db9c0fa fix: suppress tar leading slash warnings 2026-09-06 20:17:55 -07:00
Lunny Xiao d1da6ffa52 fix: create guest workspace in real macOS VM integration test 2026-09-06 18:37:15 -07:00
Lunny Xiao 880e0a148a fix: prepare macOS VM workspace and archive without AppleDouble 2026-09-05 21:56:15 -07:00
Lunny Xiao 2db24c7401 enhance: add macOS VM executor backend
Replace the Tart-based prototype with a runner-oriented macOS VM
executor backed by runner-executor-macos and Apple Virtualization.

Assisted-by: Codet:MODEL_VERSION
2026-09-05 13:20:03 -07:00
336 changed files with 14726 additions and 28057 deletions

No files matched your search

-5
View File
@@ -45,13 +45,8 @@ cpu.out
/bin
/dist
/.tmp
/.git/hooks
/.git/logs
/.git/rr-cache
/.env
/.runner
/.runner.lock
/config.yaml
/Dockerfile
.DS_Store
-29
View File
@@ -1,29 +0,0 @@
name: go-setup
description: Set up go and restore its caches, which only the cache-seeder workflow saves
runs:
using: composite
steps:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
id: setup-go
with:
go-version-file: go.mod
check-latest: true
cache: false
- if: ${{ github.workflow == 'cache-seeder' }}
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
- if: ${{ github.workflow != 'cache-seeder' }}
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
-26
View File
@@ -1,26 +0,0 @@
name: cache-seeder
on:
workflow_dispatch:
push:
branches:
- main
paths:
- go.mod
- go.sum
- .golangci.yml
- Makefile
- .gitea/actions/go-setup/action.yml
- .gitea/workflows/cache-seeder.yml
jobs:
seed:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- run: make deps-tools lint build
- name: compile the test binaries without running them
run: |
make test GOTEST_FLAGS='-race -exec true'
go test -tags e2e -exec true ./e2e/...
+2 -2
View File
@@ -19,9 +19,9 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 26
node-version: 24
- run: make lint-pr-title
env:
PR_TITLE: ${{ github.event.pull_request.title }}
+6 -13
View File
@@ -11,6 +11,7 @@ on:
env:
DOCKER_ORG: gitea
DOCKER_LATEST: nightly
jobs:
goreleaser:
@@ -34,15 +35,8 @@ jobs:
with:
go-version-file: "go.mod"
check-latest: true
- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
passphrase: ${{ secrets.PASSPHRASE }}
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
- name: goreleaser
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
with:
distribution: goreleaser-pro
args: release --nightly
@@ -54,7 +48,6 @@ jobs:
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
GORELEASER_FORCE_TOKEN: "gitea"
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
release-image:
runs-on: ubuntu-latest
@@ -81,13 +74,13 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -101,7 +94,7 @@ jobs:
echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile
+6 -5
View File
@@ -35,7 +35,7 @@ jobs:
passphrase: ${{ secrets.PASSPHRASE }}
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
- name: goreleaser
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
with:
distribution: goreleaser-pro
args: release
@@ -69,6 +69,7 @@ jobs:
image: catthehacker/ubuntu:act-latest
env:
DOCKER_ORG: gitea
DOCKER_LATEST: latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -76,13 +77,13 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -102,7 +103,7 @@ jobs:
suffix=${{ matrix.variant.tag_suffix }},onlatest=true
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile
+14 -36
View File
@@ -1,7 +1,9 @@
name: checks
on:
push:
branches:
- main
pull_request:
workflow_dispatch:
env:
# The runner image ships a stale docker.io login; point docker at an empty config so
@@ -17,24 +19,25 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
# daemon retains them between runs, and their digest pins skip the registry when it has them.
# daemon retains them between runs, so this is usually a fast manifest re-check.
- name: pre-pull test images
env:
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:ba849c60be29959425b8734d57b8b4b7d56f98edd9504c9af091d5281095a71e # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
run: |
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
for attempt in 1 2 3; do
docker pull "$image" && break
[ "$attempt" = 3 ] || sleep 5
done
fi
docker tag "$image" "${image%@*}"
done
- name: lint
@@ -50,39 +53,11 @@ jobs:
# after `make test` so the images it needs are already present on the host daemon.
- name: test against dind image
run: make test-dind
- name: test against podman
run: make test-dind TARGET=podman
- name: coverage report
run: |
make coverage-report
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
windows:
name: test on windows
runs-on: ubuntu-latest-kvm
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
- name: windows vm cache key
id: windows-vm
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/windows-vm
key: ${{ steps.windows-vm.outputs.key }}
- name: test on windows
id: test-windows
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
env:
WINDOWS_TEST_TIMEOUT: 45
- if: always() && steps.test-windows.outputs.save-cache == 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/windows-vm
key: ${{ steps.windows-vm.outputs.key }}
e2e:
name: gitea compatibility (${{ matrix.gitea_image }})
runs-on: ubuntu-latest
@@ -93,7 +68,10 @@ jobs:
- gitea/gitea:main-nightly
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
- name: e2e compatibility
-1
View File
@@ -1,5 +1,4 @@
/gitea-runner
/gitea-runner.exe
.env
!/act/runner/testdata/secrets/.env
.runner
-9
View File
@@ -39,15 +39,6 @@ linters:
deny:
- pkg: golang.org/x/exp
desc: it's experimental and unreliable
- pkg: unsafe
desc: use typed APIs instead
logging:
files:
- $all
- "!**/act/common/logging/**"
deny:
- pkg: log/slog
desc: log through gitea.com/gitea/runner/act/common/logging
nolintlint:
allow-unused: false
require-explanation: true
+22 -5
View File
@@ -28,16 +28,38 @@ builds:
ignore:
- goos: darwin
goarch: arm
- goos: darwin
goarch: ppc64le
- goos: darwin
goarch: s390x
- goos: windows
goarch: ppc64le
- goos: windows
goarch: s390x
- goos: windows
goarch: arm
goarm: "5"
- goos: windows
goarch: arm
goarm: "6"
- goos: windows
goarch: arm
goarm: "7"
- goos: windows
goarch: arm64
- goos: freebsd
goarch: ppc64le
- goos: freebsd
goarch: s390x
- goos: freebsd
goarch: arm
goarm: "5"
- goos: freebsd
goarch: arm
goarm: "6"
- goos: freebsd
goarch: arm
goarm: "7"
- goos: freebsd
goarch: arm64
flags:
@@ -80,7 +102,6 @@ builds:
publishers:
- name: cloudflare-r2
checksum: true
signature: true
extra_files:
- glob: ./**.xz
- glob: ./**.sha256
@@ -101,10 +122,6 @@ checksum:
extra_files:
- glob: ./**.xz
signs:
- artifacts: checksum
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
snapshot:
version_template: "{{ .Branch }}-devel"
+1
View File
@@ -0,0 +1 @@
@AGENTS.md
+8 -46
View File
@@ -23,57 +23,19 @@ depending on the prefix:
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
- **Forwarding a command from step output?** Preserve its escaping unless masking changes it.
Decoding for display happens in the UI.
- **Forwarding a command from step output?** Leave the payload alone, it arrived escaped and is
decoded once. Decoding here double-decodes and destroys multi-line.
- **No prefix?** Do not escape, and split multi-line values into one row each.
- **Interpolating a secret?** Masking runs after escaping. Recognised command payloads are
normalised before masking to cover partially escaped values. Their original escaping is
preserved when no replacement occurs. `AppendSecretMasker` also registers encoded forms for
other output. Complete nonblank one-character values are masked, while one-byte fragments
of multiline values are skipped.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
handler folds an annotation's location into its decoded message and escapes the result again.
## Logging
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
fields.
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
Going around either leaks them.
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
job logger.
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
package functions.
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
encoded forms too.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
decodes exactly those two when folding a location into an annotation.
## End-to-end compatibility tests
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
parallel.
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
Docker and Kubernetes.
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
## Windows tests
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
installed Windows in, so later runs skip the install. CI runs this on every pull request.
+7 -14
View File
@@ -1,7 +1,7 @@
### BUILDER STAGE
#
#
FROM golang:1.27-alpine AS builder
FROM golang:1.27-alpine3.23 AS builder
# Do not remove `git` here, it is required for getting runner version when executing `make build`
RUN apk add --no-cache make git
@@ -12,22 +12,19 @@ ENV GOPROXY=${GOPROXY:-}
COPY . /opt/src/runner
WORKDIR /opt/src/runner
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
make clean && make build
RUN make clean && make build
### DIND VARIANT
#
#
FROM docker:29.9.3-dind AS dind
FROM docker:29.7.2-dind AS dind
ARG VERSION=dev
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
LABEL org.opencontainers.image.version="${VERSION}"
# upgrade first: the docker base image can lag behind Alpine security fixes
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
RUN apk add --no-cache s6 bash git tzdata nftables
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
COPY scripts/run.sh /usr/local/bin/run.sh
@@ -35,14 +32,12 @@ COPY scripts/s6 /etc/s6
VOLUME /data
ENV TINI_SUBREAPER=1
ENTRYPOINT ["s6-svscan","/etc/s6"]
### DIND-ROOTLESS VARIANT
#
#
FROM docker:29.9.3-dind-rootless AS dind-rootless
FROM docker:29.7.2-dind-rootless AS dind-rootless
ARG VERSION=dev
@@ -50,8 +45,7 @@ LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
LABEL org.opencontainers.image.version="${VERSION}"
USER root
# upgrade first: the docker base image can lag behind Alpine security fixes
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
RUN apk add --no-cache s6 bash git tzdata nftables
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
COPY scripts/run.sh /usr/local/bin/run.sh
@@ -62,7 +56,6 @@ VOLUME /data
RUN mkdir -p /data && chown -R rootless:rootless /etc/s6 /data
ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
ENV TINI_SUBREAPER=1
USER rootless
ENTRYPOINT ["s6-svscan","/etc/s6"]
@@ -77,7 +70,7 @@ ARG VERSION=dev
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
LABEL org.opencontainers.image.version="${VERSION}"
RUN apk upgrade --no-cache && apk add --no-cache tini bash git tzdata
RUN apk add --no-cache tini bash git tzdata
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
COPY scripts/run.sh /usr/local/bin/run.sh
+79 -21
View File
@@ -1,23 +1,25 @@
DIST := dist
EXECUTABLE := gitea-runner
DIST_DIRS := $(DIST)/binaries $(DIST)/release
GO ?= go
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
SHASUM ?= shasum -a 256
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
XGO_PACKAGE ?= src.techknowlogick.com/[email protected] # renovate: datasource=go
XGO_VERSION := go-1.27.x
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:b96ba465021b1a4c6a1c9e79220c2b9bfad0f29b09dbb528bbfb35411443a3a4 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.2-k3s1@sha256:21ea6e0fac4e3930c2ce8aa40ddd41a8b18cb3e9a0222c030dc005e7e88c4332 # renovate: datasource=docker
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
LINUX_ARCHS ?= linux/amd64,linux/arm64
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
WINDOWS_ARCHS ?= windows/amd64
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
DOCKER_IMAGE ?= gitea/runner
DOCKER_TAG ?= nightly
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]3.1 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.7.0 # renovate: datasource=go
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
@@ -30,16 +32,22 @@ endif
ifeq ($(HAS_GO), GO)
GOPATH ?= $(shell $(GO) env GOPATH)
export PATH := $(GOPATH)/bin:$(PATH)
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
endif
ifeq ($(OS), Windows_NT)
GOFLAGS := -v -buildmode=exe
EXECUTABLE ?= $(EXECUTABLE).exe
GO_ENV_WINDOWS := set GOOS=windows&&
else ifeq ($(OS), Windows)
GOFLAGS := -v -buildmode=exe
EXECUTABLE ?= $(EXECUTABLE).exe
GO_ENV_WINDOWS := set GOOS=windows&&
else
GOFLAGS := -v
EXECUTABLE ?= $(EXECUTABLE)
GO_ENV_WINDOWS := GOOS=windows
endif
@@ -88,12 +96,19 @@ go-check:
fi
.PHONY: fmt-check
fmt-check: ## check the Go code formatting
$(GO) run $(GOLANGCI_LINT_PACKAGE) fmt --diff-colored
fmt-check: fmt
@diff=$$(git diff --color=always -- '*.go'); \
if [ -n "$$diff" ]; then \
echo "Please run 'make fmt' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: deps-tools
deps-tools: ## install tool dependencies
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
$(GO) install $(GXZ_PACKAGE) & \
$(GO) install $(XGO_PACKAGE) & \
$(GO) install $(GOVULNCHECK_PACKAGE) & \
wait
@@ -134,8 +149,13 @@ tidy: ## run go mod tidy
fi
.PHONY: tidy-check
tidy-check: ## check that go.mod and go.sum are tidy
$(GO) mod tidy -diff
tidy-check: tidy
@diff=$$(git diff --color=always -- go.mod go.sum); \
if [ -n "$$diff" ]; then \
echo "Please run 'make tidy' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: test
test: ## test everything (integration tests self-skip without docker/network)
@@ -148,18 +168,17 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
@echo "Wrote .tmp/coverage.md"
.PHONY: test-dind
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
.PHONY: test-windows
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless)
@./scripts/test-dind.sh $(TARGET)
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_CONCURRENCY ?= 8
.PHONY: test-e2e
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
.PHONY: install
install: $(GOFILES) ## install the runner binary via `go install`
@@ -175,6 +194,45 @@ $(EXECUTABLE): $(GOFILES)
deps-backend: ## install backend dependencies
$(GO) mod download
.PHONY: release
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
$(DIST_DIRS):
mkdir -p $(DIST_DIRS)
.PHONY: release-windows
release-windows: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-linux
release-linux: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-darwin
release-darwin: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-copy
release-copy: | $(DIST_DIRS)
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
.PHONY: release-check
release-check: | $(DIST_DIRS)
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
.PHONY: release-compress
release-compress: | $(DIST_DIRS)
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
.PHONY: docker
docker: ## build the docker image
if ! docker buildx version >/dev/null 2>&1; then \
+252 -180
View File
File diff suppressed because it is too large. Load diff
+2
View File
@@ -6,5 +6,7 @@
//
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
//
// TODO: Authorization
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
package artifactcache
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+25 -63
View File
@@ -15,6 +15,8 @@ import (
"strconv"
"strings"
"time"
"github.com/julienschmidt/httprouter"
)
// The cache service v2 API. A client on this version talks twirp to
@@ -37,26 +39,24 @@ const (
blobUploadURLTTL = time.Hour
// twirpInternal is the only error code that is not the client's fault.
twirpInternal = "internal"
twirpUnavailable = "unavailable"
twirpUnauthenticated = "unauthenticated"
twirpDenied = "permission_denied"
)
func (h *Handler) registerV2Routes(router *http.ServeMux) {
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.cacheModeAuth(true, h.v2CreateCacheEntry))
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.cacheModeAuth(true, h.v2FinalizeCacheEntryUpload))
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.cacheModeAuth(false, h.v2GetCacheEntryDownloadURL))
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
func (h *Handler) registerV2Routes(router *httprouter.Router) {
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
}
// An entry that already exists is reported as not ok, which is how the client learns to skip
// the upload.
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2CreateRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
if req.Key == "" || req.Version == "" {
@@ -74,7 +74,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
// An exact (key, version) match means the entry is already cached; the client then skips
// the upload. A prefix match must not count here, or a shorter key would be reported as
// existing and silently never saved.
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
} else if existing != nil {
@@ -83,37 +83,16 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
return
}
// A second live reservation is finalized by whichever job calls last, against the other's upload.
owner := hashedToken(bearerToken(r))
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
if pending.Owner != owner {
h.twirpNotOK(w, r)
return
}
h.touch(db, pending) // still uploading, so it must not go stale under the sweep
h.responseJSON(w, r, http.StatusOK, map[string]any{ // this job retrying its own reservation
"ok": true,
"signed_upload_url": h.signedURL(cred, blobPath, blobUploadPurpose, pending.ID, time.Now().Add(blobUploadURLTTL)),
})
return
}
now := time.Now()
now := time.Now().Unix()
cache := &Cache{
Repo: cred.Repo,
Ref: cred.Ref,
Key: req.Key,
Version: req.Version,
Owner: owner,
Size: -1, // the size is only known at finalize time
CreatedAt: now.Unix(),
CreatedAtNanos: now.Nanosecond(),
UsedAt: now.Unix(),
CreatedAt: now,
UsedAt: now,
}
if err := h.insertCache(db, cache); err != nil {
if err := insertCache(db, cache); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
}
@@ -124,11 +103,11 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
})
}
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -139,14 +118,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
}
defer db.Close()
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
cache = nil // not the reservation this job made, so not this job's to commit
}
if err == nil && cache == nil {
// A retry whose first response was lost finds the entry already committed.
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
}
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
if err != nil {
h.twirpError(w, r, twirpInternal, err)
return
@@ -155,16 +127,14 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
h.twirpNotOK(w, r)
return
}
if !cache.Complete {
db.Close() // commitCache needs the store closed
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r)
return
}
}
h.responseJSON(w, r, http.StatusOK, map[string]any{
"ok": true,
@@ -173,11 +143,11 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
})
}
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2DownloadRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -188,7 +158,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
}
defer db.Close()
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
if err != nil {
h.twirpError(w, r, twirpInternal, err)
return
@@ -208,8 +178,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
// cache is a single PUT, a large one is staged as blocks that a final block list puts
// in order.
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
if err != nil {
h.responseJSON(w, r, http.StatusBadRequest, err)
return
@@ -219,7 +189,6 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
h.responseJSON(w, r, http.StatusBadRequest, err)
return
}
h.makeRoom(r.ContentLength)
query := r.URL.Query()
switch strings.ToLower(query.Get("comp")) {
@@ -261,15 +230,8 @@ func (h *Handler) twirpNotOK(w http.ResponseWriter, r *http.Request) {
func (h *Handler) twirpError(w http.ResponseWriter, r *http.Request, code string, err error) {
h.logger.Debugf("%s %s: %v", r.Method, r.URL.Path, err)
status := http.StatusBadRequest
switch code {
case twirpInternal:
if code == twirpInternal {
status = http.StatusInternalServerError
case twirpUnavailable:
status = http.StatusServiceUnavailable
case twirpUnauthenticated:
status = http.StatusUnauthorized
case twirpDenied:
status = http.StatusForbidden
}
h.responseJSON(w, r, status, map[string]any{"code": code, "msg": err.Error()})
}
+25 -153
View File
@@ -10,16 +10,16 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// v2Call posts a twirp request to the cache service and returns the decoded response.
// Field names are the proto ones, which is what the toolkit's client sends.
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
t.Helper()
@@ -36,6 +36,7 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
return got
}
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
func putBlob(t *testing.T, url string, content []byte) int {
t.Helper()
@@ -45,6 +46,8 @@ func putBlob(t *testing.T, url string, content []byte) int {
require.NoError(t, err)
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated {
// The Azure SDK client dereferences this header without checking, so a blob upload that
// omits it panics the caller rather than failing it.
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
}
return resp.StatusCode
@@ -63,6 +66,8 @@ func getURL(t *testing.T, url string) []byte {
return body
}
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
// with the upload URL it used.
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
t.Helper()
@@ -78,22 +83,24 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
}), uploadURL
}
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
// or to replace a finalized entry.
func TestCacheServiceV2RoundTrip(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
content := []byte("the cached archive")
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
require.Equal(t, true, finalized["ok"])
assert.NotEmpty(t, finalized["entry_id"])
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
// and it is an upload URL only: nothing reads a blob back through it.
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
resp, err := http.Get(uploadURL)
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
@@ -106,8 +113,9 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
assert.Equal(t, content, getURL(t, downloadURL))
}
// A large archive is staged as blocks and only put in order by the final block list, so
// blocks that arrive out of order must still be assembled the way the client asked.
func TestCacheServiceV2BlockUpload(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
@@ -121,6 +129,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
blocks[blockID] = []byte(part)
order = append(order, blockID)
}
// Upload in an order that is not the block list order.
for _, blockID := range []string{order[2], order[0], order[1]} {
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
}
@@ -132,17 +141,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
}
list.WriteString(`</BlockList>`)
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()))
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()),
"a client that lost the first answer retries the list it already sent")
var reordered bytes.Buffer
reordered.WriteString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
for _, blockID := range []string{order[1], order[0], order[2]} {
fmt.Fprintf(&reordered, "<Latest>%s</Latest>", blockID)
}
reordered.WriteString(`</BlockList>`)
assert.Equal(t, http.StatusInternalServerError, putBlob(t, uploadURL+"&comp=blocklist", reordered.Bytes()),
"the blocks are already assembled, so a different order would silently disagree with them")
finalized := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "blocks", "version": "v1", "size_bytes": len("hello world!"),
@@ -155,7 +153,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
}
func TestCacheServiceV2Lookups(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
require.Equal(t, true, saved["ok"])
@@ -165,6 +162,8 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// The toolkit serialises with the proto field names; the camelCase spellings of the same
// proto JSON mapping are accepted alongside them.
for _, field := range []string{"restore_keys", "restoreKeys"} {
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
@@ -180,6 +179,9 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, again["ok"])
})
// A key that is only a prefix of an existing one is a different entry, so the
// reservation check must be exact and not a restore-key prefix match, or the shorter
// key would be reported as existing and silently never saved.
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
require.Equal(t, true, reserved["ok"])
@@ -205,6 +207,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// The size the client declares is what Commit validates the assembled archive against.
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
@@ -215,6 +218,8 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// Both API versions are served from one store, so an entry written through v2 is a hit for
// a v1 client asking for the same key and version.
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
require.NoError(t, err)
@@ -227,30 +232,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.NotEmpty(t, got["archiveLocation"])
})
t.Run("a read-only job only restores and a write-only one only saves", func(t *testing.T) {
handler.RegisterJob("read-only", JobCredential{Repo: testRepo, CacheMode: "read"})
readOnlyClient := &http.Client{Transport: &bearerTransport{token: "read-only"}}
assert.Equal(t, true, v2Call(t, handler, readOnlyClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})["ok"])
for _, method := range []string{"CreateCacheEntry", "FinalizeCacheEntryUpload"} {
got := v2Call(t, handler, readOnlyClient, method, map[string]any{"key": "deps-new", "version": "v1", "size_bytes": 1})
assert.Equal(t, false, got["ok"], method)
assert.True(t, strings.HasPrefix(got["message"].(string), "cache write denied:"), method)
}
handler.RegisterJob("write-only", JobCredential{Repo: testRepo, CacheMode: "write-only"})
writeOnlyClient := &http.Client{Transport: &bearerTransport{token: "write-only"}}
assert.Equal(t, true, v2Call(t, handler, writeOnlyClient, "CreateCacheEntry", map[string]any{"key": "deps-write-only", "version": "v1"})["ok"])
resp, err := writeOnlyClient.Post(handler.ExternalURL()+cacheServiceV2Path+"/GetCacheEntryDownloadURL", "application/json", strings.NewReader(`{"key":"deps-abc","version":"v1"}`))
require.NoError(t, err)
defer resp.Body.Close()
got := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got))
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
assert.Equal(t, twirpDenied, got["code"])
assert.True(t, strings.HasPrefix(got["msg"], "cache read denied:"), got["msg"])
})
// The cache of one repository must stay invisible to another, as it does for the v1 API.
t.Run("another repository sees nothing", func(t *testing.T) {
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
@@ -258,114 +240,4 @@ func TestCacheServiceV2Lookups(t *testing.T) {
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
assert.Equal(t, false, got["ok"])
})
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
clients := map[string]*http.Client{"x": testClient}
for _, ref := range []string{"y", "z"} {
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
require.Equal(t, true, created["ok"])
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
}
for _, ref := range []string{"y", "z"} {
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
}
for content, client := range clients {
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
}
})
}
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
tests := []struct {
name string
token string
body string
status int
code string
}{
{"no bearer at all", "", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a bearer nobody registered", "not-a-job", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a body that is not the request", testToken, `{`, http.StatusBadRequest, "malformed"},
{"a request missing its key", testToken, `{"version":"v"}`, http.StatusBadRequest, "invalid_argument"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
handler.ExternalURL()+cacheServiceV2Path+"/CreateCacheEntry", strings.NewReader(tt.body))
require.NoError(t, err)
if tt.token != "" {
req.Header.Set("Authorization", "Bearer "+tt.token)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.status, resp.StatusCode)
got := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got))
assert.Equal(t, tt.code, got["code"])
assert.NotEmpty(t, got["msg"])
})
}
}
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
content := []byte("the cached archive")
const otherJob = "another-jobs-token"
defer handler.RegisterJob(otherJob, JobCredential{Repo: testRepo})()
reserve := func(client *http.Client) map[string]any {
return v2Call(t, handler, client, "CreateCacheEntry", map[string]any{"key": "shared", "version": "v1"})
}
first := reserve(testClient)
require.Equal(t, true, first["ok"])
retry := reserve(testClient)
assert.Equal(t, true, retry["ok"], "a job retrying its own reservation is not a conflict")
entryOf := func(reserved map[string]any) string {
parsed, err := url.Parse(reserved["signed_upload_url"].(string))
require.NoError(t, err)
return parsed.Path
}
assert.Equal(t, entryOf(first), entryOf(retry), "the same reservation, however freshly its URL is signed")
assert.Equal(t, false, reserve(&http.Client{Transport: &bearerTransport{token: otherJob}})["ok"],
"another job's reservation would be finalized against the first upload")
otherClient := &http.Client{Transport: &bearerTransport{token: otherJob}}
reserved := v2Call(t, handler, otherClient, "CreateCacheEntry", map[string]any{"key": "owned", "version": "v1"})
require.Equal(t, true, reserved["ok"])
require.Equal(t, http.StatusCreated, putBlob(t, reserved["signed_upload_url"].(string), content))
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "owned", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"another job's upload is not this job's to finalize")
base := handler.ExternalURL() + apiPath
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
require.NoError(t, err)
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
resp, err := testClient.Do(upload)
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "v1-made", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"a reservation this job did not make through v2 is not its to commit")
finalized, _ := saveV2(t, handler, "deps", "v1", content)
require.Equal(t, true, finalized["ok"])
again := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "deps", "version": "v1", "size_bytes": strconv.Itoa(len(content)),
})
assert.Equal(t, true, again["ok"], "a retry whose first answer was lost must not report a failed save")
assert.Equal(t, finalized["entry_id"], again["entry_id"])
}
+7 -16
View File
@@ -4,8 +4,6 @@
package artifactcache
import "time"
type Request struct {
Key string `json:"key" `
Version string `json:"version"`
@@ -13,19 +11,12 @@ type Request struct {
}
type Cache struct {
ID uint64 `json:"id"`
Repo string `json:"repo"`
Ref string `json:"ref,omitzero"`
Key string `json:"key"`
Version string `json:"version"`
ID uint64 `json:"id" boltholdKey:"ID"`
Repo string `json:"repo" boltholdIndex:"Repo"`
Key string `json:"key" boltholdIndex:"Key"`
Version string `json:"version" boltholdIndex:"Version"`
Size int64 `json:"cacheSize"`
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
Complete bool `json:"complete"`
UsedAt int64 `json:"usedAt"`
CreatedAt int64 `json:"createdAt"`
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
}
func (cache *Cache) createdAtNano() int64 {
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
Complete bool `json:"complete" boltholdIndex:"Complete"`
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
}
+10 -40
View File
@@ -4,10 +4,7 @@
package artifactcache
import (
"cmp"
"crypto/tls"
"errors"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
@@ -18,38 +15,23 @@ import (
// Gitea implements only the artifact half of it. Forwarding that half from here makes this origin
// the whole service, so ACTIONS_RESULTS_URL can point at it truthfully, which is what the clients
// this runner cannot patch need, docker buildx among them.
//
// The instance to forward to travels with the job registration rather than with configuration, so
// a cache server shared between runners serves each of their instances.
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService/"
// forwardOrNotFound is the router's fallback: the artifact service of the instance the job
// registered with, and the 404 the router would have written otherwise.
func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, artifactServicePath) {
cred, ok := h.lookupCredential(bearerToken(r))
if !ok || cred.Results == "" || !strings.HasPrefix(r.URL.Path, artifactServicePath) {
http.NotFound(w, r)
return
}
cred, ok := h.lookupCredential(bearerToken(r))
if !ok {
h.twirpError(w, r, twirpUnauthenticated, errors.New("unknown bearer token"))
return
}
if cred.Results == "" {
h.twirpError(w, r, twirpInternal, errors.New("no instance is registered for this job"))
return
}
target, err := url.Parse(strings.TrimSuffix(cred.Results, "/"))
if err == nil && (target.Hostname() == "" || (target.Scheme != "http" && target.Scheme != "https")) {
err = errors.New("not an absolute http or https URL the job could use")
}
if err != nil {
h.logger.Errorf("artifact service forward to %q: %v", cred.Results, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("artifact service address %q is unusable: %w", cred.Results, err))
return
}
if !h.mustProxy(r, cred, target) {
redirect := target.JoinPath(r.URL.EscapedPath())
redirect.RawQuery = mergeQuery(target.RawQuery, r.URL.RawQuery)
h.logger.Debugf("%s %s: redirecting to %s", r.Method, r.URL.Path, target)
http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect)
w.WriteHeader(http.StatusBadGateway)
return
}
h.logger.Debugf("%s %s: forwarding to %s", r.Method, r.URL.Path, target)
@@ -62,28 +44,16 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
// directly and never through here.
r.Out.Host = target.Host
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
h.logger.Warnf("artifact service forward to %s: %v", target, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
w.WriteHeader(http.StatusBadGateway)
},
}
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
if cred.InsecureTLS {
proxy.Transport = insecureTransport
}
proxy.ServeHTTP(w, r)
}
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
}
func mergeQuery(target, request string) string {
if target == "" || request == "" {
return target + request
}
return target + "&" + request
}
// insecureTransport is shared, because a transport per request would pool no connections.
var insecureTransport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
var insecureTransport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
+16 -112
View File
@@ -4,8 +4,6 @@
package artifactcache
import (
"cmp"
"encoding/json/v2"
"io"
"net/http"
"net/http/httptest"
@@ -16,137 +14,43 @@ import (
"github.com/stretchr/testify/require"
)
// The artifact half is forwarded under the Host Gitea knows itself by, so the URLs it hands back
// still point at Gitea, and nothing else is proxied.
func TestFrontResultsService(t *testing.T) {
t.Parallel()
var gotHost, gotPath, gotProto string
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
_, _ = io.WriteString(w, `{"ok":true}`)
}))
defer gitea.Close()
selfSigned := httptest.NewTLSServer(gitea.Config.Handler)
defer selfSigned.Close()
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"})
require.NoError(t, err)
defer handler.Close()
const token, rpc = "forward-token", artifactServicePath + "CreateArtifact"
const token = "forward-token"
noRedirect := func(token string) *http.Client {
return &http.Client{
Transport: &bearerTransport{token: token},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
}
post := func(path string) (int, map[string]string) {
client := &http.Client{Transport: &bearerTransport{token: token}}
post := func(path string) int {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+path, nil)
require.NoError(t, err)
resp, err := noRedirect(token).Do(req)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := map[string]string{}
_ = json.UnmarshalRead(resp.Body, &body)
return resp.StatusCode, body
resp.Body.Close()
return resp.StatusCode
}
status, body := post(rpc)
assert.Equal(t, http.StatusUnauthorized, status, "an unregistered token is told so, not left to parse a 404 page")
assert.Equal(t, twirpUnauthenticated, body["code"])
assert.Contains(t, body["msg"], "bearer")
assert.NotEmpty(t, body["msg"])
assert.Equal(t, http.StatusNotFound, post(artifactServicePath+"CreateArtifact"),
"an unregistered token is forwarded nowhere")
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: selfSigned.URL, InsecureTLS: true})()
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: gitea.URL})()
status, _ = post(rpc)
require.Equal(t, http.StatusOK, status, "an http job reaching an https instance must proxy")
assert.Equal(t, strings.TrimPrefix(selfSigned.URL, "https://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Equal(t, http.StatusOK, post(artifactServicePath+"CreateArtifact"))
assert.Equal(t, strings.TrimPrefix(gitea.URL, "http://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Empty(t, gotProto, "a forwarded scheme would make an https Gitea mint http URLs")
assert.Equal(t, rpc, gotPath)
assert.Equal(t, artifactServicePath+"CreateArtifact", gotPath)
gotPath = ""
status, _ = post("/twirp/github.actions.results.api.v1.OtherService/Do")
assert.Equal(t, http.StatusNotFound, status)
status, _ = post("/api/v1/repos/owner/repo")
assert.Equal(t, http.StatusNotFound, status)
assert.Equal(t, http.StatusNotFound, post("/twirp/github.actions.results.api.v1.OtherService/Do"))
assert.Equal(t, http.StatusNotFound, post("/api/v1/repos/owner/repo"))
assert.Empty(t, gotPath, "only the artifact service is forwarded")
tests := []struct {
name string
cred JobCredential
forwardedProto string
reqQuery string
want int
location string
wantMsg string
}{
{name: "an instance the job reaches itself", cred: JobCredential{Results: gitea.URL}, want: http.StatusTemporaryRedirect},
{name: "a sub-path instance keeps its prefix", cred: JobCredential{Results: gitea.URL + "/sub"}, want: http.StatusTemporaryRedirect},
{name: "an instance with no certificate to distrust", cred: JobCredential{Results: gitea.URL, InsecureTLS: true}, want: http.StatusTemporaryRedirect},
{
name: "both queries survive", cred: JobCredential{Results: gitea.URL + "?tenant=one"}, reqQuery: "?run=7",
want: http.StatusTemporaryRedirect, location: gitea.URL + rpc + "?tenant=one&run=7",
},
{
name: "a trusted https instance is reached by the job itself",
cred: JobCredential{Results: selfSigned.URL}, forwardedProto: "https", want: http.StatusTemporaryRedirect,
},
{
name: "the job does not share this server's disregard for the certificate",
cred: JobCredential{Results: selfSigned.URL, InsecureTLS: true}, forwardedProto: "https", want: http.StatusOK,
},
{name: "an http job is not redirected to an https instance", cred: JobCredential{Results: selfSigned.URL}, want: http.StatusInternalServerError},
{
name: "a forwarded https scheme onto an http instance is proxied",
cred: JobCredential{Results: gitea.URL}, forwardedProto: "https", want: http.StatusOK,
},
{
name: "a scheme the job could not use is refused before it is sent one",
cred: JobCredential{Results: "ftp://gitea.example"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
{
name: "an address with no host is refused before it is sent one",
cred: JobCredential{Results: "http://:3000"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer handler.RegisterJob(tt.name, tt.cred)()
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+rpc+tt.reqQuery, nil)
require.NoError(t, err)
if tt.forwardedProto != "" {
req.Header.Set("X-Forwarded-Proto", tt.forwardedProto)
}
resp, err := noRedirect(tt.name).Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.want, resp.StatusCode)
switch tt.want {
case http.StatusTemporaryRedirect:
assert.Equal(t, cmp.Or(tt.location, tt.cred.Results+rpc), resp.Header.Get("Location"))
case http.StatusInternalServerError:
body := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &body))
assert.Equal(t, twirpInternal, body["code"])
assert.Contains(t, body["msg"], cmp.Or(tt.wantMsg, tt.cred.Results))
}
})
}
}
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
t.Parallel()
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
defer gitea.Close()
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
require.NoError(t, err)
defer handler.Close()
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
require.NoError(t, err)
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
}
-348
View File
@@ -1,348 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"bytes"
"cmp"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/json/v2"
"encoding/xml"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net/http"
"net/url"
"os"
"path"
"slices"
"strconv"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"golang.org/x/sync/errgroup"
)
// S3Options configures the bucket that stores completed cache entries.
type S3Options struct {
Endpoint string
Region string
Bucket string
Prefix string
PathStyle *bool // nil uses virtual-hosted requests on AWS and path-style ones elsewhere
AccessKey string
SecretKey string
}
type cacheStorage interface {
Exist(uint64) (bool, error)
Write(uint64, int64, io.Reader) error
WriteBlock(uint64, string, io.Reader) error
OrderBlocks(uint64, []string) error
Commit(uint64, int64) (int64, error)
Serve(http.ResponseWriter, *http.Request, uint64)
Remove(uint64) error
}
const s3PartSize = 64 << 20
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
type s3Storage struct {
*Storage
client *http.Client
bucketURL *url.URL
opts S3Options
}
type s3Part struct {
PartNumber int
ETag string
}
func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
endpoint, err := url.Parse(opts.Endpoint)
if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") {
return nil, fmt.Errorf("invalid S3 endpoint %q", opts.Endpoint)
}
transport, _ := http.DefaultTransport.(*http.Transport)
transport = transport.Clone()
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
if opts.PathStyle != nil {
pathStyle = *opts.PathStyle
}
if pathStyle {
endpoint.Path = "/" + opts.Bucket
} else {
endpoint.Host = opts.Bucket + "." + endpoint.Host
endpoint.Path = "/"
}
opts.Region = cmp.Or(opts.Region, "us-east-1")
return &s3Storage{Storage: staging, client: &http.Client{Transport: transport}, bucketURL: endpoint, opts: opts}, nil
}
func (s *s3Storage) blobKey(id uint64) string {
return path.Join(s.opts.Prefix, "blobs", strconv.FormatUint(id, 10))
}
func (s *s3Storage) metadataDir() string {
return path.Join(s.opts.Prefix, "metadata") + "/"
}
func (s *s3Storage) metadataKey(id uint64) string {
return s.metadataDir() + strconv.FormatUint(id, 10) + ".json"
}
func (s *s3Storage) url(key string, query url.Values) *url.URL {
u := *s.bucketURL
u.Path = path.Join(u.Path, key)
u.RawPath = strings.NewReplacer("+", "%20", "%2F", "/").Replace(url.QueryEscape(u.Path)) // SigV4 encoding
u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20")
return &u
}
// sign authorizes req with AWS Signature Version 4, leaving the payload unsigned so bodies can stream.
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
func (s *s3Storage) sign(req *http.Request, now time.Time) {
date := now.UTC().Format("20060102T150405Z")
req.Header.Set("X-Amz-Date", date)
if req.Header.Get("X-Amz-Content-Sha256") == "" {
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
}
headers := map[string]string{"host": req.URL.Host}
for name, values := range req.Header {
if name = strings.ToLower(name); strings.HasPrefix(name, "x-amz-") {
headers[name] = strings.Join(values, ",")
}
}
names := slices.Sorted(maps.Keys(headers))
canonical := []string{req.Method, req.URL.EscapedPath(), req.URL.RawQuery}
for _, name := range names {
canonical = append(canonical, name+":"+headers[name])
}
canonical = append(canonical, "", strings.Join(names, ";"), req.Header.Get("X-Amz-Content-Sha256"))
scope := date[:8] + "/" + s.opts.Region + "/s3/aws4_request"
signature := []byte("AWS4" + s.opts.SecretKey)
for _, data := range []string{date[:8], s.opts.Region, "s3", "aws4_request", fmt.Sprintf("AWS4-HMAC-SHA256\n%s\n%s\n%x", date, scope, sha256.Sum256([]byte(strings.Join(canonical, "\n"))))} { // key derivation, the last round signs
mac := hmac.New(sha256.New, signature)
_, _ = mac.Write([]byte(data))
signature = mac.Sum(nil)
}
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%x", s.opts.AccessKey, scope, strings.Join(names, ";"), signature))
}
// do returns the response to a signed request when it succeeded, fs.ErrNotExist for a missing key.
func (s *s3Storage) do(ctx context.Context, method, key string, query url.Values, header http.Header, body io.Reader) (*http.Response, error) {
var size int64
if sized, ok := body.(interface{ Size() int64 }); ok {
size = sized.Size()
}
if size == 0 {
body = nil
}
req, err := http.NewRequestWithContext(ctx, method, s.url(key, query).String(), body)
if err != nil {
return nil, err
}
req.ContentLength = size
maps.Copy(req.Header, header)
s.sign(req, time.Now())
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < http.StatusMultipleChoices {
return resp, nil
}
defer drain(resp)
var failure struct{ Code, Message string }
_ = xml.NewDecoder(resp.Body).Decode(&failure)
if resp.StatusCode == http.StatusNotFound && (failure.Code == "NoSuchKey" || failure.Code == "" && method == http.MethodHead) {
return nil, fs.ErrNotExist
}
return nil, fmt.Errorf("S3 %s %s: %s", method, key, cmp.Or(strings.TrimSpace(failure.Code+" "+failure.Message), resp.Status))
}
func (s *s3Storage) call(ctx context.Context, method, key string, query url.Values, body io.Reader, result any) (http.Header, error) {
resp, err := s.do(ctx, method, key, query, nil, body)
if err != nil {
return nil, err
}
defer drain(resp)
if result == nil {
return resp.Header, nil
}
return resp.Header, xml.NewDecoder(resp.Body).Decode(result)
}
// drain reads a response to its end before closing it, so its connection gets reused.
func drain(resp *http.Response) {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
func (s *s3Storage) Exist(id uint64) (bool, error) {
_, err := s.call(context.Background(), http.MethodHead, s.blobKey(id), nil, nil, nil)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return err == nil, err
}
func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
written, err := s.Storage.Commit(id, size)
if err != nil {
return 0, err
}
defer func() { _ = s.Storage.Remove(id) }()
file, err := os.Open(s.filename(id))
if err != nil {
return 0, err
}
defer file.Close()
if err := s.upload(context.Background(), s.blobKey(id), file, written); err != nil {
return 0, fmt.Errorf("upload cache to S3: %w", err)
}
return written, nil
}
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
if size <= s3PartSize {
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
return err
}
var created struct {
UploadID string `xml:"UploadId"`
}
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
return err
}
defer func() {
if uploadErr != nil && created.UploadID != "" {
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
}
}()
var completion struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []s3Part `xml:"Part"`
}
partSize := max(int64(s3PartSize), (size-1)/10000+1)
completion.Parts = make([]s3Part, (size-1)/partSize+1)
workers, workerCtx := errgroup.WithContext(ctx)
workers.SetLimit(4)
for part := range completion.Parts {
workers.Go(func() (err error) {
defer common.RecoverPanic(&err)
offset := int64(part) * partSize
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
return err
})
}
if err := workers.Wait(); err != nil {
return err
}
body, err := xml.Marshal(completion)
if err != nil {
return err
}
var result struct{ Code, Message string } // a completion can fail after its 200 status
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploadId": {created.UploadID}}, bytes.NewReader(body), &result); err != nil {
return err
}
if result.Code != "" {
return fmt.Errorf("S3 complete upload %s: %s %s", key, result.Code, result.Message)
}
return nil
}
func (s *s3Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
header := http.Header{}
if byteRange := r.Header.Get("Range"); byteRange != "" {
header.Set("Range", byteRange)
}
resp, err := s.do(r.Context(), http.MethodGet, s.blobKey(id), nil, header, nil)
if errors.Is(err, fs.ErrNotExist) {
http.NotFound(w, r)
return
} else if err != nil {
http.Error(w, "read cache from S3", http.StatusBadGateway)
return
}
defer resp.Body.Close()
for _, name := range []string{"Content-Length", "Content-Range"} {
if value := resp.Header.Get(name); value != "" {
w.Header().Set(name, value)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
// Remove unpublishes an entry before deleting its blob, so no runner imports it without its bytes.
func (s *s3Storage) Remove(id uint64) error {
for _, key := range []string{s.metadataKey(id), s.blobKey(id)} {
if _, err := s.call(context.Background(), http.MethodDelete, key, nil, nil, nil); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove %s from S3: %w", key, err)
}
}
return s.Storage.Remove(id)
}
func (s *s3Storage) publish(cache *Cache) error {
data, err := json.Marshal(cache)
if err != nil {
return err
}
if _, err := s.call(context.Background(), http.MethodPut, s.metadataKey(cache.ID), nil, bytes.NewReader(data), nil); err != nil {
return fmt.Errorf("publish cache to S3: %w", err)
}
return nil
}
func (s *s3Storage) published(ctx context.Context) (map[uint64]bool, error) {
ids := make(map[uint64]bool)
query := url.Values{"list-type": {"2"}, "prefix": {s.metadataDir()}}
for {
var page struct {
Contents []struct{ Key string }
IsTruncated bool
NextContinuationToken string
}
if _, err := s.call(ctx, http.MethodGet, "", query, nil, &page); err != nil {
return nil, fmt.Errorf("list S3 cache metadata: %w", err)
}
for _, object := range page.Contents {
if id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(object.Key, s.metadataDir()), ".json"), 10, 64); err == nil {
ids[id] = true
}
}
if !page.IsTruncated {
return ids, nil
}
query.Set("continuation-token", page.NextContinuationToken)
}
}
func (s *s3Storage) fetch(ctx context.Context, id uint64) (*Cache, error) {
resp, err := s.do(ctx, http.MethodGet, s.metadataKey(id), nil, nil, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
cache := &Cache{}
if err := json.UnmarshalRead(resp.Body, cache); err != nil {
return nil, fmt.Errorf("read S3 cache metadata %d: %w", id, err)
}
return cache, nil
}
-141
View File
@@ -1,141 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"encoding/xml"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
t.Parallel()
var objects sync.Map
var uploads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
key := strings.TrimPrefix(request.URL.Path, "/cache/")
switch {
case request.Method == http.MethodPut:
if strings.Contains(key, "/blobs/") {
uploads.Add(1)
}
body, err := io.ReadAll(request.Body)
assert.NoError(t, err)
objects.Store(key, body)
case request.URL.Query().Has("list-type"):
_, _ = io.WriteString(writer, "<ListBucketResult>")
objects.Range(func(stored, _ any) bool {
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
return true
})
_, _ = io.WriteString(writer, "</ListBucketResult>")
default:
if body, ok := objects.Load(key); ok {
_, _ = writer.Write(body.([]byte))
} else {
writer.WriteHeader(http.StatusNotFound)
}
}
}))
t.Cleanup(server.Close)
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
entry := map[string]any{"key": "deps", "version": "v1"}
started := time.Now()
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
require.Equal(t, true, finalized["ok"])
assert.EqualValues(t, 1, uploads.Load())
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
require.NoError(t, err)
cache, err := saver.s3.fetch(t.Context(), id)
require.NoError(t, err)
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
require.NoError(t, restorer.importMetadata(t.Context()))
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
require.Equal(t, true, found["ok"])
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
for _, cache := range []*Cache{
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
} {
require.NoError(t, saver.s3.publish(cache))
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
}
require.NoError(t, restorer.importMetadata(t.Context()))
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
require.Equal(t, true, found["ok"])
assert.Equal(t, "deps-new", found["matched_key"])
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
objects.Clear()
require.NoError(t, restorer.importMetadata(t.Context()))
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
}
type s3TestTransport func(*http.Request) (*http.Response, error)
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return transport(req)
}
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
t.Parallel()
for _, failingPart := range []string{"", "2"} {
var parts, aborts atomic.Int32
var completion struct {
Parts []s3Part `xml:"Part"`
}
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
switch {
case query.Has("uploads"):
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
case req.Method == http.MethodPut:
parts.Add(1)
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
if query.Get("partNumber") == failingPart {
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
}
case req.Method == http.MethodPost:
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
case req.Method == http.MethodDelete:
aborts.Add(1)
}
return response, nil
})}
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
assert.EqualValues(t, 3, parts.Load())
if failingPart == "" {
require.NoError(t, err)
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
assert.Zero(t, aborts.Load())
} else {
require.ErrorContains(t, err, "InvalidPart")
assert.EqualValues(t, 1, aborts.Load())
}
}
}
func TestS3StorageSignsLikeAWS(t *testing.T) {
t.Parallel()
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
storage.sign(req, time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
assert.Equal(t, "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", req.Header.Get("Authorization"))
}
+37 -44
View File
@@ -5,11 +5,8 @@
package artifactcache
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json/v2"
"errors"
"fmt"
"io"
"net/http"
@@ -33,11 +30,13 @@ func NewStorage(rootDir string) (*Storage, error) {
}
func (s *Storage) Exist(id uint64) (bool, error) {
_, err := os.Stat(s.filename(id))
if os.IsNotExist(err) {
name := s.filename(id)
if _, err := os.Stat(name); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return err == nil, err
return true, nil
}
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
@@ -52,10 +51,9 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, reader)
if err = errors.Join(err, file.Close()); err != nil {
_ = os.Remove(name)
}
return err
}
@@ -68,18 +66,6 @@ func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error
// rename pass is safe because a staged name always carries blockFilePrefix and a target name
// never does, so no rename can collide with a block not yet moved.
func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
wanted, err := json.Marshal(blockIDs) // a block id is client-chosen, so it cannot be a separator
if err != nil {
return err
}
list := filepath.Join(s.tempDir(id), blockFilePrefix+"list")
if recorded, err := os.ReadFile(list); err == nil {
// A retry repeats the list it already sent; another one would reorder what is now staged.
if !bytes.Equal(recorded, wanted) {
return fmt.Errorf("cache %d was already assembled from a different block list", id)
}
return nil
}
for i, blockID := range blockIDs {
if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
if os.IsNotExist(err) {
@@ -88,11 +74,13 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
return err
}
}
return os.WriteFile(list, wanted, 0o600)
return nil
}
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
defer func() {
_ = os.RemoveAll(s.tempDir(id))
}()
name := s.filename(id)
tempNames, err := s.tempNames(id)
@@ -104,50 +92,55 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
return 0, err
}
written, err := assemble(name, tempNames)
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
err = fmt.Errorf("broken file: %v != %v", written, size)
}
if err != nil {
_ = os.Remove(name)
return 0, err
}
// If size is less than 0, it means the size is unknown.
// We can't check the size of the file, just skip the check.
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
if size >= 0 && written != size {
_ = os.Remove(name)
return 0, fmt.Errorf("broken file: %v != %v", written, size)
}
return written, nil
}
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
// produces below the client's block threshold, is already the whole archive and is moved.
func assemble(name string, tempNames []string) (int64, error) {
if len(tempNames) == 0 {
return 0, errors.New("nothing was uploaded")
}
if err := os.Rename(tempNames[0], name); err != nil {
if len(tempNames) == 1 {
info, err := os.Stat(tempNames[0])
if err != nil {
return 0, err
}
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
return info.Size(), os.Rename(tempNames[0], name)
}
file, err := os.Create(name)
if err != nil {
return 0, err
}
defer file.Close()
for _, v := range tempNames[1:] {
part, err := os.Open(v)
var written int64
for _, v := range tempNames {
f, err := os.Open(v)
if err != nil {
return 0, err
}
_, err = io.Copy(file, part)
_ = part.Close()
n, err := io.Copy(file, f)
_ = f.Close()
if err != nil {
return 0, err
}
_ = os.Remove(v)
written += n
}
info, err := file.Stat()
if err != nil {
return 0, err
}
return info.Size(), file.Close()
return written, nil
}
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
http.ServeFile(w, r, s.filename(id))
name := s.filename(id)
http.ServeFile(w, r, name)
}
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
+40 -42
View File
@@ -11,15 +11,15 @@ import (
"fmt"
"io"
"io/fs"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/julienschmidt/httprouter"
)
type FileContainerResourceURL struct {
@@ -66,44 +66,39 @@ func writeJSON(w http.ResponseWriter, value any) {
}
}
func uploads(router *http.ServeMux, baseDir string) {
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
func uploads(router *httprouter.Router, baseDir string) {
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId")
writeJSON(w, FileContainerResourceURL{
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
})
})
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
itemPath := req.URL.Query().Get("itemPath")
runID := params.ByName("runId")
if req.Header.Get("Content-Encoding") == "gzip" {
itemPath += gzipExtension
}
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
safeRunPath := safeResolve(baseDir, runID)
safePath := safeResolve(safeRunPath, itemPath)
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
panic(err)
}
var start int64
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
http.Error(w, "invalid content range", http.StatusBadRequest)
return
}
}
flags := os.O_CREATE | os.O_WRONLY
if start == 0 {
flags |= os.O_TRUNC
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
appendUpload := req.Header.Get("Content-Range")
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
}
file, err := os.OpenFile(safePath, flags, 0o644)
if err != nil {
panic(err)
}
defer file.Close()
if _, err := file.Seek(start, io.SeekStart); err != nil {
panic(err)
}
if req.Body == nil {
panic(errors.New("no body given"))
}
@@ -118,19 +113,21 @@ func uploads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
writeJSON(w, ResponseMessage{
Message: "success",
})
})
}
func downloads(router *http.ServeMux, baseDir string) {
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
runID := req.PathValue("runId")
func downloads(router *httprouter.Router, baseDir string) {
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId")
entries, err := os.ReadDir(safeResolve(baseDir, runID))
if err != nil && !os.IsNotExist(err) {
safePath := safeResolve(baseDir, runID)
entries, err := os.ReadDir(safePath)
if err != nil {
panic(err)
}
@@ -148,16 +145,13 @@ func downloads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
container := req.PathValue("container")
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
container := params.ByName("container")
itemPath := req.URL.Query().Get("itemPath")
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
var files []ContainerItem
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() {
rel, err := filepath.Rel(safePath, path)
if err != nil {
@@ -166,16 +160,20 @@ func downloads(router *http.ServeMux, baseDir string) {
// if it was upload as gzip
rel = strings.TrimSuffix(rel, gzipExtension)
path := filepath.Join(itemPath, rel)
rel = filepath.ToSlash(rel)
path = filepath.ToSlash(path)
files = append(files, ContainerItem{
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
Path: path,
ItemType: "file",
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
})
}
return nil
})
if err != nil && !os.IsNotExist(err) {
if err != nil {
panic(err)
}
@@ -184,8 +182,10 @@ func downloads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
safePath := safeResolve(baseDir, req.PathValue("path"))
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
path := params.ByName("path")[1:]
safePath := safeResolve(baseDir, path)
file, err := os.Open(safePath)
if err != nil {
@@ -213,25 +213,23 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
return cancel
}
router := http.NewServeMux()
router := httprouter.New()
logger.Debugf("Artifacts base path '%s'", artifactPath)
uploads(router, artifactPath)
downloads(router, artifactPath)
registerV4(router, artifactPath)
server := &http.Server{
Addr: net.JoinHostPort(addr, port),
Addr: fmt.Sprintf("%s:%s", addr, port),
ReadHeaderTimeout: 2 * time.Second,
Handler: router,
}
// run server
go func() {
logger.Infof("Start server on http://%s", server.Addr)
logger.Infof("Start server on http://%s:%s", addr, port)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error(err)
os.Exit(1)
logger.Fatal(err)
}
}()
+8 -112
View File
@@ -19,16 +19,16 @@ import (
"testing"
"time"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/require"
)
func TestArtifactFlow(t *testing.T) {
artifactPath := t.TempDir()
router := http.NewServeMux()
router := httprouter.New()
uploads(router, artifactPath)
downloads(router, artifactPath)
registerV4(router, artifactPath)
server := httptest.NewServer(router)
defer server.Close()
@@ -49,17 +49,8 @@ func TestArtifactFlow(t *testing.T) {
return resp.StatusCode, data
}
t.Run("no-artifacts", func(t *testing.T) {
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var list NamedFileContainerResourceURLResponse
require.NoError(t, json.Unmarshal(data, &list))
require.Zero(t, list.Count)
require.Empty(t, list.Value)
})
t.Run("upload-and-download", func(t *testing.T) {
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
@@ -91,13 +82,13 @@ func TestArtifactFlow(t *testing.T) {
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "file", items.Value[0].ItemType)
require.Equal(t, item, items.Value[0].Path)
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
})
@@ -105,25 +96,16 @@ func TestArtifactFlow(t *testing.T) {
t.Run("content-range", func(t *testing.T) {
const rawURL = "/upload/4?itemPath=chunks.txt"
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
require.Equal(t, http.StatusOK, status, string(data))
for range 2 {
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
require.Equal(t, http.StatusOK, status, string(data))
}
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
require.NoError(t, err)
require.Equal(t, "first-second", string(stored))
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
require.Equal(t, http.StatusOK, status, string(data))
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
require.NoError(t, err)
require.Empty(t, stored)
})
t.Run("gzip-roundtrip", func(t *testing.T) {
@@ -175,92 +157,6 @@ func TestArtifactFlow(t *testing.T) {
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
})
t.Run("v4", func(t *testing.T) {
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
call := func(method, body string) (int, map[string]any) {
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
var out map[string]any
require.NoError(t, json.Unmarshal(data, &out), string(data))
return status, out
}
put := func(rawURL, body string) int {
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
return status
}
download := func() (string, http.Header) {
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
require.Equal(t, http.StatusOK, status, signed)
resp, err := client.Get(signed["signedUrl"].(string))
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return string(body), resp.Header
}
status, created := call("CreateArtifact", "{"+artifactRef+"}")
require.Equal(t, http.StatusOK, status, created)
upload := created["signedUploadUrl"].(string)
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
for attempt := range 2 {
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
}
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
require.Equal(t, http.StatusBadRequest, status)
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
require.Equal(t, http.StatusOK, status, finalized)
id := finalized["artifactId"].(string)
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
require.Equal(t, http.StatusOK, status)
require.Equal(t, id, refinalized["artifactId"])
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
require.Equal(t, http.StatusConflict, status)
require.Equal(t, "already_exists", conflict["code"])
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
require.Equal(t, http.StatusOK, status)
artifacts := listed["artifacts"].([]any)
require.Len(t, artifacts, 1)
artifact := artifacts[0].(map[string]any)
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
delete(artifact, "createdAt")
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
require.Empty(t, other["artifacts"])
body, header := download()
require.Equal(t, "hello world", body)
require.Equal(t, "application/zip", header.Get("Content-Type"))
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
require.Equal(t, http.StatusOK, status)
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
require.Equal(t, http.StatusNotFound, status)
require.Equal(t, "not_found", missing["code"])
status, created = call("CreateArtifact", "{"+artifactRef+"}")
require.Equal(t, http.StatusOK, status, created)
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
require.Equal(t, http.StatusCreated, status)
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
require.Equal(t, http.StatusOK, status)
body, header = download()
require.Equal(t, "gz", body)
require.Equal(t, "application/gzip", header.Get("Content-Type"))
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
rerun := http.NewServeMux()
registerV4(rerun, artifactPath)
recorder := httptest.NewRecorder()
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
})
}
func TestSafeResolve(t *testing.T) {
@@ -281,7 +177,7 @@ func TestSafeResolve(t *testing.T) {
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
})
}
}
-329
View File
@@ -1,329 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifacts
import (
"cmp"
"crypto/sha256"
"encoding/hex"
"encoding/json/v2"
"encoding/xml"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"net/url"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
)
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
dir string
mu sync.Mutex
lastID int64
created map[string]bool
}
const (
zipMimeType = "application/zip"
maxTwirpBytes = 1 << 20
)
func registerV4(router *http.ServeMux, baseDir string) {
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
}
type twirpError struct {
status int
code string
msg string
}
func (e *twirpError) Error() string { return e.msg }
func notFound(name string) error {
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
}
func badRequest(format string, args ...any) error {
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
}
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req Req
var resp Resp
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
if err != nil {
err = badRequest("malformed request: %v", err)
} else {
resp, err = handle(r, req)
}
w.Header().Set("Content-Type", "application/json")
if err != nil {
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
errors.As(err, &terr)
w.WriteHeader(terr.status)
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
return
}
_ = json.MarshalWrite(w, resp)
}
}
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
sum := sha256.Sum256([]byte(name))
return hex.EncodeToString(sum[:])
}
func (s *serverV4) runDir(runID string) string {
return filepath.Join(s.dir, fileName(runID))
}
func (s *serverV4) artifactDir(runID, name string) string {
return filepath.Join(s.runDir(runID), fileName(name))
}
func (s *serverV4) urlDir(r *http.Request) string {
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
}
func readMeta(dir string) (*monolithArtifact, error) {
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
if err != nil {
return nil, err
}
var meta monolithArtifact
return &meta, json.Unmarshal(data, &meta)
}
func writeMeta(dir string, meta *monolithArtifact) error {
data, err := json.Marshal(meta)
if err != nil {
return err
}
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
}
func blobURL(r *http.Request, endpoint, runID, name string) string {
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
}
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
s.mu.Lock()
defer s.mu.Unlock()
if s.created[dir] {
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
}
if err := os.RemoveAll(dir); err != nil {
return createArtifactResponse{}, err
}
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
return createArtifactResponse{}, err
}
s.lastID++
if err := writeMeta(dir, &monolithArtifact{
WorkflowRunBackendID: req.WorkflowRunBackendID,
DatabaseID: s.lastID,
Name: req.Name,
CreatedAt: time.Now().UTC().Truncate(time.Second),
}); err != nil {
return createArtifactResponse{}, err
}
s.created[dir] = true
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
}
type blockList struct {
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
}
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
dir := s.urlDir(r)
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
return
}
contentType := r.Header.Get("x-ms-blob-content-type")
var err error
switch comp := r.URL.Query().Get("comp"); comp {
case "block":
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
case "blocklist":
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
case "": // Put Blob, which docker/build-push-action uses
err = commitBlob(dir, contentType, func(out io.Writer) error {
_, err := io.Copy(out, r.Body)
return err
})
default:
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusCreated)
}
func writeFile(name string, body io.Reader) error {
file, err := os.Create(name)
if err != nil {
return err
}
_, err = io.Copy(file, body)
return errors.Join(err, file.Close())
}
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
out, err := os.CreateTemp(dir, "blob")
if err != nil {
return err
}
if err = errors.Join(write(out), out.Close()); err == nil {
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
}
if err != nil {
return errors.Join(err, os.Remove(out.Name()))
}
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
}
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
var list blockList
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
return err
}
for _, id := range list.Blocks {
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
return err
}
}
return nil
}
func appendBlock(out io.Writer, name string) error {
in, err := os.Open(name)
if err != nil {
return err
}
defer in.Close()
_, err = io.Copy(out, in)
return err
}
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
meta, err := readMeta(dir)
if err != nil {
return finalizeArtifactResponse{}, notFound(req.Name)
} else if meta.Digest != "" {
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
}
blob, err := os.Open(filepath.Join(dir, "blob"))
if err != nil {
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
}
defer blob.Close()
hash := sha256.New()
size, err := io.Copy(hash, blob)
if err != nil {
return finalizeArtifactResponse{}, err
}
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
if size != req.Size {
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
}
if req.Hash != "" && req.Hash != digest {
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
}
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
defer s.mu.Unlock()
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
return finalizeArtifactResponse{}, notFound(req.Name)
}
meta.Size, meta.Digest = size, digest
if err := writeMeta(dir, meta); err != nil {
return finalizeArtifactResponse{}, err
}
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
}
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
run := s.runDir(req.WorkflowRunBackendID)
entries, err := os.ReadDir(run)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return listArtifactsResponse{}, err
}
var resp listArtifactsResponse
for _, entry := range entries {
meta, err := readMeta(filepath.Join(run, entry.Name()))
if err != nil || meta.Digest == "" ||
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
continue
}
resp.Artifacts = append(resp.Artifacts, *meta)
}
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
return resp, nil
}
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
return getSignedArtifactURLResponse{}, notFound(req.Name)
}
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
}
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
dir := s.urlDir(r)
meta, err := readMeta(dir)
if err != nil || meta.Digest == "" {
http.Error(w, "artifact not found", http.StatusNotFound)
return
}
blob, err := os.Open(filepath.Join(dir, "blob"))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer blob.Close()
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
contentType, filename := string(data), meta.Name
if !strings.Contains(contentType, "/") {
contentType = zipMimeType
}
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
filename += ".zip"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
http.ServeContent(w, r, "", meta.CreatedAt, blob)
}
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
s.mu.Lock()
defer s.mu.Unlock()
meta, err := readMeta(dir)
if err != nil {
return deleteArtifactResponse{}, notFound(req.Name)
}
delete(s.created, dir)
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
}
-23
View File
@@ -1,23 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifacts
import (
"crypto/rand"
"encoding/base64"
"encoding/json/v2"
"time"
)
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
now := time.Now()
claims, _ := json.Marshal(map[string]any{
"scp": "Actions.Results:" + fileName(run) + ":1",
"ac": `[{"Scope":"","Permission":3}]`,
"nbf": now.Unix(),
"exp": now.Add(lifetime).Unix(),
"jti": rand.Text(),
})
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
}
-55
View File
@@ -1,55 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifacts
import "time"
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
createArtifactResponse struct {
Ok bool `json:"ok"`
SignedUploadURL string `json:"signedUploadUrl"`
}
finalizeArtifactRequest struct {
WorkflowRunBackendID string
Name string
Size int64 `json:",string"`
Hash string
}
finalizeArtifactResponse struct {
Ok bool `json:"ok"`
ArtifactID int64 `json:"artifactId,string"`
}
listArtifactsRequest struct {
WorkflowRunBackendID string
NameFilter string
IDFilter int64 `json:",string"`
}
listArtifactsResponse struct {
Artifacts []monolithArtifact `json:"artifacts"`
}
monolithArtifact struct {
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
DatabaseID int64 `json:"databaseId,string"`
Name string `json:"name"`
Size int64 `json:"size,string"`
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
Digest string `json:"digest"` // empty until the upload is finalized
}
getSignedArtifactURLRequest struct {
WorkflowRunBackendID string
Name string
}
getSignedArtifactURLResponse struct {
SignedURL string `json:"signedUrl"`
}
createArtifactRequest = getSignedArtifactURLRequest
deleteArtifactRequest = getSignedArtifactURLRequest
deleteArtifactResponse = finalizeArtifactResponse
)
+25 -18
View File
@@ -4,15 +4,15 @@
package common
import (
"context"
"errors"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/sirupsen/logrus"
)
func TestDryrunContext(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
if Dryrun(ctx) {
t.Fatal("plain context should not be dryrun")
}
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
}
func TestJobErrorContainer(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
err := errors.New("job failed")
SetJobError(ctx, err)
@@ -40,27 +40,34 @@ func TestJobErrorContainer(t *testing.T) {
}
}
func TestLoggerAndHandlerContext(t *testing.T) {
ctx := t.Context()
if Logger(ctx) != logging.Default() {
t.Fatal("plain context should use the default logger")
func TestLoggerAndHookContext(t *testing.T) {
ctx := context.Background()
if Logger(ctx) != logrus.StandardLogger() {
t.Fatal("plain context should use standard logger")
}
if LoggerHandler(ctx) != nil {
t.Fatal("plain context should not have a logger handler")
}
if StepIDs(ctx) != nil {
t.Fatal("plain context should not have step IDs")
if LoggerHook(ctx) != nil {
t.Fatal("plain context should not have a logger hook")
}
logger := logging.New(logtest.New())
logger := logrus.New()
ctx = WithLogger(ctx, logger)
if Logger(ctx) != logger {
t.Fatal("WithLogger should set logger")
}
handler := logtest.New()
ctx = WithLoggerHandler(ctx, handler)
if LoggerHandler(ctx) != handler {
t.Fatal("WithLoggerHandler should set handler")
hook := testHook{}
ctx = WithLoggerHook(ctx, hook)
if LoggerHook(ctx) != hook {
t.Fatal("WithLoggerHook should set hook")
}
}
type testHook struct{}
func (testHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func (testHook) Fire(*logrus.Entry) error {
return nil
}
+6 -1
View File
@@ -14,9 +14,14 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
// Dryrun returns true if the current context is dryrun
func Dryrun(ctx context.Context) bool {
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
val := ctx.Value(dryrunContextKeyVal)
if val != nil {
if dryrun, ok := val.(bool); ok {
return dryrun
}
}
return false
}
// WithDryrun adds a value to the context for dryrun
func WithDryrun(ctx context.Context, dryrun bool) context.Context {
+29 -49
View File
@@ -7,11 +7,9 @@ package common
import (
"context"
"fmt"
"math/rand/v2"
"runtime/debug"
"time"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
)
// Executor define contract for the steps of a workflow
@@ -23,7 +21,8 @@ type Conditional func(ctx context.Context) bool
// NewInfoExecutor is an executor that logs messages
func NewInfoExecutor(format string, args ...any) Executor {
return func(ctx context.Context) error {
Logger(ctx).Infof(format, args...)
logger := Logger(ctx)
logger.Infof(format, args...)
return nil
}
}
@@ -31,7 +30,8 @@ func NewInfoExecutor(format string, args ...any) Executor {
// NewDebugExecutor is an executor that logs messages
func NewDebugExecutor(format string, args ...any) Executor {
return func(ctx context.Context) error {
Logger(ctx).Debugf(format, args...)
logger := Logger(ctx)
logger.Debugf(format, args...)
return nil
}
}
@@ -39,7 +39,9 @@ func NewDebugExecutor(format string, args ...any) Executor {
// NewPipelineExecutor creates a new executor from a series of other executors
func NewPipelineExecutor(executors ...Executor) Executor {
if len(executors) == 0 {
return NewErrorExecutor(nil)
return func(ctx context.Context) error {
return nil
}
}
var rtn Executor
for _, executor := range executors {
@@ -59,13 +61,6 @@ func NewErrorExecutor(err error) Executor {
}
}
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
if recovered := recover(); recovered != nil {
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
*err = fmt.Errorf("panic: %v", recovered)
}
}
// NewParallelExecutor creates a new executor from a parallel of other executors
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
if len(executors) == 0 {
@@ -79,29 +74,34 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
errs := make(chan error, len(executors))
if 1 > parallel {
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
parallel = 1
}
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
for workerID := range parallel {
go func() {
logging.Debugf("Worker %d started", workerID)
for i := 0; i < parallel; i++ {
go func(workerID int, work <-chan Executor, errs chan<- error) {
log.Debugf("Worker %d started", workerID)
taskCount := 0
for executor := range work {
taskCount++
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
log.Debugf("Worker %d executing task %d", workerID, taskCount)
// Recover from panics in executors to avoid crashing the worker
// goroutine which would leave the runner process hung.
// https://gitea.com/gitea/runner/issues/371
errs <- func() (err error) {
defer RecoverPanic(&err)
defer func() {
if r := recover(); r != nil {
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
err = fmt.Errorf("panic: %v", r)
}
}()
return executor(ctx)
}()
}
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
}()
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
}(i, work, errs)
}
for i := range executors {
@@ -150,7 +150,12 @@ func (e Executor) If(conditional Conditional) Executor {
// IfNot only runs this executor if conditional is true
func (e Executor) IfNot(conditional Conditional) Executor {
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
return func(ctx context.Context) error {
if !conditional(ctx) {
return e(ctx)
}
return nil
}
}
// IfBool only runs this executor if conditional is true
@@ -160,38 +165,13 @@ func (e Executor) IfBool(conditional bool) Executor {
})
}
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
// Retry runs the executor up to attempts times until it succeeds.
func (e Executor) Retry(attempts int) Executor {
return func(ctx context.Context) error {
for attempt := 1; ; attempt++ {
err := e(ctx)
if err == nil || attempt == attempts || ctx.Err() != nil {
return err
}
backoff := RetryBackoff()
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
select {
case <-ctx.Done():
return err
case <-time.After(backoff):
}
}
}
}
// Finally adds an executor to run after other executor
func (e Executor) Finally(finally Executor) Executor {
return func(ctx context.Context) error {
err := e(ctx)
err2 := finally(ctx)
if err2 != nil {
if err != nil {
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
}
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
}
return err
}
+96 -81
View File
@@ -7,36 +7,53 @@ package common
import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewWorkflow(t *testing.T) {
ctx := t.Context()
require.NoError(t, NewPipelineExecutor()(ctx))
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
assert := assert.New(t)
ctx := context.Background()
// empty
emptyWorkflow := NewPipelineExecutor()
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
// error case
errorWorkflow := NewErrorExecutor(errors.New("test error"))
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
// multiple success case
runcount := 0
increment := func(_ctx context.Context) error {
successWorkflow := NewPipelineExecutor(
func(ctx context.Context) error {
runcount++
return nil
}
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
assert.Equal(t, 2, runcount)
},
func(ctx context.Context) error {
runcount++
return nil
})
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(2, runcount)
}
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
// find the gate already open so the last one still finishes with no partner left.
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
var counted, active, peak atomic.Int32
var once sync.Once
reached := make(chan struct{})
return func(_ctx context.Context) error {
return func(ctx context.Context) error {
counted.Add(1)
running := active.Add(1)
for {
@@ -55,72 +72,93 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
}
func TestNewParallelExecutor(t *testing.T) {
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
t.Run(name, func(t *testing.T) {
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
assert.Equal(t, int32(3), count.Load())
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
})
}
ctx := context.Background()
exec, count, maxActive := concurrencyProbe(2)
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
// parallelism below 1 falls back to a single worker
exec, count, maxActive = concurrencyProbe(1)
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
}
func TestNewParallelExecutorEmpty(t *testing.T) {
assert := assert.New(t)
ctx := t.Context()
ctx := context.Background()
require.NoError(t, NewParallelExecutor(2)(ctx))
canceledCtx, cancel := context.WithCancel(t.Context())
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
err := NewParallelExecutor(2)(canceledCtx)
assert.ErrorIs(err, context.Canceled)
}
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
for name, tc := range map[string]struct {
canceled bool
parallel, tasks int
}{
"failure": {false, 2, 5},
"canceled": {true, 2, 5},
"canceled single task": {true, 1, 1},
} {
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if tc.canceled {
func TestNewParallelExecutorFailed(t *testing.T) {
assert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()
count := 0
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count++
return errors.New("fake error")
})
err := NewParallelExecutor(1, errorWorkflow)(ctx)
assert.Equal(1, count)
assert.ErrorIs(context.Canceled, err)
}
var calls atomic.Int32
executors := make([]Executor, tc.tasks)
func TestNewParallelExecutorCanceled(t *testing.T) {
assert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()
errExpected := errors.New("fake error")
var count atomic.Int32
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count.Add(1)
return nil
})
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count.Add(1)
return errExpected
})
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
assert.Equal(int32(3), count.Load())
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
}
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
var successCount atomic.Int32
executors := make([]Executor, 5)
for i := range executors {
executors[i] = func(_ctx context.Context) error {
calls.Add(1)
if i == tc.tasks/2 {
executors[i] = func(ctx context.Context) error {
if i == 2 {
return errors.New("fake error")
}
successCount.Add(1)
return nil
}
}
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
if tc.canceled {
require.ErrorIs(t, err, context.Canceled)
} else {
require.EqualError(t, err, "fake error")
}
assert.Equal(t, int32(tc.tasks), calls.Load())
})
}
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
}
func TestExecutorConditionalsAndFinally(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
var calls []string
record := func(name string) Executor {
return func(_ctx context.Context) error {
return func(ctx context.Context) error {
calls = append(calls, name)
return nil
}
@@ -132,42 +170,19 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
require.NoError(t, record("if-bool").IfBool(true)(ctx))
require.NoError(t, record("main").Finally(record("finally"))(ctx))
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
}
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
backoff := RetryBackoff
RetryBackoff = func() time.Duration { return 0 }
t.Cleanup(func() { RetryBackoff = backoff })
transient := errors.New("transient")
for _, tc := range []struct {
failures, calls int
err error
}{
{failures: 2, calls: 3},
{failures: 5, calls: 3, err: transient},
} {
var calls int
err := Executor(func(context.Context) error {
calls++
if calls <= tc.failures {
return transient
}
return nil
}).Retry(3)(t.Context())
require.Equal(t, tc.err, err)
require.Equal(t, tc.calls, calls)
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("calls = %v, want %v", calls, want)
}
}
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
mainErr := errors.New("main failed")
finalErr := errors.New("cleanup failed")
for _, mainErr := range []error{context.Canceled, nil} {
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
require.ErrorIs(t, err, finalErr)
if mainErr != nil {
require.ErrorIs(t, err, mainErr)
}
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
require.Error(t, err)
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
t.Fatalf("finally error = %q, want both cleanup and original error", err)
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+424 -376
View File
File diff suppressed because it is too large. Load diff
+311 -370
View File
File diff suppressed because it is too large. Load diff
+8 -2
View File
@@ -14,9 +14,14 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
// JobError returns the job error for current context if any
func JobError(ctx context.Context) error {
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
val := ctx.Value(jobErrorContextKeyVal)
if val != nil {
if container, ok := val.(map[string]error); ok {
return container["error"]
}
}
return nil
}
func SetJobError(ctx context.Context, err error) {
if container, ok := ctx.Value(jobErrorContextKeyVal).(map[string]error); ok {
@@ -26,5 +31,6 @@ func SetJobError(ctx context.Context, err error) {
// WithJobErrorContainer adds a value to the context as a container for an error
func WithJobErrorContainer(ctx context.Context) context.Context {
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
container := map[string]error{}
return context.WithValue(ctx, jobErrorContextKeyVal, container)
}
+20 -8
View File
@@ -26,7 +26,9 @@ type lineWriter struct {
// NewLineWriter creates a new instance of a line writer
func NewLineWriter(handlers ...LineHandler) io.Writer {
return &lineWriter{handlers: handlers}
w := new(lineWriter)
w.handlers = handlers
return w
}
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
@@ -37,15 +39,24 @@ func FlushWriter(w io.Writer) {
}
}
func (lw *lineWriter) Write(p []byte) (int, error) {
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
_, _ = lw.buffer.Write(line)
if len(line) > 0 && line[len(line)-1] == '\n' {
func (lw *lineWriter) Write(p []byte) (n int, err error) {
pBuf := bytes.NewBuffer(p)
written := 0
for {
line, err := pBuf.ReadString('\n')
w, _ := lw.buffer.WriteString(line)
written += w
if err != nil {
if err == io.EOF {
break
}
return written, err
}
lw.handleLine(lw.buffer.String())
lw.buffer.Reset()
}
}
return len(p), nil
return written, nil
}
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
@@ -61,7 +72,8 @@ func (lw *lineWriter) Flush() {
func (lw *lineWriter) handleLine(line string) {
for _, h := range lw.handlers {
if !h(line) {
ok := h(line)
if !ok {
break
}
}
+35 -13
View File
@@ -9,24 +9,24 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
var lines []string
lineWriter := NewLineWriter(func(s string) bool {
func TestLineWriter(t *testing.T) {
lines := make([]string, 0)
lineHandler := func(s string) bool {
lines = append(lines, s)
return true
})
}
lineWriter := NewLineWriter(lineHandler)
assert := assert.New(t)
write := func(s string) {
n, err := lineWriter.Write([]byte(s))
require.NoError(t, err)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(len(s), n, s)
}
write("")
write("hello")
write(" ")
write("world!!\nextra")
@@ -34,17 +34,39 @@ func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
write(" line\n")
write("no newline here...")
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
assert.Equal(expected, lines)
assert.Len(lines, 4)
assert.Equal("hello world!!\n", lines[0])
assert.Equal("extra line\n", lines[1])
assert.Equal(" and another\n", lines[2])
assert.Equal("last line\n", lines[3])
}
FlushWriter(lineWriter)
expected = append(expected, "no newline here...")
assert.Equal(expected, lines)
func TestLineWriterFlush(t *testing.T) {
lines := make([]string, 0)
lineHandler := func(s string) bool {
lines = append(lines, s)
return true
}
lineWriter := NewLineWriter(lineHandler)
assert := assert.New(t)
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
// Only the newline-terminated line is emitted before flushing.
assert.Equal([]string{"complete line\n"}, lines)
// Flushing emits the buffered, not-yet-terminated trailing line.
FlushWriter(lineWriter)
assert.Equal(expected, lines)
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
// Flushing again is a no-op: nothing is buffered.
FlushWriter(lineWriter)
assert.Len(lines, 2)
}
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
// FlushWriter must be a safe no-op for writers that do not buffer lines.
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
}
+24 -29
View File
@@ -7,51 +7,46 @@ package common
import (
"context"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
)
type loggerContextKey string
const loggerContextKeyVal = loggerContextKey("logging.Logger")
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
// Logger returns the appropriate logger for current context
func Logger(ctx context.Context) *logging.Logger {
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
func Logger(ctx context.Context) logrus.FieldLogger {
val := ctx.Value(loggerContextKeyVal)
if val != nil {
if logger, ok := val.(logrus.FieldLogger); ok {
return logger
}
return logging.Default()
}
return logrus.StandardLogger()
}
// WithLogger adds a value to the context for the logger
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
return context.WithValue(ctx, loggerContextKeyVal, logger)
}
type loggerHandlerKey string
type loggerHookKey string
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
// It affects the job logger, not the global logger.
func LoggerHandler(ctx context.Context) logging.Handler {
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
return h
// LoggerHook returns the appropriate logger hook for current context
// the hook affects job logger, not global logger
func LoggerHook(ctx context.Context) logrus.Hook {
val := ctx.Value(loggerHookKeyVal)
if val != nil {
if hook, ok := val.(logrus.Hook); ok {
return hook
}
}
return nil
}
// WithLoggerHandler adds the extra handler for the job logger to the context
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
return context.WithValue(ctx, loggerHandlerKeyVal, h)
}
type stepIDsContextKey struct{}
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
func StepIDs(ctx context.Context) []string {
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
return ids
}
// WithStepIDs replaces the step IDs in the context
func WithStepIDs(ctx context.Context, ids []string) context.Context {
return context.WithValue(ctx, stepIDsContextKey{}, ids)
// WithLoggerHook adds a value to the context for the logger hook
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
return context.WithValue(ctx, loggerHookKeyVal, hook)
}
-284
View File
@@ -1,284 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"context"
"fmt"
"io"
"log/slog"
"maps"
"reflect"
"runtime"
"slices"
"strconv"
"strings"
"sync"
"time"
)
const (
ansiReset = "\x1b[0m"
ansiRed = "\x1b[31m"
ansiYellow = "\x1b[33m"
ansiCyan = "\x1b[36m"
ansiDimCyan = "\x1b[2;36m"
ansiDimWhite = "\x1b[2;37m"
)
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
var consoleStart = time.Now()
// console is the state shared by every consoleHandler derived from one another.
type console struct {
mu sync.Mutex
out io.Writer
terminal bool
opts TextOptions
reportCaller bool
level slog.LevelVar
}
func newConsole(out io.Writer) *console {
c := &console{}
c.setOutput(out)
return c
}
func (c *console) setOutput(w io.Writer) {
terminal := checkIfTerminal(w)
c.mu.Lock()
defer c.mu.Unlock()
c.out, c.terminal = w, terminal
}
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
// and a colored prefix with elapsed or full time on a terminal.
type consoleHandler struct {
c *console
attrs []slog.Attr
}
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
return level >= h.c.level.Level()
}
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
}
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
c := h.c
c.mu.Lock()
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
c.mu.Unlock()
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
for _, a := range h.attrs {
data[a.Key] = a.Value.Resolve().Any()
}
r.Attrs(func(a slog.Attr) bool {
data[a.Key] = a.Value.Resolve().Any()
return true
})
var caller *runtime.Frame
if reportCaller && r.PC != 0 {
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
caller = &frame
}
prefixFieldClashes(data, caller != nil)
keys := slices.Sorted(maps.Keys(data))
var b bytes.Buffer
if terminal && !opts.DisableColors {
printColored(&b, opts, r, caller, keys, data)
} else {
printPlain(&b, opts, r, caller, keys, data)
}
line := b.Bytes()
if opts.Filter != nil {
line = opts.Filter(line)
}
c.mu.Lock()
defer c.mu.Unlock()
_, err := c.out.Write(line)
return err
}
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
func prefixFieldClashes(data map[string]any, reportCaller bool) {
for _, key := range []string{"time", "level", "msg"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
delete(data, key)
}
}
if reportCaller {
for _, key := range []string{"func", "file"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
}
}
}
}
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
if !opts.DisableTimestamp {
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
}
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
if r.Message != "" {
appendKeyValue(b, "msg", r.Message)
}
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if funcVal != "" {
appendKeyValue(b, "func", funcVal)
}
if fileVal != "" {
appendKeyValue(b, "file", fileVal)
}
}
for _, k := range keys {
appendKeyValue(b, k, data[k])
}
b.WriteByte('\n')
}
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
if opts.CallerPrettyfier != nil {
return opts.CallerPrettyfier(caller)
}
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
}
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
msg := strings.TrimSuffix(r.Message, "\n")
var callerStr string
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if opts.CallerPrettyfier == nil && caller.Function != "" {
funcVal += "()"
}
switch {
case fileVal == "":
callerStr = funcVal
case funcVal == "":
callerStr = fileVal
default:
callerStr = fileVal + " " + funcVal
}
}
level := Level(r.Level)
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
switch {
case opts.DisableTimestamp:
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
case !opts.FullTimestamp:
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
default:
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
}
for _, k := range keys {
b.WriteByte(' ')
b.WriteString(colorize(level, k))
b.WriteByte('=')
appendValue(b, data[k])
}
b.WriteByte('\n')
}
// plainLevel spells Warn the way logrus did, as "warning".
func plainLevel(l Level) string {
if s := l.String(); s != "warn" {
return s
}
return "warning"
}
func colorize(l Level, s string) string {
var color string
switch {
case l <= LevelTrace:
color = ansiDimWhite
case l <= LevelDebug:
color = ansiDimCyan
case l <= LevelInfo:
color = ansiCyan
case l <= LevelWarn:
color = ansiYellow
default:
color = ansiRed
}
return color + s + ansiReset
}
func appendKeyValue(b *bytes.Buffer, key string, value any) {
if b.Len() > 0 {
b.WriteByte(' ')
}
b.WriteString(key)
b.WriteByte('=')
appendValue(b, value)
}
func appendValue(b *bytes.Buffer, value any) {
switch v := value.(type) {
case string:
appendString(b, v)
case []byte:
appendString(b, string(v))
case error:
appendFormatted(b, v, v.Error)
case fmt.Stringer:
appendFormatted(b, v, v.String)
default:
appendString(b, fmt.Sprint(value))
}
}
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
defer func() {
if r := recover(); r != nil {
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
appendString(b, "<nil>")
} else {
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
}
}
}()
appendString(b, get())
}
func appendString(b *bytes.Buffer, s string) {
if needsQuoting(s) {
b.WriteString(strconv.Quote(s))
return
}
b.WriteString(s)
}
func needsQuoting(s string) bool {
for i := range len(s) {
c := s[i]
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
default:
return true
}
}
return false
}
-78
View File
@@ -1,78 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
t.Helper()
var out bytes.Buffer
c := newConsole(&out)
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
c.level.Set(slog.LevelDebug)
build(New(NewSlogHandler(&consoleHandler{c: c})))
return out.String()
}
func TestConsolePlain(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
l.With("msg", "clash", "empty", "").Info("")
})
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
"level=info empty= fields.msg=clash\n", line)
}
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
opts := TextOptions{
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
},
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
}
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
}
func TestConsoleColored(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
l.With("job", "j").Warn("careful")
})
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
assert.Equal(t, "level=info msg=x\n", plain)
}
func TestConsoleLevelAndOutput(t *testing.T) {
var out bytes.Buffer
c := newConsole(&out)
h := &consoleHandler{c: c}
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
c.level.Set(slog.LevelError)
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
rec.AddAttrs(slog.String("c", "3"))
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
assert.Contains(t, out.String(), "a=1 c=3")
}
-83
View File
@@ -1,83 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"io"
"log/slog"
"os"
"runtime"
"sync/atomic"
)
// TextOptions describes how the process-wide Logger writes a record as text.
type TextOptions struct {
DisableColors bool
DisableTimestamp bool
FullTimestamp bool
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
CallerPrettyfier func(*runtime.Frame) (function, file string)
// Filter rewrites each formatted line, for example to mask secrets.
Filter func(line []byte) []byte
}
var (
std atomic.Pointer[Logger]
stdState = newConsole(os.Stderr)
)
func init() {
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
}
// Default returns the process-wide Logger.
func Default() *Logger { return std.Load() }
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
// initial console Logger.
func SetDefault(l *Logger) { std.Store(l) }
// SetLevel sets the minimum level of the initial process-wide Logger.
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
// GetLevel returns the minimum level of the initial process-wide Logger.
func GetLevel() Level { return Level(stdState.level.Level()) }
// SetOutput sets where the initial process-wide Logger writes.
func SetOutput(w io.Writer) { stdState.setOutput(w) }
// Output returns where the initial process-wide Logger writes.
func Output() io.Writer {
stdState.mu.Lock()
defer stdState.mu.Unlock()
return stdState.out
}
// SetTextFormat sets the text format of the initial process-wide Logger.
func SetTextFormat(o TextOptions) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.opts = o
}
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
func SetReportCaller(on bool) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.reportCaller = on
}
func Enabled(level Level) bool { return Default().Enabled(level) }
func With(args ...any) *Logger { return Default().With(args...) }
func WithError(err error) *Logger { return Default().WithError(err) }
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
func Debug(args ...any) { Default().log(LevelDebug, args) }
func Info(args ...any) { Default().log(LevelInfo, args) }
func Warn(args ...any) { Default().log(LevelWarn, args) }
func Error(args ...any) { Default().log(LevelError, args) }
-114
View File
@@ -1,114 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"errors"
"slices"
"time"
)
// Attr is a structured key/value pair attached to a log record.
type Attr struct {
Key string
Value any
}
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
// Logger.With are accumulated by the Handler.
type Record struct {
Time time.Time
Level Level
Message string
PC uintptr // program counter of the logging call, 0 if unknown
Attrs []Attr
}
// Attr returns the value of the last attribute named key.
func (r Record) Attr(key string) (any, bool) {
for _, a := range slices.Backward(r.Attrs) {
if a.Key == key {
return a.Value, true
}
}
return nil, false
}
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
type Handler interface {
Enabled(ctx context.Context, level Level) bool
Handle(ctx context.Context, r Record) error
// WithAttrs returns a Handler that adds attrs to every record it handles.
WithAttrs(attrs []Attr) Handler
}
type multiHandler []Handler
// NewMultiHandler fans every record out to all handlers that have its level enabled.
func NewMultiHandler(handlers ...Handler) Handler {
return multiHandler(handlers)
}
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
for _, h := range m {
if h.Enabled(ctx, level) {
return true
}
}
return false
}
func (m multiHandler) Handle(ctx context.Context, r Record) error {
var errs []error
for _, h := range m {
if h.Enabled(ctx, r.Level) {
errs = append(errs, h.Handle(ctx, r))
}
}
return errors.Join(errs...)
}
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
out := make(multiHandler, len(m))
for i, h := range m {
out[i] = h.WithAttrs(attrs)
}
return out
}
type discardHandler struct{}
// DiscardHandler drops every record.
var DiscardHandler Handler = discardHandler{}
func (discardHandler) Enabled(context.Context, Level) bool { return false }
func (discardHandler) Handle(context.Context, Record) error { return nil }
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
type levelHandler struct {
h Handler
minLevel Level
}
// NewLevelHandler passes only records at or above minLevel on to h.
func NewLevelHandler(h Handler, minLevel Level) Handler {
return levelHandler{h: h, minLevel: minLevel}
}
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
return level >= l.minLevel && l.h.Enabled(ctx, level)
}
func (l levelHandler) Handle(ctx context.Context, r Record) error {
if r.Level < l.minLevel {
return nil
}
return l.h.Handle(ctx, r)
}
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
}
-53
View File
@@ -1,53 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"fmt"
"strings"
)
// Level is a log severity. The values match log/slog, with Trace added below Debug.
type Level int
const (
LevelTrace Level = -8
LevelDebug Level = -4
LevelInfo Level = 0
LevelWarn Level = 4
LevelError Level = 8
)
func (l Level) String() string {
switch {
case l <= LevelTrace:
return "trace"
case l <= LevelDebug:
return "debug"
case l <= LevelInfo:
return "info"
case l <= LevelWarn:
return "warn"
default:
return "error"
}
}
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
// the Logger has no level above it.
func ParseLevel(s string) (Level, error) {
switch strings.ToLower(s) {
case "trace":
return LevelTrace, nil
case "debug":
return LevelDebug, nil
case "info":
return LevelInfo, nil
case "warn", "warning":
return LevelWarn, nil
case "error", "fatal", "panic":
return LevelError, nil
}
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
}
-105
View File
@@ -1,105 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"fmt"
"runtime"
"time"
)
const badKey = "!BADKEY"
// Logger writes printf-style messages with structured attributes to a Handler.
type Logger struct {
h Handler
ctx context.Context
}
func New(h Handler) *Logger {
return &Logger{h: h, ctx: context.Background()}
}
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
func (l *Logger) With(args ...any) *Logger {
if len(args) == 0 {
return l
}
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
}
func (l *Logger) WithError(err error) *Logger {
return l.With("error", err)
}
// WithContext returns a Logger that passes ctx to its Handler.
func (l *Logger) WithContext(ctx context.Context) *Logger {
return &Logger{h: l.h, ctx: ctx}
}
func (l *Logger) Enabled(level Level) bool {
return l.h.Enabled(l.ctx, level)
}
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
// Trace and the methods below format their arguments like fmt.Sprint.
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
func (l *Logger) logf(level Level, format string, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
}
func (l *Logger) log(level Level, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprint(args...))
}
// emit drops the Handler error, as a logger has nowhere to report it.
func (l *Logger) emit(pc uintptr, level Level, msg string) {
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
}
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
func callerPC(skip int) uintptr {
var pcs [1]uintptr
runtime.Callers(skip, pcs[:])
return pcs[0]
}
func argsToAttrs(args []any) []Attr {
attrs := make([]Attr, 0, len(args)/2+1)
for len(args) > 0 {
switch k := args[0].(type) {
case Attr:
attrs = append(attrs, k)
args = args[1:]
case string:
if len(args) == 1 {
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = nil
} else {
attrs = append(attrs, Attr{Key: k, Value: args[1]})
args = args[2:]
}
default:
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = args[1:]
}
}
return attrs
}
-145
View File
@@ -1,145 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging_test
import (
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseLevel(t *testing.T) {
for in, want := range map[string]logging.Level{
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
} {
got, err := logging.ParseLevel(in)
require.NoError(t, err, in)
assert.Equal(t, want, got, in)
}
_, err := logging.ParseLevel("loud")
require.Error(t, err)
assert.Equal(t, "warn", logging.LevelWarn.String())
}
type countingStringer struct{ n *int }
func (c countingStringer) String() string { *c.n++; return "x" }
func TestLogger(t *testing.T) {
h := logtest.NewWithLevel(logging.LevelTrace)
assert.Nil(t, h.LastRecord())
l := logging.New(h)
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
l.Info("a", 1, 2, "b")
l.With("k").With(7).Debug("x")
recs := h.AllRecords()
require.Len(t, recs, 3)
assert.Equal(t, "step 2", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
assert.False(t, recs[0].Time.IsZero())
job, _ := recs[0].Attr("job")
assert.Equal(t, "j1", job)
errv, _ := recs[0].Attr("error")
require.EqualError(t, errv.(error), "boom")
assert.Equal(t, "a1 2b", recs[1].Message)
bad, _ := recs[2].Attr("!BADKEY")
assert.Equal(t, 7, bad)
}
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
n := 0
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
l.Debugf("%v", countingStringer{&n})
assert.Zero(t, n)
assert.False(t, l.Enabled(logging.LevelInfo))
}
func TestMultiHandler(t *testing.T) {
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
l.Info("i")
l.Error("e")
assert.Len(t, all.AllRecords(), 2)
require.Len(t, warnOnly.AllRecords(), 1)
v, _ := warnOnly.LastRecord().Attr("a")
assert.Equal(t, 1, v)
}
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
t.Helper()
require.NotZero(t, pc)
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
return f
}
func TestCallerPC(t *testing.T) {
l, h := logtest.NewLogger()
l.Infof("method")
l.Info("method")
prev := logging.Default()
t.Cleanup(func() { logging.SetDefault(prev) })
logging.SetDefault(l)
logging.Infof("global")
logging.Error("global")
require.Len(t, h.AllRecords(), 4)
for _, r := range h.AllRecords() {
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
}
}
func TestDefaultLevel(t *testing.T) {
prev := logging.GetLevel()
t.Cleanup(func() { logging.SetLevel(prev) })
logging.SetLevel(logging.LevelWarn)
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
assert.True(t, logging.Enabled(logging.LevelError))
assert.False(t, logging.Enabled(logging.LevelInfo))
}
func TestSlogHandler(t *testing.T) {
var buf strings.Builder
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
assert.False(t, l.Enabled(logging.LevelTrace))
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
out := buf.String()
assert.Contains(t, out, "level=ERROR")
assert.Contains(t, out, `msg="fail x"`)
assert.Contains(t, out, "job=j1")
assert.Contains(t, out, "error=boom")
}
func TestSlogBridge(t *testing.T) {
h := logtest.SetDefault(t)
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
_, _, line, _ := runtime.Caller(0)
slog.Debug("dropped")
recs := h.AllRecords()
require.Len(t, recs, 1)
assert.Equal(t, "from a library", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
lib, _ := recs[0].Attr("lib")
assert.Equal(t, "actionslib", lib)
n, _ := recs[0].Attr("n")
assert.EqualValues(t, 1, n)
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
}
Loaded 50 of 336 files, more files were not shown because too many files have changed in this diff. Show more