mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-15 12:23:42 +00:00
Compare commits
49
Commits
v5.0.0
...
d8d4a4657d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 | ||
|
|
3dbed63f55 | ||
|
|
dafe8d8d0c | ||
|
|
8ae48ae033 | ||
|
|
8dd3776014 | ||
|
|
c6ca020281 | ||
|
|
fe2a66dc25 | ||
|
|
c719a88f36 | ||
|
|
d882311ec6 | ||
|
|
923a0bfb59 | ||
|
|
6f0d4f1b48 | ||
|
|
430f7bb2cf | ||
|
|
befb1b4bb9 | ||
|
|
1a7f09224f | ||
|
|
af2b3f6df5 | ||
|
|
785313cc21 | ||
|
|
41f1e08bec | ||
|
|
4158e05dae | ||
|
|
e356c2ed85 | ||
|
|
ed940db4f0 | ||
|
|
755af87095 | ||
|
|
b9aa47d1a5 | ||
|
|
0d84ec6dab | ||
|
|
b769a9765f | ||
|
|
9c4c66e70f | ||
|
|
42858c61df | ||
|
|
470969dc74 | ||
|
|
be7ddcd15a | ||
|
|
c3c647ef83 | ||
|
|
160e18e705 | ||
|
|
20c8b4abc4 | ||
|
|
ccb9afd7a5 | ||
|
|
49e34813ae | ||
|
|
e0ef637805 | ||
|
|
8502663d0e | ||
|
|
4e8037a93b | ||
|
|
76110557ab | ||
|
|
3bc5cc03f8 | ||
|
|
de961b91eb | ||
|
|
ad9eeb668a | ||
|
|
b741335222 | ||
|
|
20143c9558 |
No files matched your search
@@ -45,8 +45,13 @@ cpu.out
|
|||||||
|
|
||||||
/bin
|
/bin
|
||||||
/dist
|
/dist
|
||||||
|
/.tmp
|
||||||
|
/.git/hooks
|
||||||
|
/.git/logs
|
||||||
|
/.git/rr-cache
|
||||||
/.env
|
/.env
|
||||||
/.runner
|
/.runner
|
||||||
|
/.runner.lock
|
||||||
/config.yaml
|
/config.yaml
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: go-setup
|
||||||
|
description: Set up go and restore its caches, which only the cache-seeder workflow saves
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||||
|
id: setup-go
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: false
|
||||||
|
- if: ${{ github.workflow == 'cache-seeder' }}
|
||||||
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
- if: ${{ github.workflow != 'cache-seeder' }}
|
||||||
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/go/pkg/mod
|
||||||
|
~/.cache/go-build
|
||||||
|
~/.cache/golangci-lint
|
||||||
|
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
|
||||||
|
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
name: cache-seeder
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- go.mod
|
||||||
|
- go.sum
|
||||||
|
- .golangci.yml
|
||||||
|
- Makefile
|
||||||
|
- .gitea/actions/go-setup/action.yml
|
||||||
|
- .gitea/workflows/cache-seeder.yml
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
seed:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- run: make deps-tools lint build
|
||||||
|
- name: compile the test binaries without running them
|
||||||
|
run: |
|
||||||
|
make test GOTEST_FLAGS='-race -exec true'
|
||||||
|
go test -tags e2e -exec true ./e2e/...
|
||||||
@@ -19,7 +19,7 @@ jobs:
|
|||||||
timeout-minutes: 5
|
timeout-minutes: 5
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 24
|
||||||
- run: make lint-pr-title
|
- run: make lint-pr-title
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ on:
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: nightly
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
goreleaser:
|
goreleaser:
|
||||||
@@ -36,7 +35,7 @@ jobs:
|
|||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release --nightly
|
args: release --nightly
|
||||||
@@ -80,7 +79,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
passphrase: ${{ secrets.PASSPHRASE }}
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
distribution: goreleaser-pro
|
distribution: goreleaser-pro
|
||||||
args: release
|
args: release
|
||||||
@@ -69,7 +69,6 @@ jobs:
|
|||||||
image: catthehacker/ubuntu:act-latest
|
image: catthehacker/ubuntu:act-latest
|
||||||
env:
|
env:
|
||||||
DOCKER_ORG: gitea
|
DOCKER_ORG: gitea
|
||||||
DOCKER_LATEST: latest
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
@@ -83,7 +82,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|||||||
+11
-15
@@ -17,25 +17,24 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
|
||||||
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
|
||||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||||
- name: pre-pull test images
|
- name: pre-pull test images
|
||||||
env:
|
env:
|
||||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:0e0ff40c39bc087845bfb27465a0df4ea419520094bc35842ff83dd8cbe6f9b6 # renovate: datasource=docker
|
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 # renovate: datasource=docker
|
||||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||||
run: |
|
run: |
|
||||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||||
for attempt in 1 2 3; do
|
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
|
||||||
docker pull "$image" && break
|
for attempt in 1 2 3; do
|
||||||
[ "$attempt" = 3 ] || sleep 5
|
docker pull "$image" && break
|
||||||
done
|
[ "$attempt" = 3 ] || sleep 5
|
||||||
|
done
|
||||||
|
fi
|
||||||
docker tag "$image" "${image%@*}"
|
docker tag "$image" "${image%@*}"
|
||||||
done
|
done
|
||||||
- name: lint
|
- name: lint
|
||||||
@@ -68,10 +67,7 @@ jobs:
|
|||||||
- gitea/gitea:main-nightly
|
- gitea/gitea:main-nightly
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
- uses: ./.gitea/actions/go-setup
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
check-latest: true
|
|
||||||
- name: prepare anonymous docker config
|
- name: prepare anonymous docker config
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
- name: e2e compatibility
|
- name: e2e compatibility
|
||||||
|
|||||||
@@ -39,6 +39,13 @@ linters:
|
|||||||
deny:
|
deny:
|
||||||
- pkg: golang.org/x/exp
|
- pkg: golang.org/x/exp
|
||||||
desc: it's experimental and unreliable
|
desc: it's experimental and unreliable
|
||||||
|
logging:
|
||||||
|
files:
|
||||||
|
- $all
|
||||||
|
- "!**/act/common/logging/**"
|
||||||
|
deny:
|
||||||
|
- pkg: log/slog
|
||||||
|
desc: log through gitea.com/gitea/runner/act/common/logging
|
||||||
nolintlint:
|
nolintlint:
|
||||||
allow-unused: false
|
allow-unused: false
|
||||||
require-explanation: true
|
require-explanation: true
|
||||||
|
|||||||
@@ -28,38 +28,18 @@ builds:
|
|||||||
ignore:
|
ignore:
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: arm
|
goarch: arm
|
||||||
- goos: darwin
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: darwin
|
- goos: darwin
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: windows
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: windows
|
- goos: windows
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
- goos: freebsd
|
|
||||||
goarch: ppc64le
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: s390x
|
goarch: s390x
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm
|
goarch: arm
|
||||||
goarm: "5"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "6"
|
|
||||||
- goos: freebsd
|
|
||||||
goarch: arm
|
|
||||||
goarm: "7"
|
|
||||||
- goos: freebsd
|
- goos: freebsd
|
||||||
goarch: arm64
|
goarch: arm64
|
||||||
flags:
|
flags:
|
||||||
|
|||||||
+24
-2
@@ -28,8 +28,25 @@ depending on the prefix:
|
|||||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
||||||
encoded forms too.
|
encoded forms too.
|
||||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
|
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||||
decodes exactly those two when folding a location into an annotation.
|
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||||
|
|
||||||
|
## Logging
|
||||||
|
|
||||||
|
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
|
||||||
|
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
|
||||||
|
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
|
||||||
|
fields.
|
||||||
|
|
||||||
|
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
|
||||||
|
Going around either leaks them.
|
||||||
|
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
|
||||||
|
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
|
||||||
|
job logger.
|
||||||
|
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
|
||||||
|
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
|
||||||
|
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
|
||||||
|
package functions.
|
||||||
|
|
||||||
## Built-in actions
|
## Built-in actions
|
||||||
|
|
||||||
@@ -42,5 +59,10 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
|||||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||||
parallel.
|
parallel.
|
||||||
|
|
||||||
|
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||||
|
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped, tolerated and timed-out jobs.
|
||||||
|
`label_routing` and `capacity` cover scheduling, `reconnect` drops the connection to Gitea through a proxy, and
|
||||||
|
`host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||||
|
|
||||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
||||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||||
+6
-4
@@ -1,7 +1,7 @@
|
|||||||
### BUILDER STAGE
|
### BUILDER STAGE
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM golang:1.27-alpine3.23 AS builder
|
FROM golang:1.27-alpine AS builder
|
||||||
|
|
||||||
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
||||||
RUN apk add --no-cache make git
|
RUN apk add --no-cache make git
|
||||||
@@ -12,12 +12,14 @@ ENV GOPROXY=${GOPROXY:-}
|
|||||||
COPY . /opt/src/runner
|
COPY . /opt/src/runner
|
||||||
WORKDIR /opt/src/runner
|
WORKDIR /opt/src/runner
|
||||||
|
|
||||||
RUN make clean && make build
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
make clean && make build
|
||||||
|
|
||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind AS dind
|
FROM docker:29.9.1-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -39,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.7.2-dind-rootless AS dind-rootless
|
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
|
|||||||
@@ -1,31 +1,22 @@
|
|||||||
DIST := dist
|
DIST := dist
|
||||||
EXECUTABLE := gitea-runner
|
EXECUTABLE := gitea-runner
|
||||||
DIST_DIRS := $(DIST)/binaries $(DIST)/release
|
|
||||||
GO ?= go
|
GO ?= go
|
||||||
SHASUM ?= shasum -a 256
|
|
||||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||||
XGO_PACKAGE ?= src.techknowlogick.com/[email protected] # renovate: datasource=go
|
|
||||||
XGO_VERSION := go-1.27.x
|
|
||||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected] # renovate: datasource=go
|
|
||||||
|
|
||||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.4-immutable@sha256:90405e3c6a86f33c5a6f8dbdfa8e02a98143a8ba4337b84bdaec5336d2ef008a # renovate: datasource=docker
|
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:8856b200466e9a5af9cc0d4de4ca96f73547833bcdddf9a567189ff772c4d372 # renovate: datasource=docker
|
||||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.36.4-k3s1@sha256:edad48e12bf81c3a09ac1c05c0c0ffaaa22145980b989d6fae84543a76b83657 # renovate: datasource=docker
|
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||||
|
|
||||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
|
||||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
|
||||||
WINDOWS_ARCHS ?= windows/amd64
|
|
||||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||||
|
|
||||||
DOCKER_IMAGE ?= gitea/runner
|
DOCKER_IMAGE ?= gitea/runner
|
||||||
DOCKER_TAG ?= nightly
|
DOCKER_TAG ?= nightly
|
||||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
|
||||||
|
|
||||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]4.0 # renovate: datasource=go
|
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]5.0 # renovate: datasource=go
|
||||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
|
||||||
|
|
||||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||||
|
|
||||||
@@ -38,22 +29,16 @@ endif
|
|||||||
ifeq ($(HAS_GO), GO)
|
ifeq ($(HAS_GO), GO)
|
||||||
GOPATH ?= $(shell $(GO) env GOPATH)
|
GOPATH ?= $(shell $(GO) env GOPATH)
|
||||||
export PATH := $(GOPATH)/bin:$(PATH)
|
export PATH := $(GOPATH)/bin:$(PATH)
|
||||||
|
|
||||||
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
|
|
||||||
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(OS), Windows_NT)
|
ifeq ($(OS), Windows_NT)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else ifeq ($(OS), Windows)
|
else ifeq ($(OS), Windows)
|
||||||
GOFLAGS := -v -buildmode=exe
|
GOFLAGS := -v -buildmode=exe
|
||||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
|
||||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||||
else
|
else
|
||||||
GOFLAGS := -v
|
GOFLAGS := -v
|
||||||
EXECUTABLE ?= $(EXECUTABLE)
|
|
||||||
GO_ENV_WINDOWS := GOOS=windows
|
GO_ENV_WINDOWS := GOOS=windows
|
||||||
endif
|
endif
|
||||||
|
|
||||||
@@ -108,8 +93,6 @@ fmt-check: ## check the Go code formatting
|
|||||||
.PHONY: deps-tools
|
.PHONY: deps-tools
|
||||||
deps-tools: ## install tool dependencies
|
deps-tools: ## install tool dependencies
|
||||||
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
||||||
$(GO) install $(GXZ_PACKAGE) & \
|
|
||||||
$(GO) install $(XGO_PACKAGE) & \
|
|
||||||
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
||||||
wait
|
wait
|
||||||
|
|
||||||
@@ -187,45 +170,6 @@ $(EXECUTABLE): $(GOFILES)
|
|||||||
deps-backend: ## install backend dependencies
|
deps-backend: ## install backend dependencies
|
||||||
$(GO) mod download
|
$(GO) mod download
|
||||||
|
|
||||||
.PHONY: release
|
|
||||||
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
|
|
||||||
|
|
||||||
$(DIST_DIRS):
|
|
||||||
mkdir -p $(DIST_DIRS)
|
|
||||||
|
|
||||||
.PHONY: release-windows
|
|
||||||
release-windows: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-linux
|
|
||||||
release-linux: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-darwin
|
|
||||||
release-darwin: | $(DIST_DIRS)
|
|
||||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
|
||||||
ifeq ($(CI),true)
|
|
||||||
cp -r /build/* $(DIST)/binaries/
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: release-copy
|
|
||||||
release-copy: | $(DIST_DIRS)
|
|
||||||
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
|
|
||||||
|
|
||||||
.PHONY: release-check
|
|
||||||
release-check: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
|
|
||||||
|
|
||||||
.PHONY: release-compress
|
|
||||||
release-compress: | $(DIST_DIRS)
|
|
||||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
|
|
||||||
|
|
||||||
.PHONY: docker
|
.PHONY: docker
|
||||||
docker: ## build the docker image
|
docker: ## build the docker image
|
||||||
if ! docker buildx version >/dev/null 2>&1; then \
|
if ! docker buildx version >/dev/null 2>&1; then \
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
On the runner:
|
On the runner:
|
||||||
|
|
||||||
- `git` 2.34.1 or newer on `PATH`, to download actions and reusable workflows.
|
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
|
||||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||||
|
|
||||||
Where steps run, in the job image or on the runner in host mode:
|
Where steps run, in the job image or on the runner in host mode:
|
||||||
@@ -242,7 +242,7 @@ A job's `services` are started before its steps run. When a service's image or i
|
|||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:17
|
image: postgres:18
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd pg_isready
|
--health-cmd pg_isready
|
||||||
--health-interval 5s
|
--health-interval 5s
|
||||||
@@ -368,7 +368,7 @@ While the runner is idle it cleans up after earlier jobs:
|
|||||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||||
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||||
|
|
||||||
|
|||||||
+333
-188
File diff suppressed because it is too large.
Load diff
+298
-684
File diff suppressed because it is too large.
Load diff
@@ -15,8 +15,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The cache service v2 API. A client on this version talks twirp to
|
// The cache service v2 API. A client on this version talks twirp to
|
||||||
@@ -44,16 +42,16 @@ const (
|
|||||||
twirpUnauthenticated = "unauthenticated"
|
twirpUnauthenticated = "unauthenticated"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *Handler) registerV2Routes(router *httprouter.Router) {
|
func (h *Handler) registerV2Routes(router *http.ServeMux) {
|
||||||
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
||||||
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
||||||
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
||||||
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
||||||
}
|
}
|
||||||
|
|
||||||
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
||||||
// the upload.
|
// the upload.
|
||||||
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -102,15 +100,16 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
cache := &Cache{
|
cache := &Cache{
|
||||||
Repo: cred.Repo,
|
Repo: cred.Repo,
|
||||||
Key: req.Key,
|
Key: req.Key,
|
||||||
Version: req.Version,
|
Version: req.Version,
|
||||||
Owner: owner,
|
Owner: owner,
|
||||||
Size: -1, // the size is only known at finalize time
|
Size: -1, // the size is only known at finalize time
|
||||||
CreatedAt: now,
|
CreatedAt: now.Unix(),
|
||||||
UsedAt: now,
|
CreatedAtNanos: now.Nanosecond(),
|
||||||
|
UsedAt: now.Unix(),
|
||||||
}
|
}
|
||||||
if err := h.insertCache(db, cache); err != nil {
|
if err := h.insertCache(db, cache); err != nil {
|
||||||
h.twirpError(w, r, twirpInternal, err)
|
h.twirpError(w, r, twirpInternal, err)
|
||||||
@@ -123,7 +122,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -172,7 +171,7 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
|
||||||
cred := credFromContext(r.Context())
|
cred := credFromContext(r.Context())
|
||||||
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -207,8 +206,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
|||||||
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
||||||
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
||||||
// in order.
|
// in order.
|
||||||
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||||
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
|
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// v2Call posts a twirp request to the cache service and returns the decoded response.
|
|
||||||
// Field names are the proto ones, which is what the toolkit's client sends.
|
|
||||||
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -37,7 +35,6 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
|
|||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
|
|
||||||
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
|
|
||||||
func putBlob(t *testing.T, url string, content []byte) int {
|
func putBlob(t *testing.T, url string, content []byte) int {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -47,8 +44,6 @@ func putBlob(t *testing.T, url string, content []byte) int {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode == http.StatusCreated {
|
if resp.StatusCode == http.StatusCreated {
|
||||||
// The Azure SDK client dereferences this header without checking, so a blob upload that
|
|
||||||
// omits it panics the caller rather than failing it.
|
|
||||||
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
||||||
}
|
}
|
||||||
return resp.StatusCode
|
return resp.StatusCode
|
||||||
@@ -67,8 +62,6 @@ func getURL(t *testing.T, url string) []byte {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
|
|
||||||
// with the upload URL it used.
|
|
||||||
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -84,10 +77,8 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
|||||||
}), uploadURL
|
}), uploadURL
|
||||||
}
|
}
|
||||||
|
|
||||||
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
|
|
||||||
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
|
|
||||||
// or to replace a finalized entry.
|
|
||||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -98,10 +89,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
assert.NotEmpty(t, finalized["entry_id"])
|
assert.NotEmpty(t, finalized["entry_id"])
|
||||||
|
|
||||||
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
|
|
||||||
// and it is an upload URL only: nothing reads a blob back through it.
|
|
||||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
||||||
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
|
resp, err := http.Get(uploadURL)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
||||||
@@ -114,9 +103,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
assert.Equal(t, content, getURL(t, downloadURL))
|
assert.Equal(t, content, getURL(t, downloadURL))
|
||||||
}
|
}
|
||||||
|
|
||||||
// A large archive is staged as blocks and only put in order by the final block list, so
|
|
||||||
// blocks that arrive out of order must still be assembled the way the client asked.
|
|
||||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
|
||||||
@@ -130,7 +118,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
blocks[blockID] = []byte(part)
|
blocks[blockID] = []byte(part)
|
||||||
order = append(order, blockID)
|
order = append(order, blockID)
|
||||||
}
|
}
|
||||||
// Upload in an order that is not the block list order.
|
|
||||||
for _, blockID := range []string{order[2], order[0], order[1]} {
|
for _, blockID := range []string{order[2], order[0], order[1]} {
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
||||||
}
|
}
|
||||||
@@ -165,6 +152,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2Lookups(t *testing.T) {
|
func TestCacheServiceV2Lookups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
|
||||||
require.Equal(t, true, saved["ok"])
|
require.Equal(t, true, saved["ok"])
|
||||||
@@ -174,8 +162,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The toolkit serialises with the proto field names; the camelCase spellings of the same
|
|
||||||
// proto JSON mapping are accepted alongside them.
|
|
||||||
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
||||||
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
||||||
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
||||||
@@ -191,9 +177,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, again["ok"])
|
assert.Equal(t, false, again["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// A key that is only a prefix of an existing one is a different entry, so the
|
|
||||||
// reservation check must be exact and not a restore-key prefix match, or the shorter
|
|
||||||
// key would be reported as existing and silently never saved.
|
|
||||||
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
||||||
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
||||||
require.Equal(t, true, reserved["ok"])
|
require.Equal(t, true, reserved["ok"])
|
||||||
@@ -219,7 +202,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The size the client declares is what Commit validates the assembled archive against.
|
|
||||||
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
||||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
||||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
||||||
@@ -230,8 +212,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.Equal(t, false, got["ok"])
|
assert.Equal(t, false, got["ok"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// Both API versions are served from one store, so an entry written through v2 is a hit for
|
|
||||||
// a v1 client asking for the same key and version.
|
|
||||||
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
||||||
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -244,7 +224,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
assert.NotEmpty(t, got["archiveLocation"])
|
assert.NotEmpty(t, got["archiveLocation"])
|
||||||
})
|
})
|
||||||
|
|
||||||
// The cache of one repository must stay invisible to another, as it does for the v1 API.
|
|
||||||
t.Run("another repository sees nothing", func(t *testing.T) {
|
t.Run("another repository sees nothing", func(t *testing.T) {
|
||||||
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
||||||
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
||||||
@@ -255,6 +234,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -291,6 +271,7 @@ func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
handler := newTestHandler(t, Policy{})
|
handler := newTestHandler(t, Policy{})
|
||||||
content := []byte("the cached archive")
|
content := []byte("the cached archive")
|
||||||
|
|
||||||
@@ -321,20 +302,12 @@ func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
|||||||
"another job's upload is not this job's to finalize")
|
"another job's upload is not this job's to finalize")
|
||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
reserveV1, err := json.Marshal(&Request{Key: "v1-made", Version: "v1", Size: int64(len(content))})
|
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
|
||||||
require.NoError(t, err)
|
|
||||||
resp, err := testClient.Post(base+"/caches", "application/json", bytes.NewReader(reserveV1))
|
|
||||||
require.NoError(t, err)
|
|
||||||
var madeByV1 struct {
|
|
||||||
CacheID uint64 `json:"cacheId"`
|
|
||||||
}
|
|
||||||
require.NoError(t, json.UnmarshalRead(resp.Body, &madeByV1))
|
|
||||||
resp.Body.Close()
|
|
||||||
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
||||||
fmt.Sprintf("%s/caches/%d", base, madeByV1.CacheID), bytes.NewReader(content))
|
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
||||||
resp, err = testClient.Do(upload)
|
resp, err := testClient.Do(upload)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
type Request struct {
|
type Request struct {
|
||||||
Key string `json:"key" `
|
Key string `json:"key" `
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
@@ -11,13 +13,18 @@ type Request struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Cache struct {
|
type Cache struct {
|
||||||
ID uint64 `json:"id" boltholdKey:"ID"`
|
ID uint64 `json:"id"`
|
||||||
Repo string `json:"repo" boltholdIndex:"Repo"`
|
Repo string `json:"repo"`
|
||||||
Key string `json:"key" boltholdIndex:"Key"`
|
Key string `json:"key"`
|
||||||
Version string `json:"version" boltholdIndex:"Version"`
|
Version string `json:"version"`
|
||||||
Size int64 `json:"cacheSize"`
|
Size int64 `json:"cacheSize"`
|
||||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||||
Complete bool `json:"complete" boltholdIndex:"Complete"`
|
Complete bool `json:"complete"`
|
||||||
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
|
UsedAt int64 `json:"usedAt"`
|
||||||
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cache *Cache) createdAtNano() int64 {
|
||||||
|
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"cmp"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -74,14 +75,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
||||||
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
||||||
if cred.InsecureTLS && target.Scheme == "https" {
|
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
|
||||||
return true
|
|
||||||
}
|
|
||||||
scheme := r.Header.Get("X-Forwarded-Proto")
|
|
||||||
if scheme == "" {
|
|
||||||
scheme = "http"
|
|
||||||
}
|
|
||||||
return scheme != target.Scheme
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeQuery(target, request string) string {
|
func mergeQuery(target, request string) string {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFrontResultsService(t *testing.T) {
|
func TestFrontResultsService(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
var gotHost, gotPath, gotProto string
|
var gotHost, gotPath, gotProto string
|
||||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
// S3Options configures the bucket that stores completed cache entries.
|
// S3Options configures the bucket that stores completed cache entries.
|
||||||
@@ -47,7 +51,7 @@ type cacheStorage interface {
|
|||||||
Remove(uint64) error
|
Remove(uint64) error
|
||||||
}
|
}
|
||||||
|
|
||||||
const s3PartSize = 5 << 30 // the largest part S3 accepts
|
const s3PartSize = 64 << 20
|
||||||
|
|
||||||
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
||||||
type s3Storage struct {
|
type s3Storage struct {
|
||||||
@@ -69,6 +73,7 @@ func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
|
|||||||
}
|
}
|
||||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||||
transport = transport.Clone()
|
transport = transport.Clone()
|
||||||
|
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
|
||||||
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
||||||
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
||||||
if opts.PathStyle != nil {
|
if opts.PathStyle != nil {
|
||||||
@@ -209,26 +214,43 @@ func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// upload always goes in parts, the only way S3 takes an object over 5 GiB.
|
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
|
||||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) error {
|
if size <= s3PartSize {
|
||||||
|
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
var created struct {
|
var created struct {
|
||||||
UploadID string `xml:"UploadId"`
|
UploadID string `xml:"UploadId"`
|
||||||
}
|
}
|
||||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
if uploadErr != nil && created.UploadID != "" {
|
||||||
|
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
|
||||||
|
}
|
||||||
|
}()
|
||||||
var completion struct {
|
var completion struct {
|
||||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||||
Parts []s3Part `xml:"Part"`
|
Parts []s3Part `xml:"Part"`
|
||||||
}
|
}
|
||||||
for offset := int64(0); offset == 0 || offset < size; offset += s3PartSize {
|
partSize := max(int64(s3PartSize), (size-1)/10000+1)
|
||||||
number, length := len(completion.Parts)+1, min(s3PartSize, size-offset)
|
completion.Parts = make([]s3Part, (size-1)/partSize+1)
|
||||||
query := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {created.UploadID}}
|
workers, workerCtx := errgroup.WithContext(ctx)
|
||||||
header, err := s.call(ctx, http.MethodPut, key, query, io.NewSectionReader(file, offset, length), nil)
|
workers.SetLimit(4)
|
||||||
if err != nil {
|
for part := range completion.Parts {
|
||||||
|
workers.Go(func() (err error) {
|
||||||
|
defer common.RecoverPanic(&err)
|
||||||
|
offset := int64(part) * partSize
|
||||||
|
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
|
||||||
|
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
|
||||||
return err
|
return err
|
||||||
}
|
})
|
||||||
completion.Parts = append(completion.Parts, s3Part{number, header.Get("ETag")})
|
}
|
||||||
|
if err := workers.Wait(); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
body, err := xml.Marshal(completion)
|
body, err := xml.Marshal(completion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,40 +4,135 @@
|
|||||||
package artifactcache
|
package artifactcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
|
||||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||||
backend := s3mem.New()
|
t.Parallel()
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
var objects sync.Map
|
||||||
server := httptest.NewServer(gofakes3.New(backend).Server())
|
var uploads atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
key := strings.TrimPrefix(request.URL.Path, "/cache/")
|
||||||
|
switch {
|
||||||
|
case request.Method == http.MethodPut:
|
||||||
|
if strings.Contains(key, "/blobs/") {
|
||||||
|
uploads.Add(1)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(request.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
objects.Store(key, body)
|
||||||
|
case request.URL.Query().Has("list-type"):
|
||||||
|
_, _ = io.WriteString(writer, "<ListBucketResult>")
|
||||||
|
objects.Range(func(stored, _ any) bool {
|
||||||
|
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
_, _ = io.WriteString(writer, "</ListBucketResult>")
|
||||||
|
default:
|
||||||
|
if body, ok := objects.Load(key); ok {
|
||||||
|
_, _ = writer.Write(body.([]byte))
|
||||||
|
} else {
|
||||||
|
writer.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
||||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
|
assert.EqualValues(t, 1, uploads.Load())
|
||||||
|
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
cache, err := saver.s3.fetch(t.Context(), id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||||
require.Equal(t, true, found["ok"])
|
require.Equal(t, true, found["ok"])
|
||||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
require.NoError(t, backend.ForceDeleteBucket("cache"))
|
for _, cache := range []*Cache{
|
||||||
require.NoError(t, backend.CreateBucket("cache"))
|
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
|
||||||
|
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
|
||||||
|
} {
|
||||||
|
require.NoError(t, saver.s3.publish(cache))
|
||||||
|
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
|
||||||
|
}
|
||||||
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
|
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
|
||||||
|
require.Equal(t, true, found["ok"])
|
||||||
|
assert.Equal(t, "deps-new", found["matched_key"])
|
||||||
|
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
|
||||||
|
|
||||||
|
objects.Clear()
|
||||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type s3TestTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
return transport(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, failingPart := range []string{"", "2"} {
|
||||||
|
var parts, aborts atomic.Int32
|
||||||
|
var completion struct {
|
||||||
|
Parts []s3Part `xml:"Part"`
|
||||||
|
}
|
||||||
|
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
|
||||||
|
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
|
||||||
|
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
|
||||||
|
switch {
|
||||||
|
case query.Has("uploads"):
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
|
||||||
|
case req.Method == http.MethodPut:
|
||||||
|
parts.Add(1)
|
||||||
|
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
|
||||||
|
if query.Get("partNumber") == failingPart {
|
||||||
|
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
|
||||||
|
}
|
||||||
|
case req.Method == http.MethodPost:
|
||||||
|
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
|
||||||
|
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
|
||||||
|
case req.Method == http.MethodDelete:
|
||||||
|
aborts.Add(1)
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
|
||||||
|
assert.EqualValues(t, 3, parts.Load())
|
||||||
|
if failingPart == "" {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
|
||||||
|
assert.Zero(t, aborts.Load())
|
||||||
|
} else {
|
||||||
|
require.ErrorContains(t, err, "InvalidPart")
|
||||||
|
assert.EqualValues(t, 1, aborts.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|||||||
@@ -32,13 +32,11 @@ func NewStorage(rootDir string) (*Storage, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Exist(id uint64) (bool, error) {
|
func (s *Storage) Exist(id uint64) (bool, error) {
|
||||||
name := s.filename(id)
|
_, err := os.Stat(s.filename(id))
|
||||||
if _, err := os.Stat(name); os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
} else if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
}
|
||||||
return true, nil
|
return err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
||||||
@@ -92,9 +90,7 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||||
defer func() {
|
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
|
||||||
_ = os.RemoveAll(s.tempDir(id))
|
|
||||||
}()
|
|
||||||
|
|
||||||
name := s.filename(id)
|
name := s.filename(id)
|
||||||
tempNames, err := s.tempNames(id)
|
tempNames, err := s.tempNames(id)
|
||||||
@@ -153,8 +149,7 @@ func assemble(name string, tempNames []string) (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||||
name := s.filename(id)
|
http.ServeFile(w, r, s.filename(id))
|
||||||
http.ServeFile(w, r, name)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
||||||
|
|||||||
+39
-38
@@ -13,14 +13,13 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileContainerResourceURL struct {
|
type FileContainerResourceURL struct {
|
||||||
@@ -67,39 +66,44 @@ func writeJSON(w http.ResponseWriter, value any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploads(router *httprouter.Router, baseDir string) {
|
func uploads(router *http.ServeMux, baseDir string) {
|
||||||
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
writeJSON(w, FileContainerResourceURL{
|
writeJSON(w, FileContainerResourceURL{
|
||||||
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
|
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
runID := params.ByName("runId")
|
|
||||||
|
|
||||||
if req.Header.Get("Content-Encoding") == "gzip" {
|
if req.Header.Get("Content-Encoding") == "gzip" {
|
||||||
itemPath += gzipExtension
|
itemPath += gzipExtension
|
||||||
}
|
}
|
||||||
|
|
||||||
safeRunPath := safeResolve(baseDir, runID)
|
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
|
||||||
safePath := safeResolve(safeRunPath, itemPath)
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
|
var start int64
|
||||||
appendUpload := req.Header.Get("Content-Range")
|
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
|
||||||
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
|
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
|
||||||
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
|
http.Error(w, "invalid content range", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if start == 0 {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
}
|
}
|
||||||
file, err := os.OpenFile(safePath, flags, 0o644)
|
file, err := os.OpenFile(safePath, flags, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
if _, err := file.Seek(start, io.SeekStart); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
if req.Body == nil {
|
if req.Body == nil {
|
||||||
panic(errors.New("no body given"))
|
panic(errors.New("no body given"))
|
||||||
}
|
}
|
||||||
@@ -114,21 +118,19 @@ func uploads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
|
||||||
writeJSON(w, ResponseMessage{
|
writeJSON(w, ResponseMessage{
|
||||||
Message: "success",
|
Message: "success",
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloads(router *httprouter.Router, baseDir string) {
|
func downloads(router *http.ServeMux, baseDir string) {
|
||||||
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||||
runID := params.ByName("runId")
|
runID := req.PathValue("runId")
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, runID)
|
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
entries, err := os.ReadDir(safePath)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,13 +148,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
container := params.ByName("container")
|
container := req.PathValue("container")
|
||||||
itemPath := req.URL.Query().Get("itemPath")
|
itemPath := req.URL.Query().Get("itemPath")
|
||||||
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
||||||
|
|
||||||
var files []ContainerItem
|
var files []ContainerItem
|
||||||
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !entry.IsDir() {
|
if !entry.IsDir() {
|
||||||
rel, err := filepath.Rel(safePath, path)
|
rel, err := filepath.Rel(safePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -161,20 +166,16 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
|
|
||||||
// if it was upload as gzip
|
// if it was upload as gzip
|
||||||
rel = strings.TrimSuffix(rel, gzipExtension)
|
rel = strings.TrimSuffix(rel, gzipExtension)
|
||||||
path := filepath.Join(itemPath, rel)
|
|
||||||
|
|
||||||
rel = filepath.ToSlash(rel)
|
|
||||||
path = filepath.ToSlash(path)
|
|
||||||
|
|
||||||
files = append(files, ContainerItem{
|
files = append(files, ContainerItem{
|
||||||
Path: path,
|
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||||
ItemType: "file",
|
ItemType: "file",
|
||||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
|
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,10 +184,8 @@ func downloads(router *httprouter.Router, baseDir string) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
|
||||||
path := params.ByName("path")[1:]
|
safePath := safeResolve(baseDir, req.PathValue("path"))
|
||||||
|
|
||||||
safePath := safeResolve(baseDir, path)
|
|
||||||
|
|
||||||
file, err := os.Open(safePath)
|
file, err := os.Open(safePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -214,11 +213,12 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
return cancel
|
return cancel
|
||||||
}
|
}
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
|
|
||||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: net.JoinHostPort(addr, port),
|
Addr: net.JoinHostPort(addr, port),
|
||||||
@@ -230,7 +230,8 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
|||||||
go func() {
|
go func() {
|
||||||
logger.Infof("Start server on http://%s", server.Addr)
|
logger.Infof("Start server on http://%s", server.Addr)
|
||||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
logger.Fatal(err)
|
logger.Error(err)
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -19,16 +19,16 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/julienschmidt/httprouter"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestArtifactFlow(t *testing.T) {
|
func TestArtifactFlow(t *testing.T) {
|
||||||
artifactPath := t.TempDir()
|
artifactPath := t.TempDir()
|
||||||
|
|
||||||
router := httprouter.New()
|
router := http.NewServeMux()
|
||||||
uploads(router, artifactPath)
|
uploads(router, artifactPath)
|
||||||
downloads(router, artifactPath)
|
downloads(router, artifactPath)
|
||||||
|
registerV4(router, artifactPath)
|
||||||
server := httptest.NewServer(router)
|
server := httptest.NewServer(router)
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -49,8 +49,17 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
return resp.StatusCode, data
|
return resp.StatusCode, data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("no-artifacts", func(t *testing.T) {
|
||||||
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
var list NamedFileContainerResourceURLResponse
|
||||||
|
require.NoError(t, json.Unmarshal(data, &list))
|
||||||
|
require.Zero(t, list.Count)
|
||||||
|
require.Empty(t, list.Value)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("upload-and-download", func(t *testing.T) {
|
t.Run("upload-and-download", func(t *testing.T) {
|
||||||
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
||||||
|
|
||||||
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
@@ -82,13 +91,13 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(data, &items))
|
require.NoError(t, json.Unmarshal(data, &items))
|
||||||
require.Len(t, items.Value, 1)
|
require.Len(t, items.Value, 1)
|
||||||
require.Equal(t, "file", items.Value[0].ItemType)
|
require.Equal(t, "file", items.Value[0].ItemType)
|
||||||
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
|
require.Equal(t, item, items.Value[0].Path)
|
||||||
|
|
||||||
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(stored))
|
require.Equal(t, content, string(stored))
|
||||||
})
|
})
|
||||||
@@ -96,16 +105,25 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
t.Run("content-range", func(t *testing.T) {
|
t.Run("content-range", func(t *testing.T) {
|
||||||
const rawURL = "/upload/4?itemPath=chunks.txt"
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
||||||
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
||||||
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
|
||||||
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
for range 2 {
|
||||||
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
||||||
require.Equal(t, http.StatusOK, status, string(data))
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "first-second", string(stored))
|
require.Equal(t, "first-second", string(stored))
|
||||||
|
|
||||||
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
||||||
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
||||||
|
require.Equal(t, http.StatusOK, status, string(data))
|
||||||
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, stored)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("gzip-roundtrip", func(t *testing.T) {
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
||||||
@@ -157,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, status)
|
require.Equal(t, http.StatusOK, status)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("v4", func(t *testing.T) {
|
||||||
|
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||||
|
call := func(method, body string) (int, map[string]any) {
|
||||||
|
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||||
|
var out map[string]any
|
||||||
|
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||||
|
return status, out
|
||||||
|
}
|
||||||
|
put := func(rawURL, body string) int {
|
||||||
|
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
download := func() (string, http.Header) {
|
||||||
|
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, signed)
|
||||||
|
resp, err := client.Get(signed["signedUrl"].(string))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(body), resp.Header
|
||||||
|
}
|
||||||
|
|
||||||
|
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
upload := created["signedUploadUrl"].(string)
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||||
|
for attempt := range 2 {
|
||||||
|
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusBadRequest, status)
|
||||||
|
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status, finalized)
|
||||||
|
id := finalized["artifactId"].(string)
|
||||||
|
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
require.Equal(t, id, refinalized["artifactId"])
|
||||||
|
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusConflict, status)
|
||||||
|
require.Equal(t, "already_exists", conflict["code"])
|
||||||
|
|
||||||
|
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
artifacts := listed["artifacts"].([]any)
|
||||||
|
require.Len(t, artifacts, 1)
|
||||||
|
artifact := artifacts[0].(map[string]any)
|
||||||
|
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||||
|
delete(artifact, "createdAt")
|
||||||
|
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||||
|
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||||
|
require.Empty(t, other["artifacts"])
|
||||||
|
|
||||||
|
body, header := download()
|
||||||
|
require.Equal(t, "hello world", body)
|
||||||
|
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusNotFound, status)
|
||||||
|
require.Equal(t, "not_found", missing["code"])
|
||||||
|
|
||||||
|
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||||
|
require.Equal(t, http.StatusOK, status, created)
|
||||||
|
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||||
|
require.Equal(t, http.StatusCreated, status)
|
||||||
|
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||||
|
require.Equal(t, http.StatusOK, status)
|
||||||
|
body, header = download()
|
||||||
|
require.Equal(t, "gz", body)
|
||||||
|
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||||
|
|
||||||
|
rerun := http.NewServeMux()
|
||||||
|
registerV4(rerun, artifactPath)
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||||
|
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSafeResolve(t *testing.T) {
|
func TestSafeResolve(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||||
|
dir string
|
||||||
|
mu sync.Mutex
|
||||||
|
lastID int64
|
||||||
|
created map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
zipMimeType = "application/zip"
|
||||||
|
maxTwirpBytes = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerV4(router *http.ServeMux, baseDir string) {
|
||||||
|
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||||
|
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||||
|
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||||
|
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
type twirpError struct {
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *twirpError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
func notFound(name string) error {
|
||||||
|
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(format string, args ...any) error {
|
||||||
|
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req Req
|
||||||
|
var resp Resp
|
||||||
|
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||||
|
if err != nil {
|
||||||
|
err = badRequest("malformed request: %v", err)
|
||||||
|
} else {
|
||||||
|
resp, err = handle(r, req)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err != nil {
|
||||||
|
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||||
|
errors.As(err, &terr)
|
||||||
|
w.WriteHeader(terr.status)
|
||||||
|
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.MarshalWrite(w, resp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||||
|
sum := sha256.Sum256([]byte(name))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) runDir(runID string) string {
|
||||||
|
return filepath.Join(s.dir, fileName(runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) artifactDir(runID, name string) string {
|
||||||
|
return filepath.Join(s.runDir(runID), fileName(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) urlDir(r *http.Request) string {
|
||||||
|
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMeta(dir string) (*monolithArtifact, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var meta monolithArtifact
|
||||||
|
return &meta, json.Unmarshal(data, &meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||||
|
data, err := json.Marshal(meta)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||||
|
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.created[dir] {
|
||||||
|
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.lastID++
|
||||||
|
if err := writeMeta(dir, &monolithArtifact{
|
||||||
|
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||||
|
DatabaseID: s.lastID,
|
||||||
|
Name: req.Name,
|
||||||
|
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||||
|
}); err != nil {
|
||||||
|
return createArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
s.created[dir] = true
|
||||||
|
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockList struct {
|
||||||
|
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||||
|
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||||
|
var err error
|
||||||
|
switch comp := r.URL.Query().Get("comp"); comp {
|
||||||
|
case "block":
|
||||||
|
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||||
|
case "blocklist":
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||||
|
case "": // Put Blob, which docker/build-push-action uses
|
||||||
|
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||||
|
_, err := io.Copy(out, r.Body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(name string, body io.Reader) error {
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(file, body)
|
||||||
|
return errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||||
|
out, err := os.CreateTemp(dir, "blob")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||||
|
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, os.Remove(out.Name()))
|
||||||
|
}
|
||||||
|
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||||
|
var list blockList
|
||||||
|
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range list.Blocks {
|
||||||
|
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendBlock(out io.Writer, name string) error {
|
||||||
|
in, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
} else if meta.Digest != "" {
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
size, err := io.Copy(hash, blob)
|
||||||
|
if err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||||
|
if size != req.Size {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||||
|
}
|
||||||
|
if req.Hash != "" && req.Hash != digest {
|
||||||
|
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||||
|
}
|
||||||
|
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||||
|
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
meta.Size, meta.Digest = size, digest
|
||||||
|
if err := writeMeta(dir, meta); err != nil {
|
||||||
|
return finalizeArtifactResponse{}, err
|
||||||
|
}
|
||||||
|
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||||
|
run := s.runDir(req.WorkflowRunBackendID)
|
||||||
|
entries, err := os.ReadDir(run)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return listArtifactsResponse{}, err
|
||||||
|
}
|
||||||
|
var resp listArtifactsResponse
|
||||||
|
for _, entry := range entries {
|
||||||
|
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||||
|
if err != nil || meta.Digest == "" ||
|
||||||
|
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||||
|
}
|
||||||
|
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||||
|
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||||
|
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
dir := s.urlDir(r)
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil || meta.Digest == "" {
|
||||||
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||||
|
contentType, filename := string(data), meta.Name
|
||||||
|
if !strings.Contains(contentType, "/") {
|
||||||
|
contentType = zipMimeType
|
||||||
|
}
|
||||||
|
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||||
|
filename += ".zip"
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", contentType)
|
||||||
|
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||||
|
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||||
|
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
meta, err := readMeta(dir)
|
||||||
|
if err != nil {
|
||||||
|
return deleteArtifactResponse{}, notFound(req.Name)
|
||||||
|
}
|
||||||
|
delete(s.created, dir)
|
||||||
|
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||||
|
now := time.Now()
|
||||||
|
claims, _ := json.Marshal(map[string]any{
|
||||||
|
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||||
|
"ac": `[{"Scope":"","Permission":3}]`,
|
||||||
|
"nbf": now.Unix(),
|
||||||
|
"exp": now.Add(lifetime).Unix(),
|
||||||
|
"jti": rand.Text(),
|
||||||
|
})
|
||||||
|
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
|
||||||
|
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||||
|
createArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
SignedUploadURL string `json:"signedUploadUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeArtifactRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
Size int64 `json:",string"`
|
||||||
|
Hash string
|
||||||
|
}
|
||||||
|
finalizeArtifactResponse struct {
|
||||||
|
Ok bool `json:"ok"`
|
||||||
|
ArtifactID int64 `json:"artifactId,string"`
|
||||||
|
}
|
||||||
|
|
||||||
|
listArtifactsRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
NameFilter string
|
||||||
|
IDFilter int64 `json:",string"`
|
||||||
|
}
|
||||||
|
listArtifactsResponse struct {
|
||||||
|
Artifacts []monolithArtifact `json:"artifacts"`
|
||||||
|
}
|
||||||
|
monolithArtifact struct {
|
||||||
|
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||||
|
DatabaseID int64 `json:"databaseId,string"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size int64 `json:"size,string"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||||
|
Digest string `json:"digest"` // empty until the upload is finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
getSignedArtifactURLRequest struct {
|
||||||
|
WorkflowRunBackendID string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
getSignedArtifactURLResponse struct {
|
||||||
|
SignedURL string `json:"signedUrl"`
|
||||||
|
}
|
||||||
|
|
||||||
|
createArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||||
|
deleteArtifactResponse = finalizeArtifactResponse
|
||||||
|
)
|
||||||
@@ -4,15 +4,15 @@
|
|||||||
package common
|
package common
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDryrunContext(t *testing.T) {
|
func TestDryrunContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Dryrun(ctx) {
|
if Dryrun(ctx) {
|
||||||
t.Fatal("plain context should not be dryrun")
|
t.Fatal("plain context should not be dryrun")
|
||||||
}
|
}
|
||||||
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestJobErrorContainer(t *testing.T) {
|
func TestJobErrorContainer(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
err := errors.New("job failed")
|
err := errors.New("job failed")
|
||||||
|
|
||||||
SetJobError(ctx, err)
|
SetJobError(ctx, err)
|
||||||
@@ -40,34 +40,27 @@ func TestJobErrorContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoggerAndHookContext(t *testing.T) {
|
func TestLoggerAndHandlerContext(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
if Logger(ctx) != logrus.StandardLogger() {
|
if Logger(ctx) != logging.Default() {
|
||||||
t.Fatal("plain context should use standard logger")
|
t.Fatal("plain context should use the default logger")
|
||||||
}
|
}
|
||||||
if LoggerHook(ctx) != nil {
|
if LoggerHandler(ctx) != nil {
|
||||||
t.Fatal("plain context should not have a logger hook")
|
t.Fatal("plain context should not have a logger handler")
|
||||||
|
}
|
||||||
|
if StepIDs(ctx) != nil {
|
||||||
|
t.Fatal("plain context should not have step IDs")
|
||||||
}
|
}
|
||||||
|
|
||||||
logger := logrus.New()
|
logger := logging.New(logtest.New())
|
||||||
ctx = WithLogger(ctx, logger)
|
ctx = WithLogger(ctx, logger)
|
||||||
if Logger(ctx) != logger {
|
if Logger(ctx) != logger {
|
||||||
t.Fatal("WithLogger should set logger")
|
t.Fatal("WithLogger should set logger")
|
||||||
}
|
}
|
||||||
|
|
||||||
hook := testHook{}
|
handler := logtest.New()
|
||||||
ctx = WithLoggerHook(ctx, hook)
|
ctx = WithLoggerHandler(ctx, handler)
|
||||||
if LoggerHook(ctx) != hook {
|
if LoggerHandler(ctx) != handler {
|
||||||
t.Fatal("WithLoggerHook should set hook")
|
t.Fatal("WithLoggerHandler should set handler")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type testHook struct{}
|
|
||||||
|
|
||||||
func (testHook) Levels() []logrus.Level {
|
|
||||||
return logrus.AllLevels
|
|
||||||
}
|
|
||||||
|
|
||||||
func (testHook) Fire(*logrus.Entry) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -14,13 +14,8 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
|
|||||||
|
|
||||||
// Dryrun returns true if the current context is dryrun
|
// Dryrun returns true if the current context is dryrun
|
||||||
func Dryrun(ctx context.Context) bool {
|
func Dryrun(ctx context.Context) bool {
|
||||||
val := ctx.Value(dryrunContextKeyVal)
|
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
|
||||||
if val != nil {
|
return dryrun
|
||||||
if dryrun, ok := val.(bool); ok {
|
|
||||||
return dryrun
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithDryrun adds a value to the context for dryrun
|
// WithDryrun adds a value to the context for dryrun
|
||||||
|
|||||||
+25
-29
@@ -9,7 +9,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Executor define contract for the steps of a workflow
|
// Executor define contract for the steps of a workflow
|
||||||
@@ -21,8 +21,7 @@ type Conditional func(ctx context.Context) bool
|
|||||||
// NewInfoExecutor is an executor that logs messages
|
// NewInfoExecutor is an executor that logs messages
|
||||||
func NewInfoExecutor(format string, args ...any) Executor {
|
func NewInfoExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Infof(format, args...)
|
||||||
logger.Infof(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -30,8 +29,7 @@ func NewInfoExecutor(format string, args ...any) Executor {
|
|||||||
// NewDebugExecutor is an executor that logs messages
|
// NewDebugExecutor is an executor that logs messages
|
||||||
func NewDebugExecutor(format string, args ...any) Executor {
|
func NewDebugExecutor(format string, args ...any) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := Logger(ctx)
|
Logger(ctx).Debugf(format, args...)
|
||||||
logger.Debugf(format, args...)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -39,9 +37,7 @@ func NewDebugExecutor(format string, args ...any) Executor {
|
|||||||
// NewPipelineExecutor creates a new executor from a series of other executors
|
// NewPipelineExecutor creates a new executor from a series of other executors
|
||||||
func NewPipelineExecutor(executors ...Executor) Executor {
|
func NewPipelineExecutor(executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
return func(ctx context.Context) error {
|
return NewErrorExecutor(nil)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
var rtn Executor
|
var rtn Executor
|
||||||
for _, executor := range executors {
|
for _, executor := range executors {
|
||||||
@@ -61,6 +57,13 @@ func NewErrorExecutor(err error) Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
|
||||||
|
if recovered := recover(); recovered != nil {
|
||||||
|
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
|
||||||
|
*err = fmt.Errorf("panic: %v", recovered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewParallelExecutor creates a new executor from a parallel of other executors
|
// NewParallelExecutor creates a new executor from a parallel of other executors
|
||||||
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||||
if len(executors) == 0 {
|
if len(executors) == 0 {
|
||||||
@@ -74,34 +77,29 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
|||||||
errs := make(chan error, len(executors))
|
errs := make(chan error, len(executors))
|
||||||
|
|
||||||
if 1 > parallel {
|
if 1 > parallel {
|
||||||
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
||||||
parallel = 1
|
parallel = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||||
|
|
||||||
for i := 0; i < parallel; i++ {
|
for workerID := range parallel {
|
||||||
go func(workerID int, work <-chan Executor, errs chan<- error) {
|
go func() {
|
||||||
log.Debugf("Worker %d started", workerID)
|
logging.Debugf("Worker %d started", workerID)
|
||||||
taskCount := 0
|
taskCount := 0
|
||||||
for executor := range work {
|
for executor := range work {
|
||||||
taskCount++
|
taskCount++
|
||||||
log.Debugf("Worker %d executing task %d", workerID, taskCount)
|
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
|
||||||
// Recover from panics in executors to avoid crashing the worker
|
// Recover from panics in executors to avoid crashing the worker
|
||||||
// goroutine which would leave the runner process hung.
|
// goroutine which would leave the runner process hung.
|
||||||
// https://gitea.com/gitea/runner/issues/371
|
// https://gitea.com/gitea/runner/issues/371
|
||||||
errs <- func() (err error) {
|
errs <- func() (err error) {
|
||||||
defer func() {
|
defer RecoverPanic(&err)
|
||||||
if r := recover(); r != nil {
|
|
||||||
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
|
|
||||||
err = fmt.Errorf("panic: %v", r)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return executor(ctx)
|
return executor(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||||
}(i, work, errs)
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range executors {
|
for i := range executors {
|
||||||
@@ -150,12 +148,7 @@ func (e Executor) If(conditional Conditional) Executor {
|
|||||||
|
|
||||||
// IfNot only runs this executor if conditional is true
|
// IfNot only runs this executor if conditional is true
|
||||||
func (e Executor) IfNot(conditional Conditional) Executor {
|
func (e Executor) IfNot(conditional Conditional) Executor {
|
||||||
return func(ctx context.Context) error {
|
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
|
||||||
if !conditional(ctx) {
|
|
||||||
return e(ctx)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IfBool only runs this executor if conditional is true
|
// IfBool only runs this executor if conditional is true
|
||||||
@@ -171,7 +164,10 @@ func (e Executor) Finally(finally Executor) Executor {
|
|||||||
err := e(ctx)
|
err := e(ctx)
|
||||||
err2 := finally(ctx)
|
err2 := finally(ctx)
|
||||||
if err2 != nil {
|
if err2 != nil {
|
||||||
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+65
-106
@@ -7,8 +7,7 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -18,42 +17,25 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestNewWorkflow(t *testing.T) {
|
func TestNewWorkflow(t *testing.T) {
|
||||||
assert := assert.New(t)
|
ctx := t.Context()
|
||||||
|
require.NoError(t, NewPipelineExecutor()(ctx))
|
||||||
|
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// empty
|
|
||||||
emptyWorkflow := NewPipelineExecutor()
|
|
||||||
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// error case
|
|
||||||
errorWorkflow := NewErrorExecutor(errors.New("test error"))
|
|
||||||
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
|
|
||||||
// multiple success case
|
|
||||||
runcount := 0
|
runcount := 0
|
||||||
successWorkflow := NewPipelineExecutor(
|
increment := func(_ctx context.Context) error {
|
||||||
func(ctx context.Context) error {
|
runcount++
|
||||||
runcount++
|
return nil
|
||||||
return nil
|
}
|
||||||
},
|
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
|
||||||
func(ctx context.Context) error {
|
assert.Equal(t, 2, runcount)
|
||||||
runcount++
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
assert.Equal(2, runcount)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
|
||||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
|
||||||
// find the gate already open so the last one still finishes with no partner left.
|
|
||||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||||
var counted, active, peak atomic.Int32
|
var counted, active, peak atomic.Int32
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
reached := make(chan struct{})
|
reached := make(chan struct{})
|
||||||
|
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
counted.Add(1)
|
counted.Add(1)
|
||||||
running := active.Add(1)
|
running := active.Add(1)
|
||||||
for {
|
for {
|
||||||
@@ -72,93 +54,72 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutor(t *testing.T) {
|
func TestNewParallelExecutor(t *testing.T) {
|
||||||
ctx := context.Background()
|
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
exec, count, maxActive := concurrencyProbe(2)
|
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
|
||||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
assert.Equal(t, int32(3), count.Load())
|
||||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
|
||||||
|
})
|
||||||
// parallelism below 1 falls back to a single worker
|
}
|
||||||
exec, count, maxActive = concurrencyProbe(1)
|
|
||||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
|
||||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
|
||||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
require.NoError(t, NewParallelExecutor(2)(ctx))
|
require.NoError(t, NewParallelExecutor(2)(ctx))
|
||||||
|
|
||||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
canceledCtx, cancel := context.WithCancel(t.Context())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
err := NewParallelExecutor(2)(canceledCtx)
|
err := NewParallelExecutor(2)(canceledCtx)
|
||||||
assert.ErrorIs(err, context.Canceled)
|
assert.ErrorIs(err, context.Canceled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewParallelExecutorFailed(t *testing.T) {
|
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
|
||||||
assert := assert.New(t)
|
for name, tc := range map[string]struct {
|
||||||
|
canceled bool
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
parallel, tasks int
|
||||||
cancel()
|
}{
|
||||||
|
"failure": {false, 2, 5},
|
||||||
count := 0
|
"canceled": {true, 2, 5},
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
"canceled single task": {true, 1, 1},
|
||||||
count++
|
} {
|
||||||
return errors.New("fake error")
|
t.Run(name, func(t *testing.T) {
|
||||||
})
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
err := NewParallelExecutor(1, errorWorkflow)(ctx)
|
defer cancel()
|
||||||
assert.Equal(1, count)
|
if tc.canceled {
|
||||||
assert.ErrorIs(context.Canceled, err)
|
cancel()
|
||||||
}
|
}
|
||||||
|
var calls atomic.Int32
|
||||||
func TestNewParallelExecutorCanceled(t *testing.T) {
|
executors := make([]Executor, tc.tasks)
|
||||||
assert := assert.New(t)
|
for i := range executors {
|
||||||
|
executors[i] = func(_ctx context.Context) error {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
calls.Add(1)
|
||||||
cancel()
|
if i == tc.tasks/2 {
|
||||||
|
return errors.New("fake error")
|
||||||
errExpected := errors.New("fake error")
|
}
|
||||||
|
return nil
|
||||||
var count atomic.Int32
|
}
|
||||||
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
|
||||||
count.Add(1)
|
|
||||||
return errExpected
|
|
||||||
})
|
|
||||||
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
|
|
||||||
assert.Equal(int32(3), count.Load())
|
|
||||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
|
||||||
var successCount atomic.Int32
|
|
||||||
executors := make([]Executor, 5)
|
|
||||||
for i := range executors {
|
|
||||||
executors[i] = func(ctx context.Context) error {
|
|
||||||
if i == 2 {
|
|
||||||
return errors.New("fake error")
|
|
||||||
}
|
}
|
||||||
successCount.Add(1)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
|
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
|
||||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
if tc.canceled {
|
||||||
|
require.ErrorIs(t, err, context.Canceled)
|
||||||
|
} else {
|
||||||
|
require.EqualError(t, err, "fake error")
|
||||||
|
}
|
||||||
|
assert.Equal(t, int32(tc.tasks), calls.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := t.Context()
|
||||||
var calls []string
|
var calls []string
|
||||||
record := func(name string) Executor {
|
record := func(name string) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(_ctx context.Context) error {
|
||||||
calls = append(calls, name)
|
calls = append(calls, name)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -170,19 +131,17 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
||||||
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
||||||
|
|
||||||
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
|
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||||
if !reflect.DeepEqual(calls, want) {
|
|
||||||
t.Fatalf("calls = %v, want %v", calls, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
mainErr := errors.New("main failed")
|
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
|
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||||
require.Error(t, err)
|
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
|
||||||
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
|
require.ErrorIs(t, err, finalErr)
|
||||||
t.Fatalf("finally error = %q, want both cleanup and original error", err)
|
if mainErr != nil {
|
||||||
|
require.ErrorIs(t, err, mainErr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"math/rand/v2"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
var actionFetches sync.Map
|
||||||
|
|
||||||
|
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||||
|
|
||||||
|
type treeFile struct {
|
||||||
|
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetch struct {
|
||||||
|
done chan struct{}
|
||||||
|
dir, sha string
|
||||||
|
err error
|
||||||
|
retry bool // the leader was cancelled or panicked, so followers fetch themselves
|
||||||
|
}
|
||||||
|
|
||||||
|
var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/runner backs off
|
||||||
|
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
||||||
|
return min(max(time.Duration(seconds)*time.Second, 10*time.Second), 10*time.Minute)
|
||||||
|
}
|
||||||
|
return time.Duration(10+rand.IntN(21)) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) { // input.Dir is one clone URL and ref's, holding a directory per commit
|
||||||
|
if !reflect.ValueOf(input).Comparable() {
|
||||||
|
return fetchAction(ctx, input)
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
call := &actionFetch{done: make(chan struct{}), retry: true}
|
||||||
|
previous, loaded := actionFetches.LoadOrStore(input, call)
|
||||||
|
if !loaded {
|
||||||
|
func() {
|
||||||
|
defer func() { actionFetches.Delete(input); close(call.done) }()
|
||||||
|
call.dir, call.sha, call.err = fetchAction(ctx, input)
|
||||||
|
call.retry = ctx.Err() != nil
|
||||||
|
}()
|
||||||
|
if call.retry {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return call.dir, call.sha, call.err
|
||||||
|
}
|
||||||
|
call, _ = previous.(*actionFetch)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-call.done:
|
||||||
|
if !call.retry && ctx.Err() == nil {
|
||||||
|
return call.dir, call.sha, call.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", "", ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) {
|
||||||
|
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.MkdirAll(input.Dir, 0o755); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
tree := func(sha string) string { return filepath.Join(input.Dir, sha) }
|
||||||
|
record := filepath.Join(input.Dir, "ref")
|
||||||
|
data, _ := os.ReadFile(record)
|
||||||
|
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||||
|
switch {
|
||||||
|
case !isFullObjectID(sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||||
|
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||||
|
case !treeIntact(input.Dir, sha):
|
||||||
|
sha, etag = "", ""
|
||||||
|
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||||
|
return tree(sha), sha, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
newSha, newEtag, err := downloadArchive(ctx, input, sha, etag)
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
common.Logger(ctx).Debugf("Cloning %s@%s: %v", input.URL, input.Ref, err)
|
||||||
|
newEtag = ""
|
||||||
|
newSha, err = exportClone(ctx, input)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if newSha != sha || newEtag != etag {
|
||||||
|
if err := os.WriteFile(record, []byte(newSha+"\n"+newEtag), 0o600); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tree(newSha), newSha, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadArchive(ctx context.Context, input NewGitCloneExecutorInput, sha, etag string) (string, string, error) {
|
||||||
|
rewritten, err := gitOutput(ctx, "", "ls-remote", "--get-url", input.URL)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
source, err := url.Parse(rewritten)
|
||||||
|
if err != nil || source.Scheme != "http" && source.Scheme != "https" || hexRefRegex.MatchString(input.Ref) && !isFullObjectID(input.Ref) { // only git tells a short commit from a branch
|
||||||
|
return "", "", errors.New("no archive")
|
||||||
|
}
|
||||||
|
archive := source.JoinPath("archive", input.Ref+".tar.gz")
|
||||||
|
if source.Host == "github.com" {
|
||||||
|
source.Host = "codeload.github.com"
|
||||||
|
archive = source.JoinPath("tar.gz", input.Ref)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: input.Transport, Timeout: 5 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
||||||
|
if len(via) >= 10 {
|
||||||
|
return errors.New("stopped after 10 redirects")
|
||||||
|
}
|
||||||
|
if request.URL.Host != via[0].URL.Host || request.URL.Scheme != via[0].URL.Scheme { // the token stays on its origin
|
||||||
|
request.Header.Del("Authorization")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
newSha, newEtag, retryAfter, err := downloadArchiveOnce(ctx, client, archive.String(), input.Token, sha, etag, input.Dir)
|
||||||
|
if retryAfter == 0 || attempt == 3 || ctx.Err() != nil {
|
||||||
|
return newSha, newEtag, err
|
||||||
|
}
|
||||||
|
common.Logger(ctx).Warnf("Failed to download action '%s'. Error: %v. Back off %s before retry.", archive.Redacted(), err, retryAfter)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", "", ctx.Err()
|
||||||
|
case <-time.After(retryAfter):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, token, sha, etag, dir string) (string, string, time.Duration, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archive, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", 0, err
|
||||||
|
}
|
||||||
|
if etag != "" {
|
||||||
|
request.Header.Set("If-None-Match", etag)
|
||||||
|
}
|
||||||
|
if token != "" {
|
||||||
|
request.SetBasicAuth("token", token)
|
||||||
|
}
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
var opErr *net.OpError
|
||||||
|
if errors.As(err, &opErr) && opErr.Op == "dial" {
|
||||||
|
return "", "", 0, err
|
||||||
|
}
|
||||||
|
return "", "", archiveRetryDelay(""), err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
switch status := response.StatusCode; {
|
||||||
|
case status == http.StatusNotModified && sha != "":
|
||||||
|
return sha, etag, 0, nil
|
||||||
|
case status == http.StatusOK:
|
||||||
|
newEtag := response.Header.Get("ETag")
|
||||||
|
gz, err := gzip.NewReader(response.Body)
|
||||||
|
if err == nil {
|
||||||
|
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||||
|
}
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||||
|
return "", "", archiveRetryDelay(""), err
|
||||||
|
}
|
||||||
|
return sha, newEtag, 0, err
|
||||||
|
case status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError:
|
||||||
|
return "", "", archiveRetryDelay(response.Header.Get("Retry-After")), errors.New(response.Status)
|
||||||
|
default:
|
||||||
|
return "", "", 0, errors.New(response.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, error) {
|
||||||
|
clone := input
|
||||||
|
clone.Dir = filepath.Join(input.Dir, "git")
|
||||||
|
sha, _, err := resolveClone(ctx, clone)
|
||||||
|
if err != nil || treeIntact(input.Dir, sha) {
|
||||||
|
return sha, err
|
||||||
|
}
|
||||||
|
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||||
|
return sha, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||||
|
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(temp)
|
||||||
|
root, err := os.OpenRoot(temp)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
|
||||||
|
sha := ""
|
||||||
|
sizes := map[string]int64{}
|
||||||
|
reader := tar.NewReader(archive)
|
||||||
|
for {
|
||||||
|
header, err := reader.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||||
|
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := header.Name
|
||||||
|
if strip {
|
||||||
|
_, name, _ = strings.Cut(name, "/")
|
||||||
|
}
|
||||||
|
if name = strings.TrimSuffix(name, "/"); name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch header.Typeflag {
|
||||||
|
case tar.TypeDir:
|
||||||
|
err = root.MkdirAll(name, 0o755)
|
||||||
|
case tar.TypeReg:
|
||||||
|
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||||
|
case tar.TypeSymlink:
|
||||||
|
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||||
|
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isFullObjectID(sha) {
|
||||||
|
return "", errors.New("the archive names no commit")
|
||||||
|
}
|
||||||
|
if err := root.Close(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||||
|
|
||||||
|
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||||
|
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||||
|
files := make([]treeFile, 0, len(sizes))
|
||||||
|
for name, size := range sizes {
|
||||||
|
files = append(files, treeFile{[]byte(name), size})
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tree := filepath.Join(dir, sha)
|
||||||
|
if !exists(tree) {
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||||
|
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(temp, tree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||||
|
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||||
|
func treeIntact(dir, sha string) bool {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||||
|
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var files []treeFile
|
||||||
|
if json.Unmarshal(data, &files) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||||
|
perm := os.FileMode(0o644)
|
||||||
|
if mode&0o111 != 0 {
|
||||||
|
perm = 0o755
|
||||||
|
}
|
||||||
|
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
n, err := io.Copy(file, content)
|
||||||
|
return n, errors.Join(err, file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
func exists(name string) bool {
|
||||||
|
_, err := os.Stat(name)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,412 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cgi"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFetchAction(t *testing.T) {
|
||||||
|
original := archiveRetryDelay
|
||||||
|
archiveRetryDelay = func(_retryAfter string) time.Duration { return time.Nanosecond }
|
||||||
|
t.Cleanup(func() { archiveRetryDelay = original })
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
remote := filepath.Join(root, "org", "action")
|
||||||
|
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||||
|
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||||
|
expectedSha := gitRevParse(t, remote, "main")
|
||||||
|
|
||||||
|
gitPath, err := exec.LookPath("git")
|
||||||
|
require.NoError(t, err)
|
||||||
|
backend := &cgi.Handler{Path: gitPath, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
|
||||||
|
var mu sync.Mutex
|
||||||
|
var archiveRequests []http.Header
|
||||||
|
var mirror *httptest.Server
|
||||||
|
archiveCache := map[string][]byte{}
|
||||||
|
requests, archives, failures := 0, "", 0
|
||||||
|
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
requests++
|
||||||
|
ref, ok := strings.CutPrefix(request.URL.Path, "/org/action/archive/")
|
||||||
|
if !ok {
|
||||||
|
backend.ServeHTTP(writer, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
archiveRequests = append(archiveRequests, request.Header.Clone())
|
||||||
|
switch {
|
||||||
|
case failures > 0:
|
||||||
|
failures--
|
||||||
|
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
case archives == "none":
|
||||||
|
http.NotFound(writer, request)
|
||||||
|
return
|
||||||
|
case archives == "unreachable":
|
||||||
|
http.Redirect(writer, request, "http://127.0.0.1:1"+request.URL.Path, http.StatusFound)
|
||||||
|
return
|
||||||
|
case archives == "redirect" && request.Host != mirror.Listener.Addr().String():
|
||||||
|
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||||
|
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||||
|
if request.Header.Get("If-None-Match") == etag {
|
||||||
|
writer.WriteHeader(http.StatusNotModified)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
args := []string{"archive", "--format=tar.gz", sha}
|
||||||
|
if archives != "unprefixed" {
|
||||||
|
args = append(args, "--prefix=action/")
|
||||||
|
}
|
||||||
|
archiveKey := strings.Join(args, "\x00")
|
||||||
|
archive, ok := archiveCache[archiveKey]
|
||||||
|
if !ok {
|
||||||
|
archive, _ = runGit(request.Context(), remote, nil, args...)
|
||||||
|
archiveCache[archiveKey] = archive
|
||||||
|
}
|
||||||
|
writer.Header().Set("ETag", etag)
|
||||||
|
_, _ = writer.Write(archive)
|
||||||
|
})
|
||||||
|
server := httptest.NewServer(handler)
|
||||||
|
mirror = httptest.NewServer(handler)
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
t.Cleanup(mirror.Close)
|
||||||
|
serve := func(mode string, failing int) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
requests, archives, failures, archiveRequests = 0, mode, failing, nil
|
||||||
|
}
|
||||||
|
served := func() (int, []http.Header) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return requests, append([]http.Header(nil), archiveRequests...)
|
||||||
|
}
|
||||||
|
newInput := func(ref string) NewGitCloneExecutorInput {
|
||||||
|
return NewGitCloneExecutorInput{URL: server.URL + "/org/action", Ref: ref, Dir: t.TempDir(), Token: "secret"}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput("main")
|
||||||
|
started, release := make(chan struct{}), make(chan struct{})
|
||||||
|
releaseFetches := sync.OnceFunc(func() { close(release) })
|
||||||
|
t.Cleanup(releaseFetches)
|
||||||
|
var once sync.Once
|
||||||
|
var panicked atomic.Bool
|
||||||
|
transport := actionFetchTransport(func(request *http.Request) (*http.Response, error) {
|
||||||
|
once.Do(func() { close(started) })
|
||||||
|
select {
|
||||||
|
case <-request.Context().Done():
|
||||||
|
return nil, request.Context().Err()
|
||||||
|
case <-release:
|
||||||
|
}
|
||||||
|
if scenario == "leader panics" && panicked.CompareAndSwap(false, true) {
|
||||||
|
panic("broken archive")
|
||||||
|
}
|
||||||
|
return http.DefaultTransport.RoundTrip(request)
|
||||||
|
})
|
||||||
|
input.Transport = &transport
|
||||||
|
leaderCtx, cancelLeader := context.WithCancel(t.Context())
|
||||||
|
defer cancelLeader()
|
||||||
|
waiting := &actionFetchContext{Context: t.Context(), waiting: make(chan struct{})}
|
||||||
|
followerInput := input
|
||||||
|
expectedRequests := 1
|
||||||
|
if scenario == "different token" {
|
||||||
|
followerInput.Token, expectedRequests = "other", 2
|
||||||
|
}
|
||||||
|
leaderDone, followerDone := make(chan error, 1), make(chan error, 1)
|
||||||
|
fetch := func(ctx context.Context, input NewGitCloneExecutorInput, done chan<- error) {
|
||||||
|
var err error
|
||||||
|
defer func() { done <- err }()
|
||||||
|
defer common.RecoverPanic(&err)
|
||||||
|
var tree, sha string
|
||||||
|
if tree, sha, err = FetchAction(ctx, input); err == nil {
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go fetch(leaderCtx, input, leaderDone)
|
||||||
|
<-started
|
||||||
|
go fetch(waiting, followerInput, followerDone)
|
||||||
|
<-waiting.waiting
|
||||||
|
if scenario == "leader cancelled" {
|
||||||
|
cancelLeader()
|
||||||
|
require.ErrorIs(t, <-leaderDone, context.Canceled)
|
||||||
|
}
|
||||||
|
releaseFetches()
|
||||||
|
switch scenario {
|
||||||
|
case "leader cancelled":
|
||||||
|
case "leader panics":
|
||||||
|
require.ErrorContains(t, <-leaderDone, "panic: broken archive")
|
||||||
|
default:
|
||||||
|
require.NoError(t, <-leaderDone)
|
||||||
|
}
|
||||||
|
require.NoError(t, <-followerDone)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, expectedRequests)
|
||||||
|
if scenario == "different token" {
|
||||||
|
assert.NotEqual(t, headers[0].Get("Authorization"), headers[1].Get("Authorization"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads, revalidates and follows a moved tag", func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput("v1")
|
||||||
|
tree, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||||
|
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||||
|
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||||
|
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "action.yml", link)
|
||||||
|
|
||||||
|
again, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tree, again)
|
||||||
|
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||||
|
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||||
|
expectedSha = gitRevParse(t, remote, "v1")
|
||||||
|
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, movedSha)
|
||||||
|
assert.NotEqual(t, tree, moved)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
|
||||||
|
input.OfflineMode = true
|
||||||
|
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, movedSha, offlineSha)
|
||||||
|
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||||
|
reused, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, moved, reused)
|
||||||
|
|
||||||
|
requests, headers := served()
|
||||||
|
assert.Equal(t, 4, requests)
|
||||||
|
require.Len(t, headers, 4)
|
||||||
|
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||||
|
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||||
|
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||||
|
assert.Equal(t, headers[1].Get("If-None-Match"), headers[2].Get("If-None-Match"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||||
|
serve("prefixed", 0)
|
||||||
|
sha := expectedSha
|
||||||
|
input := newInput(sha)
|
||||||
|
for range 2 {
|
||||||
|
tree, got, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, sha, got)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||||
|
}
|
||||||
|
_, _, err := FetchAction(t.Context(), newInput(sha[:7]))
|
||||||
|
require.ErrorIs(t, err, ErrShortRef)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||||
|
sha := gitRevParse(t, remote, "main")
|
||||||
|
for _, mode := range []string{"prefixed", "none"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||||
|
_, _, err = FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
for _, damage := range []func() error{
|
||||||
|
func() error { return os.Truncate(yml, 4) },
|
||||||
|
func() error { return os.Remove(tree + treeSizes) },
|
||||||
|
} {
|
||||||
|
require.NoError(t, damage(), mode)
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, "name: action\n", string(body), mode)
|
||||||
|
}
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 3, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
input := newInput(sha)
|
||||||
|
tree, _, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
yml := filepath.Join(tree, "action.yml")
|
||||||
|
require.NoError(t, os.Truncate(yml, 4))
|
||||||
|
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||||
|
require.NoError(t, err)
|
||||||
|
release := sync.OnceFunc(unlock)
|
||||||
|
defer release()
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||||
|
close(waiting)
|
||||||
|
return AcquireCloneLock(ctx, dir)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, _, err := FetchAction(t.Context(), input)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||||
|
case <-waiting:
|
||||||
|
}
|
||||||
|
info, err := os.Stat(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||||
|
release()
|
||||||
|
require.NoError(t, <-done)
|
||||||
|
body, err := os.ReadFile(yml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "name: action\n", string(body))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||||
|
serve("redirect", 2)
|
||||||
|
input := newInput("main")
|
||||||
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
_, headers := served()
|
||||||
|
require.Len(t, headers, 4)
|
||||||
|
assert.NotEmpty(t, headers[2].Get("Authorization"))
|
||||||
|
assert.Empty(t, headers[3].Get("Authorization"))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("keeps an unprefixed archive whole, downloads where insteadOf points and exports a clone without an archive or from an unreachable host", func(t *testing.T) {
|
||||||
|
for _, mode := range []string{"unprefixed", "none", "unreachable"} {
|
||||||
|
serve(mode, 0)
|
||||||
|
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||||
|
require.NoError(t, err, mode)
|
||||||
|
assert.Equal(t, expectedSha, sha, mode)
|
||||||
|
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||||
|
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
serve("prefixed", 0)
|
||||||
|
config := filepath.Join(t.TempDir(), "gitconfig")
|
||||||
|
require.NoError(t, os.WriteFile(config, []byte("[url \""+server.URL+"/org/\"]\n\tinsteadOf = https://example.invalid/org/\n"), 0o644))
|
||||||
|
t.Setenv("GIT_CONFIG_GLOBAL", config)
|
||||||
|
input := newInput("main")
|
||||||
|
input.URL = "https://example.invalid/org/action"
|
||||||
|
_, sha, err := FetchAction(t.Context(), input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedSha, sha)
|
||||||
|
_, headers := served()
|
||||||
|
assert.Len(t, headers, 1)
|
||||||
|
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetchTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (transport actionFetchTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
return transport(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
type actionFetchContext struct {
|
||||||
|
context.Context
|
||||||
|
waiting chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ctx *actionFetchContext) Done() <-chan struct{} {
|
||||||
|
ctx.once.Do(func() { close(ctx.waiting) })
|
||||||
|
return ctx.Context.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||||
|
outside := t.TempDir()
|
||||||
|
for name, entries := range map[string][]*tar.Header{
|
||||||
|
"parent path": {{Name: "../escape", Typeflag: tar.TypeReg}},
|
||||||
|
"through a symlink": {{Name: "out", Typeflag: tar.TypeSymlink, Linkname: outside}, {Name: "out/escape", Typeflag: tar.TypeReg}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||||
|
for _, entry := range entries {
|
||||||
|
require.NoError(t, writer.WriteHeader(entry))
|
||||||
|
}
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||||
|
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||||
|
}
|
||||||
|
sha := strings.Repeat("a", 40)
|
||||||
|
var archive bytes.Buffer
|
||||||
|
writer := tar.NewWriter(&archive)
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||||
|
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||||
|
_, err := writer.Write([]byte("x"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||||
|
assert.True(t, treeIntact(dir, sha))
|
||||||
|
}
|
||||||
+145
-68
@@ -10,8 +10,12 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -21,16 +25,13 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common"
|
"gitea.com/gitea/runner/act/common"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
|
||||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
|
||||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||||
|
|
||||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||||
@@ -44,12 +45,29 @@ var (
|
|||||||
ErrShortRef = errors.New("short SHA references are not supported")
|
ErrShortRef = errors.New("short SHA references are not supported")
|
||||||
)
|
)
|
||||||
|
|
||||||
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
|
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
|
||||||
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
|
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
|
||||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
dir = filepath.Clean(dir)
|
||||||
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
|
if runtime.GOOS == "plan9" {
|
||||||
func AcquireCloneLock(dir string) func() {
|
return cloneLocks.Lock(dir), nil
|
||||||
return cloneLocks.Lock(dir)
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
release, err := lock.TryLock(dir)
|
||||||
|
if err == nil {
|
||||||
|
return func() { _ = release() }, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, lock.ErrLocked) {
|
||||||
|
return cloneLocks.Lock(dir), nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Error struct {
|
type Error struct {
|
||||||
@@ -62,25 +80,25 @@ func (e *Error) Unwrap() error { return e.err }
|
|||||||
func (e *Error) Commit() string { return e.commit }
|
func (e *Error) Commit() string { return e.commit }
|
||||||
|
|
||||||
// FindGitRevision get the current git revision
|
// FindGitRevision get the current git revision
|
||||||
func FindGitRevision(ctx context.Context, file string) (shortSHA, sha string, err error) {
|
func FindGitRevision(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
sha, err = gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||||
return "", "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if len(sha) < 7 {
|
if len(sha) < 7 {
|
||||||
return "", "", errors.New("HEAD could not be resolved")
|
return "", errors.New("HEAD could not be resolved")
|
||||||
}
|
}
|
||||||
logger.Debugf("Found revision: %s", sha)
|
logger.Debugf("Found revision: %s", sha)
|
||||||
return sha[:7], sha, nil
|
return sha, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindGitRef get the current git ref
|
// FindGitRef get the current git ref
|
||||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
logger.Debugf("Loading revision from git directory")
|
logger.Debugf("Loading revision from git directory")
|
||||||
_, revision, err := FindGitRevision(ctx, file)
|
revision, err := FindGitRevision(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -109,13 +127,12 @@ func FindGitRef(ctx context.Context, file string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FindGithubRepo get the repo
|
// FindGithubRepo get the repo
|
||||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
func FindGithubRepo(ctx context.Context, file string) (string, error) {
|
||||||
url, err := findGitRemoteURL(ctx, file)
|
url, err := findGitRemoteURL(ctx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
_, slug := findGitSlug(url, githubInstance)
|
return findGitSlug(url)
|
||||||
return slug, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||||
@@ -130,25 +147,25 @@ func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
|||||||
return url, nil
|
return url, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findGitSlug(url, githubInstance string) (string, string) {
|
func findGitSlug(remoteURL string) (string, error) {
|
||||||
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
|
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
|
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||||
return "CodeCommit", matches[2]
|
return matches[2], nil
|
||||||
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if githubInstance != "github.com" {
|
|
||||||
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
|
|
||||||
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
|
|
||||||
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
|
|
||||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return "", url
|
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
|
||||||
|
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(remoteURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("cannot parse the origin remote URL")
|
||||||
|
}
|
||||||
|
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
|
||||||
|
owner = path.Base(owner)
|
||||||
|
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
|
||||||
|
return "", errors.New("cannot determine owner and repository from the origin remote URL")
|
||||||
|
}
|
||||||
|
return owner + "/" + repo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
||||||
@@ -163,17 +180,14 @@ type NewGitCloneExecutorInput struct {
|
|||||||
// 0 for full clone.
|
// 0 for full clone.
|
||||||
Depth int
|
Depth int
|
||||||
|
|
||||||
// Quiet drops the informational clone line to debug level, for callers that log their own
|
|
||||||
// download summary (the setup section's action report).
|
|
||||||
Quiet bool
|
|
||||||
|
|
||||||
// For Gitea
|
// For Gitea
|
||||||
InsecureSkipTLS bool
|
InsecureSkipTLS bool
|
||||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||||
|
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneIfRequired reports whether an existing local clone was reused.
|
// CloneIfRequired reports whether an existing local clone was reused.
|
||||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger log.FieldLogger) (bool, error) {
|
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
|
||||||
var origin string
|
var origin string
|
||||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -182,6 +196,9 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
if err == nil && origin == input.URL {
|
if err == nil && origin == input.URL {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
|
||||||
|
return false, ctxErr
|
||||||
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||||
@@ -203,32 +220,18 @@ func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger
|
|||||||
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
if input.Quiet {
|
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||||
logger.Debugf("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
} else {
|
|
||||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
|
||||||
}
|
|
||||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||||
defer AcquireCloneLock(input.Dir)()
|
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
reused, err := CloneIfRequired(ctx, input, logger)
|
resolved, reused, err := resolveClone(ctx, input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
|
||||||
if err := fetch(ctx, input); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||||
}
|
}
|
||||||
@@ -242,6 +245,28 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
|
||||||
|
logger := common.Logger(ctx)
|
||||||
|
reused, err := CloneIfRequired(ctx, input, logger)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||||
|
if err := fetch(ctx, input); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||||
|
}
|
||||||
|
return resolved, reused, err
|
||||||
|
}
|
||||||
|
|
||||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||||
@@ -282,7 +307,7 @@ func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||||
args := []string{"fetch", "--force", "--prune"}
|
args := []string{"fetch", "--force", "--no-prune"}
|
||||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||||
args = append(args, "--depth", "1", "--no-tags")
|
args = append(args, "--depth", "1", "--no-tags")
|
||||||
}
|
}
|
||||||
@@ -296,9 +321,18 @@ func cachedRef(ref string) string {
|
|||||||
|
|
||||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||||
if !isFullObjectID(ref) {
|
if !isFullObjectID(ref) {
|
||||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
|
||||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
if strings.ContainsAny(ref, "\x00\n") {
|
||||||
return sha, nil
|
for _, revision := range revisions {
|
||||||
|
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
|
||||||
|
for sha := range strings.SplitSeq(string(output), "\n") {
|
||||||
|
if isFullObjectID(sha) {
|
||||||
|
return sha, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -364,7 +398,49 @@ func ConfigEnv(config ...string) map[string]string {
|
|||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
||||||
|
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var files, repos []string
|
||||||
|
for entry := range strings.SplitSeq(string(listed), "\x00") {
|
||||||
|
tag, file, ok := strings.Cut(entry, " ")
|
||||||
|
if tag != "?" {
|
||||||
|
var meta string
|
||||||
|
meta, file, ok = strings.Cut(file, "\t")
|
||||||
|
if strings.HasPrefix(meta, "160000 ") {
|
||||||
|
file += "/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
|
||||||
|
repos = append(repos, repo)
|
||||||
|
} else if ok {
|
||||||
|
files = append(files, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(repos)
|
||||||
|
for _, repo := range slices.Compact(repos) {
|
||||||
|
files = append(files, repo)
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
|
||||||
|
continue // git lists an uninitialized submodule as "./" from inside it
|
||||||
|
}
|
||||||
|
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, file := range nested {
|
||||||
|
files = append(files, repo+"/"+file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||||
|
return runGitWithStdin(ctx, dir, input, nil, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
|
||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
@@ -376,6 +452,7 @@ func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, ar
|
|||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
cmd := exec.CommandContext(ctx, "git", args...)
|
||||||
|
cmd.Stdin = stdin
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||||
name, _, _ := strings.Cut(entry, "=")
|
name, _, _ := strings.Cut(entry, "=")
|
||||||
return slices.Contains(LocalEnvVars, name)
|
return slices.Contains(LocalEnvVars, name)
|
||||||
|
|||||||
+214
-251
File diff suppressed because it is too large.
Load diff
@@ -14,13 +14,8 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
|
|||||||
|
|
||||||
// JobError returns the job error for current context if any
|
// JobError returns the job error for current context if any
|
||||||
func JobError(ctx context.Context) error {
|
func JobError(ctx context.Context) error {
|
||||||
val := ctx.Value(jobErrorContextKeyVal)
|
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
|
||||||
if val != nil {
|
return container["error"]
|
||||||
if container, ok := val.(map[string]error); ok {
|
|
||||||
return container["error"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetJobError(ctx context.Context, err error) {
|
func SetJobError(ctx context.Context, err error) {
|
||||||
@@ -31,6 +26,5 @@ func SetJobError(ctx context.Context, err error) {
|
|||||||
|
|
||||||
// WithJobErrorContainer adds a value to the context as a container for an error
|
// WithJobErrorContainer adds a value to the context as a container for an error
|
||||||
func WithJobErrorContainer(ctx context.Context) context.Context {
|
func WithJobErrorContainer(ctx context.Context) context.Context {
|
||||||
container := map[string]error{}
|
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
|
||||||
return context.WithValue(ctx, jobErrorContextKeyVal, container)
|
|
||||||
}
|
}
|
||||||
@@ -26,9 +26,7 @@ type lineWriter struct {
|
|||||||
|
|
||||||
// NewLineWriter creates a new instance of a line writer
|
// NewLineWriter creates a new instance of a line writer
|
||||||
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
||||||
w := new(lineWriter)
|
return &lineWriter{handlers: handlers}
|
||||||
w.handlers = handlers
|
|
||||||
return w
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
||||||
@@ -39,24 +37,15 @@ func FlushWriter(w io.Writer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lw *lineWriter) Write(p []byte) (n int, err error) {
|
func (lw *lineWriter) Write(p []byte) (int, error) {
|
||||||
pBuf := bytes.NewBuffer(p)
|
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
|
||||||
written := 0
|
_, _ = lw.buffer.Write(line)
|
||||||
for {
|
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||||
line, err := pBuf.ReadString('\n')
|
lw.handleLine(lw.buffer.String())
|
||||||
w, _ := lw.buffer.WriteString(line)
|
lw.buffer.Reset()
|
||||||
written += w
|
|
||||||
if err != nil {
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return written, err
|
|
||||||
}
|
}
|
||||||
lw.handleLine(lw.buffer.String())
|
|
||||||
lw.buffer.Reset()
|
|
||||||
}
|
}
|
||||||
|
return len(p), nil
|
||||||
return written, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
||||||
@@ -72,8 +61,7 @@ func (lw *lineWriter) Flush() {
|
|||||||
|
|
||||||
func (lw *lineWriter) handleLine(line string) {
|
func (lw *lineWriter) handleLine(line string) {
|
||||||
for _, h := range lw.handlers {
|
for _, h := range lw.handlers {
|
||||||
ok := h(line)
|
if !h(line) {
|
||||||
if !ok {
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,24 +9,24 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLineWriter(t *testing.T) {
|
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
|
||||||
lines := make([]string, 0)
|
var lines []string
|
||||||
lineHandler := func(s string) bool {
|
lineWriter := NewLineWriter(func(s string) bool {
|
||||||
lines = append(lines, s)
|
lines = append(lines, s)
|
||||||
return true
|
return true
|
||||||
}
|
})
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
write := func(s string) {
|
write := func(s string) {
|
||||||
n, err := lineWriter.Write([]byte(s))
|
n, err := lineWriter.Write([]byte(s))
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
require.NoError(t, err)
|
||||||
assert.Equal(len(s), n, s)
|
assert.Equal(len(s), n, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
write("")
|
||||||
write("hello")
|
write("hello")
|
||||||
write(" ")
|
write(" ")
|
||||||
write("world!!\nextra")
|
write("world!!\nextra")
|
||||||
@@ -34,39 +34,17 @@ func TestLineWriter(t *testing.T) {
|
|||||||
write(" line\n")
|
write(" line\n")
|
||||||
write("no newline here...")
|
write("no newline here...")
|
||||||
|
|
||||||
assert.Len(lines, 4)
|
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
|
||||||
assert.Equal("hello world!!\n", lines[0])
|
assert.Equal(expected, lines)
|
||||||
assert.Equal("extra line\n", lines[1])
|
|
||||||
assert.Equal(" and another\n", lines[2])
|
|
||||||
assert.Equal("last line\n", lines[3])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLineWriterFlush(t *testing.T) {
|
|
||||||
lines := make([]string, 0)
|
|
||||||
lineHandler := func(s string) bool {
|
|
||||||
lines = append(lines, s)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
lineWriter := NewLineWriter(lineHandler)
|
|
||||||
|
|
||||||
assert := assert.New(t)
|
|
||||||
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
|
|
||||||
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
|
|
||||||
|
|
||||||
// Only the newline-terminated line is emitted before flushing.
|
|
||||||
assert.Equal([]string{"complete line\n"}, lines)
|
|
||||||
|
|
||||||
// Flushing emits the buffered, not-yet-terminated trailing line.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
|
expected = append(expected, "no newline here...")
|
||||||
|
assert.Equal(expected, lines)
|
||||||
|
|
||||||
// Flushing again is a no-op: nothing is buffered.
|
|
||||||
FlushWriter(lineWriter)
|
FlushWriter(lineWriter)
|
||||||
assert.Len(lines, 2)
|
assert.Equal(expected, lines)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
||||||
// FlushWriter must be a safe no-op for writers that do not buffer lines.
|
|
||||||
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
||||||
}
|
}
|
||||||
+30
-25
@@ -7,46 +7,51 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
type loggerContextKey string
|
type loggerContextKey string
|
||||||
|
|
||||||
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
|
const loggerContextKeyVal = loggerContextKey("logging.Logger")
|
||||||
|
|
||||||
// Logger returns the appropriate logger for current context
|
// Logger returns the appropriate logger for current context
|
||||||
func Logger(ctx context.Context) logrus.FieldLogger {
|
func Logger(ctx context.Context) *logging.Logger {
|
||||||
val := ctx.Value(loggerContextKeyVal)
|
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
|
||||||
if val != nil {
|
return logger
|
||||||
if logger, ok := val.(logrus.FieldLogger); ok {
|
|
||||||
return logger
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return logrus.StandardLogger()
|
return logging.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLogger adds a value to the context for the logger
|
// WithLogger adds a value to the context for the logger
|
||||||
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
|
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
|
||||||
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
||||||
}
|
}
|
||||||
|
|
||||||
type loggerHookKey string
|
type loggerHandlerKey string
|
||||||
|
|
||||||
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
|
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
|
||||||
|
|
||||||
// LoggerHook returns the appropriate logger hook for current context
|
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
|
||||||
// the hook affects job logger, not global logger
|
// It affects the job logger, not the global logger.
|
||||||
func LoggerHook(ctx context.Context) logrus.Hook {
|
func LoggerHandler(ctx context.Context) logging.Handler {
|
||||||
val := ctx.Value(loggerHookKeyVal)
|
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
|
||||||
if val != nil {
|
return h
|
||||||
if hook, ok := val.(logrus.Hook); ok {
|
|
||||||
return hook
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLoggerHook adds a value to the context for the logger hook
|
// WithLoggerHandler adds the extra handler for the job logger to the context
|
||||||
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
|
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
|
||||||
return context.WithValue(ctx, loggerHookKeyVal, hook)
|
return context.WithValue(ctx, loggerHandlerKeyVal, h)
|
||||||
|
}
|
||||||
|
|
||||||
|
type stepIDsContextKey struct{}
|
||||||
|
|
||||||
|
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
|
||||||
|
func StepIDs(ctx context.Context) []string {
|
||||||
|
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithStepIDs replaces the step IDs in the context
|
||||||
|
func WithStepIDs(ctx context.Context, ids []string) context.Context {
|
||||||
|
return context.WithValue(ctx, stepIDsContextKey{}, ids)
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ansiReset = "\x1b[0m"
|
||||||
|
ansiRed = "\x1b[31m"
|
||||||
|
ansiYellow = "\x1b[33m"
|
||||||
|
ansiCyan = "\x1b[36m"
|
||||||
|
ansiDimCyan = "\x1b[2;36m"
|
||||||
|
ansiDimWhite = "\x1b[2;37m"
|
||||||
|
)
|
||||||
|
|
||||||
|
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
|
||||||
|
var consoleStart = time.Now()
|
||||||
|
|
||||||
|
// console is the state shared by every consoleHandler derived from one another.
|
||||||
|
type console struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
out io.Writer
|
||||||
|
terminal bool
|
||||||
|
opts TextOptions
|
||||||
|
reportCaller bool
|
||||||
|
level slog.LevelVar
|
||||||
|
}
|
||||||
|
|
||||||
|
func newConsole(out io.Writer) *console {
|
||||||
|
c := &console{}
|
||||||
|
c.setOutput(out)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *console) setOutput(w io.Writer) {
|
||||||
|
terminal := checkIfTerminal(w)
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.out, c.terminal = w, terminal
|
||||||
|
}
|
||||||
|
|
||||||
|
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
|
||||||
|
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
|
||||||
|
// and a colored prefix with elapsed or full time on a terminal.
|
||||||
|
type consoleHandler struct {
|
||||||
|
c *console
|
||||||
|
attrs []slog.Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||||
|
return level >= h.c.level.Level()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||||
|
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
|
||||||
|
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
|
||||||
|
|
||||||
|
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||||
|
c := h.c
|
||||||
|
c.mu.Lock()
|
||||||
|
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
|
||||||
|
for _, a := range h.attrs {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
}
|
||||||
|
r.Attrs(func(a slog.Attr) bool {
|
||||||
|
data[a.Key] = a.Value.Resolve().Any()
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
|
||||||
|
var caller *runtime.Frame
|
||||||
|
if reportCaller && r.PC != 0 {
|
||||||
|
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
|
||||||
|
caller = &frame
|
||||||
|
}
|
||||||
|
prefixFieldClashes(data, caller != nil)
|
||||||
|
|
||||||
|
keys := slices.Sorted(maps.Keys(data))
|
||||||
|
|
||||||
|
var b bytes.Buffer
|
||||||
|
if terminal && !opts.DisableColors {
|
||||||
|
printColored(&b, opts, r, caller, keys, data)
|
||||||
|
} else {
|
||||||
|
printPlain(&b, opts, r, caller, keys, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
line := b.Bytes()
|
||||||
|
if opts.Filter != nil {
|
||||||
|
line = opts.Filter(line)
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
_, err := c.out.Write(line)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
|
||||||
|
func prefixFieldClashes(data map[string]any, reportCaller bool) {
|
||||||
|
for _, key := range []string{"time", "level", "msg"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
delete(data, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reportCaller {
|
||||||
|
for _, key := range []string{"func", "file"} {
|
||||||
|
if v, ok := data[key]; ok {
|
||||||
|
data["fields."+key] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
if !opts.DisableTimestamp {
|
||||||
|
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
|
||||||
|
if r.Message != "" {
|
||||||
|
appendKeyValue(b, "msg", r.Message)
|
||||||
|
}
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if funcVal != "" {
|
||||||
|
appendKeyValue(b, "func", funcVal)
|
||||||
|
}
|
||||||
|
if fileVal != "" {
|
||||||
|
appendKeyValue(b, "file", fileVal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
appendKeyValue(b, k, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
|
||||||
|
if opts.CallerPrettyfier != nil {
|
||||||
|
return opts.CallerPrettyfier(caller)
|
||||||
|
}
|
||||||
|
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||||
|
msg := strings.TrimSuffix(r.Message, "\n")
|
||||||
|
|
||||||
|
var callerStr string
|
||||||
|
if caller != nil {
|
||||||
|
funcVal, fileVal := callerText(opts, caller)
|
||||||
|
if opts.CallerPrettyfier == nil && caller.Function != "" {
|
||||||
|
funcVal += "()"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case fileVal == "":
|
||||||
|
callerStr = funcVal
|
||||||
|
case funcVal == "":
|
||||||
|
callerStr = fileVal
|
||||||
|
default:
|
||||||
|
callerStr = fileVal + " " + funcVal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
level := Level(r.Level)
|
||||||
|
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
|
||||||
|
switch {
|
||||||
|
case opts.DisableTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
|
||||||
|
case !opts.FullTimestamp:
|
||||||
|
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
b.WriteString(colorize(level, k))
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, data[k])
|
||||||
|
}
|
||||||
|
b.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainLevel spells Warn the way logrus did, as "warning".
|
||||||
|
func plainLevel(l Level) string {
|
||||||
|
if s := l.String(); s != "warn" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "warning"
|
||||||
|
}
|
||||||
|
|
||||||
|
func colorize(l Level, s string) string {
|
||||||
|
var color string
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
color = ansiDimWhite
|
||||||
|
case l <= LevelDebug:
|
||||||
|
color = ansiDimCyan
|
||||||
|
case l <= LevelInfo:
|
||||||
|
color = ansiCyan
|
||||||
|
case l <= LevelWarn:
|
||||||
|
color = ansiYellow
|
||||||
|
default:
|
||||||
|
color = ansiRed
|
||||||
|
}
|
||||||
|
return color + s + ansiReset
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendKeyValue(b *bytes.Buffer, key string, value any) {
|
||||||
|
if b.Len() > 0 {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
b.WriteString(key)
|
||||||
|
b.WriteByte('=')
|
||||||
|
appendValue(b, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendValue(b *bytes.Buffer, value any) {
|
||||||
|
switch v := value.(type) {
|
||||||
|
case string:
|
||||||
|
appendString(b, v)
|
||||||
|
case []byte:
|
||||||
|
appendString(b, string(v))
|
||||||
|
case error:
|
||||||
|
appendFormatted(b, v, v.Error)
|
||||||
|
case fmt.Stringer:
|
||||||
|
appendFormatted(b, v, v.String)
|
||||||
|
default:
|
||||||
|
appendString(b, fmt.Sprint(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
|
||||||
|
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
|
||||||
|
appendString(b, "<nil>")
|
||||||
|
} else {
|
||||||
|
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
appendString(b, get())
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendString(b *bytes.Buffer, s string) {
|
||||||
|
if needsQuoting(s) {
|
||||||
|
b.WriteString(strconv.Quote(s))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.WriteString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsQuoting(s string) bool {
|
||||||
|
for i := range len(s) {
|
||||||
|
c := s[i]
|
||||||
|
switch {
|
||||||
|
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
|
||||||
|
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
|
||||||
|
t.Helper()
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
|
||||||
|
c.level.Set(slog.LevelDebug)
|
||||||
|
build(New(NewSlogHandler(&consoleHandler{c: c})))
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlain(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
|
||||||
|
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
|
||||||
|
l.With("msg", "clash", "empty", "").Info("")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
|
||||||
|
"level=info empty= fields.msg=clash\n", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
|
||||||
|
opts := TextOptions{
|
||||||
|
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
|
||||||
|
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
|
||||||
|
},
|
||||||
|
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
|
||||||
|
}
|
||||||
|
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
|
||||||
|
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleColored(t *testing.T) {
|
||||||
|
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
|
||||||
|
l.With("job", "j").Warn("careful")
|
||||||
|
})
|
||||||
|
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
|
||||||
|
|
||||||
|
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
|
||||||
|
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
|
||||||
|
|
||||||
|
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
|
||||||
|
|
||||||
|
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
|
||||||
|
assert.Equal(t, "level=info msg=x\n", plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsoleLevelAndOutput(t *testing.T) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
c := newConsole(&out)
|
||||||
|
h := &consoleHandler{c: c}
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
|
||||||
|
c.level.Set(slog.LevelError)
|
||||||
|
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
|
||||||
|
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
|
||||||
|
|
||||||
|
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
|
||||||
|
rec.AddAttrs(slog.String("c", "3"))
|
||||||
|
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
|
||||||
|
assert.Contains(t, out.String(), "a=1 c=3")
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TextOptions describes how the process-wide Logger writes a record as text.
|
||||||
|
type TextOptions struct {
|
||||||
|
DisableColors bool
|
||||||
|
DisableTimestamp bool
|
||||||
|
FullTimestamp bool
|
||||||
|
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
|
||||||
|
CallerPrettyfier func(*runtime.Frame) (function, file string)
|
||||||
|
// Filter rewrites each formatted line, for example to mask secrets.
|
||||||
|
Filter func(line []byte) []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
std atomic.Pointer[Logger]
|
||||||
|
stdState = newConsole(os.Stderr)
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default returns the process-wide Logger.
|
||||||
|
func Default() *Logger { return std.Load() }
|
||||||
|
|
||||||
|
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
|
||||||
|
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
|
||||||
|
// initial console Logger.
|
||||||
|
func SetDefault(l *Logger) { std.Store(l) }
|
||||||
|
|
||||||
|
// SetLevel sets the minimum level of the initial process-wide Logger.
|
||||||
|
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
|
||||||
|
|
||||||
|
// GetLevel returns the minimum level of the initial process-wide Logger.
|
||||||
|
func GetLevel() Level { return Level(stdState.level.Level()) }
|
||||||
|
|
||||||
|
// SetOutput sets where the initial process-wide Logger writes.
|
||||||
|
func SetOutput(w io.Writer) { stdState.setOutput(w) }
|
||||||
|
|
||||||
|
// Output returns where the initial process-wide Logger writes.
|
||||||
|
func Output() io.Writer {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
return stdState.out
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetTextFormat sets the text format of the initial process-wide Logger.
|
||||||
|
func SetTextFormat(o TextOptions) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.opts = o
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
|
||||||
|
func SetReportCaller(on bool) {
|
||||||
|
stdState.mu.Lock()
|
||||||
|
defer stdState.mu.Unlock()
|
||||||
|
stdState.reportCaller = on
|
||||||
|
}
|
||||||
|
|
||||||
|
func Enabled(level Level) bool { return Default().Enabled(level) }
|
||||||
|
|
||||||
|
func With(args ...any) *Logger { return Default().With(args...) }
|
||||||
|
func WithError(err error) *Logger { return Default().WithError(err) }
|
||||||
|
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
|
||||||
|
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
|
||||||
|
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
|
||||||
|
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
|
||||||
|
func Debug(args ...any) { Default().log(LevelDebug, args) }
|
||||||
|
func Info(args ...any) { Default().log(LevelInfo, args) }
|
||||||
|
func Warn(args ...any) { Default().log(LevelWarn, args) }
|
||||||
|
func Error(args ...any) { Default().log(LevelError, args) }
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Attr is a structured key/value pair attached to a log record.
|
||||||
|
type Attr struct {
|
||||||
|
Key string
|
||||||
|
Value any
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
|
||||||
|
// Logger.With are accumulated by the Handler.
|
||||||
|
type Record struct {
|
||||||
|
Time time.Time
|
||||||
|
Level Level
|
||||||
|
Message string
|
||||||
|
PC uintptr // program counter of the logging call, 0 if unknown
|
||||||
|
Attrs []Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attr returns the value of the last attribute named key.
|
||||||
|
func (r Record) Attr(key string) (any, bool) {
|
||||||
|
for _, a := range slices.Backward(r.Attrs) {
|
||||||
|
if a.Key == key {
|
||||||
|
return a.Value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
|
||||||
|
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
|
||||||
|
type Handler interface {
|
||||||
|
Enabled(ctx context.Context, level Level) bool
|
||||||
|
Handle(ctx context.Context, r Record) error
|
||||||
|
// WithAttrs returns a Handler that adds attrs to every record it handles.
|
||||||
|
WithAttrs(attrs []Attr) Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
type multiHandler []Handler
|
||||||
|
|
||||||
|
// NewMultiHandler fans every record out to all handlers that have its level enabled.
|
||||||
|
func NewMultiHandler(handlers ...Handler) Handler {
|
||||||
|
return multiHandler(handlers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, level) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
var errs []error
|
||||||
|
for _, h := range m {
|
||||||
|
if h.Enabled(ctx, r.Level) {
|
||||||
|
errs = append(errs, h.Handle(ctx, r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
out := make(multiHandler, len(m))
|
||||||
|
for i, h := range m {
|
||||||
|
out[i] = h.WithAttrs(attrs)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
type discardHandler struct{}
|
||||||
|
|
||||||
|
// DiscardHandler drops every record.
|
||||||
|
var DiscardHandler Handler = discardHandler{}
|
||||||
|
|
||||||
|
func (discardHandler) Enabled(context.Context, Level) bool { return false }
|
||||||
|
func (discardHandler) Handle(context.Context, Record) error { return nil }
|
||||||
|
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
|
||||||
|
|
||||||
|
type levelHandler struct {
|
||||||
|
h Handler
|
||||||
|
minLevel Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLevelHandler passes only records at or above minLevel on to h.
|
||||||
|
func NewLevelHandler(h Handler, minLevel Level) Handler {
|
||||||
|
return levelHandler{h: h, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
return level >= l.minLevel && l.h.Enabled(ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
if r.Level < l.minLevel {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return l.h.Handle(ctx, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Level is a log severity. The values match log/slog, with Trace added below Debug.
|
||||||
|
type Level int
|
||||||
|
|
||||||
|
const (
|
||||||
|
LevelTrace Level = -8
|
||||||
|
LevelDebug Level = -4
|
||||||
|
LevelInfo Level = 0
|
||||||
|
LevelWarn Level = 4
|
||||||
|
LevelError Level = 8
|
||||||
|
)
|
||||||
|
|
||||||
|
func (l Level) String() string {
|
||||||
|
switch {
|
||||||
|
case l <= LevelTrace:
|
||||||
|
return "trace"
|
||||||
|
case l <= LevelDebug:
|
||||||
|
return "debug"
|
||||||
|
case l <= LevelInfo:
|
||||||
|
return "info"
|
||||||
|
case l <= LevelWarn:
|
||||||
|
return "warn"
|
||||||
|
default:
|
||||||
|
return "error"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
|
||||||
|
// the Logger has no level above it.
|
||||||
|
func ParseLevel(s string) (Level, error) {
|
||||||
|
switch strings.ToLower(s) {
|
||||||
|
case "trace":
|
||||||
|
return LevelTrace, nil
|
||||||
|
case "debug":
|
||||||
|
return LevelDebug, nil
|
||||||
|
case "info":
|
||||||
|
return LevelInfo, nil
|
||||||
|
case "warn", "warning":
|
||||||
|
return LevelWarn, nil
|
||||||
|
case "error", "fatal", "panic":
|
||||||
|
return LevelError, nil
|
||||||
|
}
|
||||||
|
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const badKey = "!BADKEY"
|
||||||
|
|
||||||
|
// Logger writes printf-style messages with structured attributes to a Handler.
|
||||||
|
type Logger struct {
|
||||||
|
h Handler
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(h Handler) *Logger {
|
||||||
|
return &Logger{h: h, ctx: context.Background()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
|
||||||
|
func (l *Logger) With(args ...any) *Logger {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) WithError(err error) *Logger {
|
||||||
|
return l.With("error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithContext returns a Logger that passes ctx to its Handler.
|
||||||
|
func (l *Logger) WithContext(ctx context.Context) *Logger {
|
||||||
|
return &Logger{h: l.h, ctx: ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Enabled(level Level) bool {
|
||||||
|
return l.h.Enabled(l.ctx, level)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
|
||||||
|
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
|
||||||
|
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
|
||||||
|
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
|
||||||
|
|
||||||
|
// Trace and the methods below format their arguments like fmt.Sprint.
|
||||||
|
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
|
||||||
|
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
|
||||||
|
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
|
||||||
|
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
|
||||||
|
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
|
||||||
|
|
||||||
|
func (l *Logger) logf(level Level, format string, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Logger) log(level Level, args []any) {
|
||||||
|
if !l.Enabled(level) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.emit(callerPC(4), level, fmt.Sprint(args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// emit drops the Handler error, as a logger has nowhere to report it.
|
||||||
|
func (l *Logger) emit(pc uintptr, level Level, msg string) {
|
||||||
|
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
|
||||||
|
func callerPC(skip int) uintptr {
|
||||||
|
var pcs [1]uintptr
|
||||||
|
runtime.Callers(skip, pcs[:])
|
||||||
|
return pcs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func argsToAttrs(args []any) []Attr {
|
||||||
|
attrs := make([]Attr, 0, len(args)/2+1)
|
||||||
|
for len(args) > 0 {
|
||||||
|
switch k := args[0].(type) {
|
||||||
|
case Attr:
|
||||||
|
attrs = append(attrs, k)
|
||||||
|
args = args[1:]
|
||||||
|
case string:
|
||||||
|
if len(args) == 1 {
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = nil
|
||||||
|
} else {
|
||||||
|
attrs = append(attrs, Attr{Key: k, Value: args[1]})
|
||||||
|
args = args[2:]
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return attrs
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseLevel(t *testing.T) {
|
||||||
|
for in, want := range map[string]logging.Level{
|
||||||
|
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
|
||||||
|
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
|
||||||
|
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
|
||||||
|
} {
|
||||||
|
got, err := logging.ParseLevel(in)
|
||||||
|
require.NoError(t, err, in)
|
||||||
|
assert.Equal(t, want, got, in)
|
||||||
|
}
|
||||||
|
_, err := logging.ParseLevel("loud")
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, "warn", logging.LevelWarn.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingStringer struct{ n *int }
|
||||||
|
|
||||||
|
func (c countingStringer) String() string { *c.n++; return "x" }
|
||||||
|
|
||||||
|
func TestLogger(t *testing.T) {
|
||||||
|
h := logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
assert.Nil(t, h.LastRecord())
|
||||||
|
l := logging.New(h)
|
||||||
|
|
||||||
|
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
|
||||||
|
l.Info("a", 1, 2, "b")
|
||||||
|
l.With("k").With(7).Debug("x")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 3)
|
||||||
|
assert.Equal(t, "step 2", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
assert.False(t, recs[0].Time.IsZero())
|
||||||
|
job, _ := recs[0].Attr("job")
|
||||||
|
assert.Equal(t, "j1", job)
|
||||||
|
errv, _ := recs[0].Attr("error")
|
||||||
|
require.EqualError(t, errv.(error), "boom")
|
||||||
|
assert.Equal(t, "a1 2b", recs[1].Message)
|
||||||
|
bad, _ := recs[2].Attr("!BADKEY")
|
||||||
|
assert.Equal(t, 7, bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
|
||||||
|
n := 0
|
||||||
|
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
|
||||||
|
l.Debugf("%v", countingStringer{&n})
|
||||||
|
assert.Zero(t, n)
|
||||||
|
assert.False(t, l.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiHandler(t *testing.T) {
|
||||||
|
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
|
||||||
|
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
|
||||||
|
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
|
||||||
|
l.Info("i")
|
||||||
|
l.Error("e")
|
||||||
|
assert.Len(t, all.AllRecords(), 2)
|
||||||
|
require.Len(t, warnOnly.AllRecords(), 1)
|
||||||
|
v, _ := warnOnly.LastRecord().Attr("a")
|
||||||
|
assert.Equal(t, 1, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
|
||||||
|
t.Helper()
|
||||||
|
require.NotZero(t, pc)
|
||||||
|
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCallerPC(t *testing.T) {
|
||||||
|
l, h := logtest.NewLogger()
|
||||||
|
l.Infof("method")
|
||||||
|
l.Info("method")
|
||||||
|
|
||||||
|
prev := logging.Default()
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
logging.SetDefault(l)
|
||||||
|
logging.Infof("global")
|
||||||
|
logging.Error("global")
|
||||||
|
|
||||||
|
require.Len(t, h.AllRecords(), 4)
|
||||||
|
for _, r := range h.AllRecords() {
|
||||||
|
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultLevel(t *testing.T) {
|
||||||
|
prev := logging.GetLevel()
|
||||||
|
t.Cleanup(func() { logging.SetLevel(prev) })
|
||||||
|
|
||||||
|
logging.SetLevel(logging.LevelWarn)
|
||||||
|
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
|
||||||
|
assert.True(t, logging.Enabled(logging.LevelError))
|
||||||
|
assert.False(t, logging.Enabled(logging.LevelInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogHandler(t *testing.T) {
|
||||||
|
var buf strings.Builder
|
||||||
|
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
|
||||||
|
|
||||||
|
assert.False(t, l.Enabled(logging.LevelTrace))
|
||||||
|
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
|
||||||
|
|
||||||
|
out := buf.String()
|
||||||
|
assert.Contains(t, out, "level=ERROR")
|
||||||
|
assert.Contains(t, out, `msg="fail x"`)
|
||||||
|
assert.Contains(t, out, "job=j1")
|
||||||
|
assert.Contains(t, out, "error=boom")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSlogBridge(t *testing.T) {
|
||||||
|
h := logtest.SetDefault(t)
|
||||||
|
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
|
||||||
|
_, _, line, _ := runtime.Caller(0)
|
||||||
|
slog.Debug("dropped")
|
||||||
|
|
||||||
|
recs := h.AllRecords()
|
||||||
|
require.Len(t, recs, 1)
|
||||||
|
assert.Equal(t, "from a library", recs[0].Message)
|
||||||
|
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||||
|
lib, _ := recs[0].Attr("lib")
|
||||||
|
assert.Equal(t, "actionslib", lib)
|
||||||
|
n, _ := recs[0].Attr("n")
|
||||||
|
assert.EqualValues(t, 1, n)
|
||||||
|
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
// Package logtest provides a Handler that keeps records in memory for assertions.
|
||||||
|
package logtest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
|
)
|
||||||
|
|
||||||
|
type store struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
records []logging.Record
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
|
||||||
|
type Handler struct {
|
||||||
|
store *store
|
||||||
|
attrs []logging.Attr
|
||||||
|
minLevel logging.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New records from LevelInfo up, like the logger of logrus's hooks/test.
|
||||||
|
func New() *Handler {
|
||||||
|
return NewWithLevel(logging.LevelInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
|
||||||
|
func NewLogger() (*logging.Logger, *Handler) {
|
||||||
|
h := New()
|
||||||
|
return logging.New(h), h
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWithLevel(minLevel logging.Level) *Handler {
|
||||||
|
return &Handler{store: &store{}, minLevel: minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
|
||||||
|
|
||||||
|
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
|
||||||
|
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
|
||||||
|
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
h.store.records = append(h.store.records, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
|
||||||
|
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) AllRecords() []logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
return slices.Clone(h.store.records)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LastRecord returns nil when nothing was logged.
|
||||||
|
func (h *Handler) LastRecord() *logging.Record {
|
||||||
|
h.store.mu.Lock()
|
||||||
|
defer h.store.mu.Unlock()
|
||||||
|
if len(h.store.records) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r := h.store.records[len(h.store.records)-1]
|
||||||
|
return &r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
|
||||||
|
func SetDefault(t testing.TB) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
prev := logging.Default()
|
||||||
|
logger, h := NewLogger()
|
||||||
|
logging.SetDefault(logger)
|
||||||
|
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||||
|
return h
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
slog.SetDefault(slog.New(slogBridge{}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// slogBridge hands what libraries log through slog's global logger to the process-wide Logger.
|
||||||
|
type slogBridge struct {
|
||||||
|
attrs []Attr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) Enabled(ctx context.Context, level slog.Level) bool {
|
||||||
|
return Default().h.Enabled(ctx, Level(level))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) Handle(ctx context.Context, r slog.Record) error {
|
||||||
|
h := Default().h
|
||||||
|
if len(b.attrs) > 0 {
|
||||||
|
h = h.WithAttrs(b.attrs)
|
||||||
|
}
|
||||||
|
attrs := make([]Attr, 0, r.NumAttrs())
|
||||||
|
r.Attrs(func(a slog.Attr) bool {
|
||||||
|
attrs = append(attrs, fromSlogAttr(a))
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return h.Handle(ctx, Record{Time: r.Time, Level: Level(r.Level), Message: r.Message, PC: r.PC, Attrs: attrs})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||||
|
out := slices.Clone(b.attrs)
|
||||||
|
for _, a := range attrs {
|
||||||
|
out = append(out, fromSlogAttr(a))
|
||||||
|
}
|
||||||
|
return slogBridge{attrs: out}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b slogBridge) WithGroup(string) slog.Handler { return b }
|
||||||
|
|
||||||
|
func fromSlogAttr(a slog.Attr) Attr {
|
||||||
|
return Attr{Key: a.Key, Value: a.Value.Resolve().Any()}
|
||||||
|
}
|
||||||
|
|
||||||
|
type slogHandler struct {
|
||||||
|
h slog.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSlogHandler writes to h. Levels map one to one, as Level uses slog's values.
|
||||||
|
func NewSlogHandler(h slog.Handler) Handler {
|
||||||
|
return slogHandler{h: h}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) Enabled(ctx context.Context, level Level) bool {
|
||||||
|
return s.h.Enabled(ctx, slog.Level(level))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) Handle(ctx context.Context, r Record) error {
|
||||||
|
rec := slog.NewRecord(r.Time, slog.Level(r.Level), r.Message, r.PC)
|
||||||
|
rec.AddAttrs(toSlogAttrs(r.Attrs)...)
|
||||||
|
return s.h.Handle(ctx, rec)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s slogHandler) WithAttrs(attrs []Attr) Handler {
|
||||||
|
return slogHandler{h: s.h.WithAttrs(toSlogAttrs(attrs))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func toSlogAttrs(attrs []Attr) []slog.Attr {
|
||||||
|
out := make([]slog.Attr, len(attrs))
|
||||||
|
for i, a := range attrs {
|
||||||
|
out[i] = slog.Any(a.Key, a.Value)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package logging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"golang.org/x/term"
|
||||||
|
)
|
||||||
|
|
||||||
|
func checkIfTerminal(w io.Writer) bool {
|
||||||
|
if f, ok := w.(*os.File); ok {
|
||||||
|
return term.IsTerminal(int(f.Fd()))
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
Loaded 50 of 247 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user